diff --git a/apps/api-extractor/src/generators/ExcerptBuilder.ts b/apps/api-extractor/src/generators/ExcerptBuilder.ts index ce1662c789..c50814a604 100644 --- a/apps/api-extractor/src/generators/ExcerptBuilder.ts +++ b/apps/api-extractor/src/generators/ExcerptBuilder.ts @@ -13,6 +13,7 @@ import { import { Span } from '../analyzer/Span'; import type { DeclarationReferenceGenerator } from './DeclarationReferenceGenerator'; import type { AstDeclaration } from '../analyzer/AstDeclaration'; +import { condenseTokens } from './condenseTokens'; /** * Used to provide ExcerptBuilder with a list of nodes whose token range we want to capture. @@ -128,7 +129,7 @@ export class ExcerptBuilder { lastAppendedTokenIsSeparator: false }); - ExcerptBuilder._condenseTokens(excerptTokens, captureTokenRanges); + condenseTokens(excerptTokens, captureTokenRanges); } public static createEmptyTokenRange(): IExcerptTokenRange { @@ -251,94 +252,6 @@ export class ExcerptBuilder { excerptTokens.push(excerptToken); } - /** - * Condenses the provided excerpt tokens by merging tokens where possible. Updates the provided token ranges to - * remain accurate after token merging. - * - * @remarks - * For example, suppose we have excerpt tokens ["A", "B", "C"] and a token range [0, 2]. If the excerpt tokens - * are condensed to ["AB", "C"], then the token range would be updated to [0, 1]. Note that merges are only - * performed if they are compatible with the provided token ranges. In the example above, if our token range was - * originally [0, 1], we would not be able to merge tokens "A" and "B". - */ - private static _condenseTokens(excerptTokens: IExcerptToken[], tokenRanges: IExcerptTokenRange[]): void { - // This set is used to quickly lookup a start or end index. - const startOrEndIndices: Set = new Set(); - for (const tokenRange of tokenRanges) { - startOrEndIndices.add(tokenRange.startIndex); - startOrEndIndices.add(tokenRange.endIndex); - } - - for (let currentIndex: number = 1; currentIndex < excerptTokens.length; ++currentIndex) { - while (currentIndex < excerptTokens.length) { - const prevPrevToken: IExcerptToken = excerptTokens[currentIndex - 2]; // May be undefined - const prevToken: IExcerptToken = excerptTokens[currentIndex - 1]; - const currentToken: IExcerptToken = excerptTokens[currentIndex]; - - // The number of excerpt tokens that are merged in this iteration. We need this to determine - // how to update the start and end indices of our token ranges. - let mergeCount: number; - - // There are two types of merges that can occur. We only perform these merges if they are - // compatible with all of our token ranges. - if ( - prevPrevToken && - prevPrevToken.kind === ExcerptTokenKind.Reference && - prevToken.kind === ExcerptTokenKind.Content && - prevToken.text.trim() === '.' && - currentToken.kind === ExcerptTokenKind.Reference && - !startOrEndIndices.has(currentIndex) && - !startOrEndIndices.has(currentIndex - 1) - ) { - // If the current token is a reference token, the previous token is a ".", and the previous- - // previous token is a reference token, then merge all three tokens into a reference token. - // - // For example: Given ["MyNamespace" (R), ".", "MyClass" (R)], tokens "." and "MyClass" might - // be merged into "MyNamespace". The condensed token would be ["MyNamespace.MyClass" (R)]. - prevPrevToken.text += prevToken.text + currentToken.text; - prevPrevToken.canonicalReference = currentToken.canonicalReference; - mergeCount = 2; - currentIndex--; - } else if ( - // If the current and previous tokens are both content tokens, then merge the tokens into a - // single content token. For example: Given ["export ", "declare class"], these tokens - // might be merged into "export declare class". - prevToken.kind === ExcerptTokenKind.Content && - prevToken.kind === currentToken.kind && - !startOrEndIndices.has(currentIndex) - ) { - prevToken.text += currentToken.text; - mergeCount = 1; - } else { - // Otherwise, no merging can occur here. Continue to the next index. - break; - } - - // Remove the now redundant excerpt token(s), as they were merged into a previous token. - excerptTokens.splice(currentIndex, mergeCount); - - // Update the start and end indices for all token ranges based upon how many excerpt - // tokens were merged and in what positions. - for (const tokenRange of tokenRanges) { - if (tokenRange.startIndex > currentIndex) { - tokenRange.startIndex -= mergeCount; - } - - if (tokenRange.endIndex > currentIndex) { - tokenRange.endIndex -= mergeCount; - } - } - - // Clear and repopulate our set with the updated indices. - startOrEndIndices.clear(); - for (const tokenRange of tokenRanges) { - startOrEndIndices.add(tokenRange.startIndex); - startOrEndIndices.add(tokenRange.endIndex); - } - } - } - } - private static _isDeclarationName(name: ts.Identifier): boolean { return ExcerptBuilder._isDeclaration(name.parent) && name.parent.name === name; } diff --git a/apps/api-extractor/src/generators/condenseTokens.ts b/apps/api-extractor/src/generators/condenseTokens.ts new file mode 100644 index 0000000000..ab4ae56316 --- /dev/null +++ b/apps/api-extractor/src/generators/condenseTokens.ts @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { + ExcerptTokenKind, + type IExcerptToken, + type IExcerptTokenRange +} from '@microsoft/api-extractor-model'; + +/** + * Condenses the provided excerpt tokens by merging tokens where possible. Updates the provided token ranges to + * remain accurate after token merging. + * + * @remarks + * For example, suppose we have excerpt tokens ["A", "B", "C"] and a token range [0, 2]. If the excerpt tokens + * are condensed to ["AB", "C"], then the token range would be updated to [0, 1]. Note that merges are only + * performed if they are compatible with the provided token ranges. In the example above, if our token range was + * originally [0, 1], we would not be able to merge tokens "A" and "B". + */ +export function condenseTokens(excerptTokens: IExcerptToken[], tokenRanges: IExcerptTokenRange[]): void { + const originalTokenCount: number = excerptTokens.length; + + // A token that sits at the start or end index of any token range must be preserved (never merged + // away), so that every range can be accurately remapped after condensing. These indices refer to + // the original positions in `excerptTokens`, and since preserved tokens are never removed, this + // set never needs to be rebuilt. + const preservedIndices: Set = new Set(); + for (const tokenRange of tokenRanges) { + preservedIndices.add(tokenRange.startIndex); + preservedIndices.add(tokenRange.endIndex); + } + + // Build the condensed token list in a single forward pass, treating it as a stack so that a + // token merged into its predecessor can itself be merged into a further predecessor (e.g. a + // chain of reference tokens such as "A" "." "B" "." "C"). + // + // `newIndexByOriginalIndex` maps each kept token's original index to its index in the condensed + // list, which is then used to remap the token ranges. Every range boundary refers to a preserved + // token (or, for an exclusive `endIndex`, the token count), and preserved tokens are never merged + // away, so a direct lookup always resolves. It is sized `originalTokenCount + 1` to hold the + // mapping for an `endIndex` equal to the token count. + const condensedTokens: IExcerptToken[] = []; + const newIndexByOriginalIndex: Int32Array = new Int32Array(originalTokenCount + 1); + + // Whether the token currently on top of the stack is preserved. Only consulted when that token is + // the "." of a reference merge; because a "." only ever becomes the top via a push (content tokens + // are always separated by references, so a "." is never uncovered by a pop), this scalar is always + // up to date at the point it is read, avoiding a parallel array of original indices. + let prevTokenIsPreserved: boolean = false; + + for (let currentIndex: number = 0; currentIndex < originalTokenCount; ++currentIndex) { + const currentToken: IExcerptToken = excerptTokens[currentIndex]; + const currentIsPreserved: boolean = preservedIndices.has(currentIndex); + const condensedCount: number = condensedTokens.length; + + // A preserved token must never be merged away, so merges are only attempted when the current + // token is not preserved. There are two types of merges that can occur, and both consume the + // current token. Reads of the top two stack entries are guarded so they never index out of + // bounds. + let merged: boolean = false; + if (!currentIsPreserved && condensedCount >= 1) { + const prevToken: IExcerptToken = condensedTokens[condensedCount - 1]; + + if ( + condensedCount >= 2 && + currentToken.kind === ExcerptTokenKind.Reference && + prevToken.kind === ExcerptTokenKind.Content && + prevToken.text.trim() === '.' && + !prevTokenIsPreserved && + condensedTokens[condensedCount - 2].kind === ExcerptTokenKind.Reference + ) { + // If the current token is a reference token, the previous token is a ".", and the previous- + // previous token is a reference token, then merge all three tokens into a reference token. + // + // For example: Given ["MyNamespace" (R), ".", "MyClass" (R)], tokens "." and "MyClass" might + // be merged into "MyNamespace". The condensed token would be ["MyNamespace.MyClass" (R)]. + const prevPrevToken: IExcerptToken = condensedTokens[condensedCount - 2]; + prevPrevToken.text += prevToken.text + currentToken.text; + prevPrevToken.canonicalReference = currentToken.canonicalReference; + + // The "." token (already kept) and the current token are both merged into prevPrevToken. + condensedTokens.pop(); + merged = true; + } else if ( + // If the current and previous tokens are both content tokens, then merge the tokens into a + // single content token. For example: Given ["export ", "declare class"], these tokens + // might be merged into "export declare class". + prevToken.kind === ExcerptTokenKind.Content && + currentToken.kind === ExcerptTokenKind.Content + ) { + prevToken.text += currentToken.text; + merged = true; + } + } + + if (!merged) { + // No merging occurred, so keep the current token, record its new index, and update the + // preservation flag to reflect the new top of the stack. + newIndexByOriginalIndex[currentIndex] = condensedTokens.length; + condensedTokens.push(currentToken); + prevTokenIsPreserved = currentIsPreserved; + } + } + + // Remap the token ranges directly. Each boundary is a preserved token's original index (or the + // token count, for an exclusive `endIndex`), which maps straight to its position in the condensed + // list. `endIndex` is clamped because it may equal the token count. + newIndexByOriginalIndex[originalTokenCount] = condensedTokens.length; + for (const tokenRange of tokenRanges) { + tokenRange.startIndex = newIndexByOriginalIndex[Math.min(tokenRange.startIndex, originalTokenCount)]; + tokenRange.endIndex = newIndexByOriginalIndex[Math.min(tokenRange.endIndex, originalTokenCount)]; + } + + // Replace the excerpt tokens in place with the condensed list. + excerptTokens.length = 0; + for (const token of condensedTokens) { + excerptTokens.push(token); + } +} diff --git a/apps/api-extractor/src/generators/test/condenseTokens.test.ts b/apps/api-extractor/src/generators/test/condenseTokens.test.ts new file mode 100644 index 0000000000..f445412450 --- /dev/null +++ b/apps/api-extractor/src/generators/test/condenseTokens.test.ts @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { + ExcerptTokenKind, + type IExcerptToken, + type IExcerptTokenRange +} from '@microsoft/api-extractor-model'; + +import { condenseTokens } from '../condenseTokens'; + +function content(text: string): IExcerptToken { + return { kind: ExcerptTokenKind.Content, text }; +} + +function reference(text: string, canonicalReference?: string): IExcerptToken { + return { kind: ExcerptTokenKind.Reference, text, canonicalReference }; +} + +/** + * A deliberately naive reference implementation of the token-condensing algorithm, used as an oracle + * in the randomized test below. It mirrors the original (pre-optimization) behavior: repeatedly + * splicing merged tokens out of the array and re-deriving the set of protected indices after each + * merge. It is intentionally O(n^2) and simple so that it is easy to verify by inspection. + */ +function condenseTokensReference(excerptTokens: IExcerptToken[], tokenRanges: IExcerptTokenRange[]): void { + const startOrEndIndices: Set = new Set(); + for (const tokenRange of tokenRanges) { + startOrEndIndices.add(tokenRange.startIndex); + startOrEndIndices.add(tokenRange.endIndex); + } + + for (let currentIndex: number = 1; currentIndex < excerptTokens.length; ++currentIndex) { + while (currentIndex < excerptTokens.length) { + const prevPrevToken: IExcerptToken = excerptTokens[currentIndex - 2]; + const prevToken: IExcerptToken = excerptTokens[currentIndex - 1]; + const currentToken: IExcerptToken = excerptTokens[currentIndex]; + + let mergeCount: number; + if ( + prevPrevToken && + prevPrevToken.kind === ExcerptTokenKind.Reference && + prevToken.kind === ExcerptTokenKind.Content && + prevToken.text.trim() === '.' && + currentToken.kind === ExcerptTokenKind.Reference && + !startOrEndIndices.has(currentIndex) && + !startOrEndIndices.has(currentIndex - 1) + ) { + prevPrevToken.text += prevToken.text + currentToken.text; + prevPrevToken.canonicalReference = currentToken.canonicalReference; + mergeCount = 2; + currentIndex--; + } else if ( + prevToken.kind === ExcerptTokenKind.Content && + prevToken.kind === currentToken.kind && + !startOrEndIndices.has(currentIndex) + ) { + prevToken.text += currentToken.text; + mergeCount = 1; + } else { + break; + } + + excerptTokens.splice(currentIndex, mergeCount); + for (const tokenRange of tokenRanges) { + if (tokenRange.startIndex > currentIndex) { + tokenRange.startIndex -= mergeCount; + } + if (tokenRange.endIndex > currentIndex) { + tokenRange.endIndex -= mergeCount; + } + } + + startOrEndIndices.clear(); + for (const tokenRange of tokenRanges) { + startOrEndIndices.add(tokenRange.startIndex); + startOrEndIndices.add(tokenRange.endIndex); + } + } + } +} + +describe('condenseTokens', () => { + it('merges adjacent content tokens', () => { + const tokens: IExcerptToken[] = [content('export '), content('declare '), content('class')]; + const ranges: IExcerptTokenRange[] = []; + + condenseTokens(tokens, ranges); + + expect(tokens).toEqual([content('export declare class')]); + }); + + it('does not merge content into an adjacent reference token', () => { + const tokens: IExcerptToken[] = [reference('Foo', 'foo'), content(' bar')]; + const ranges: IExcerptTokenRange[] = []; + + condenseTokens(tokens, ranges); + + expect(tokens).toEqual([reference('Foo', 'foo'), content(' bar')]); + }); + + it('merges a "Reference . Reference" sequence into a single reference token', () => { + const tokens: IExcerptToken[] = [ + reference('MyNamespace', 'ns'), + content('.'), + reference('MyClass', 'cls') + ]; + const ranges: IExcerptTokenRange[] = []; + + condenseTokens(tokens, ranges); + + // The canonical reference of the last reference token wins. + expect(tokens).toEqual([reference('MyNamespace.MyClass', 'cls')]); + }); + + it('merges a chain of reference tokens', () => { + const tokens: IExcerptToken[] = [ + reference('A', 'a'), + content('.'), + reference('B', 'b'), + content('.'), + reference('C', 'c') + ]; + const ranges: IExcerptTokenRange[] = []; + + condenseTokens(tokens, ranges); + + expect(tokens).toEqual([reference('A.B.C', 'c')]); + }); + + it('remaps a token range after merging preceding tokens', () => { + // ["A", "B", "C"] with range [0, 2) should become ["AB", "C"] with range [0, 1). + const tokens: IExcerptToken[] = [content('A'), content('B'), content('C')]; + const ranges: IExcerptTokenRange[] = [{ startIndex: 0, endIndex: 2 }]; + + condenseTokens(tokens, ranges); + + expect(tokens).toEqual([content('AB'), content('C')]); + expect(ranges).toEqual([{ startIndex: 0, endIndex: 1 }]); + }); + + it('does not merge across a range boundary', () => { + // With range [0, 1), token "B" is protected as the exclusive end, so "A" and "B" cannot merge. + const tokens: IExcerptToken[] = [content('A'), content('B'), content('C')]; + const ranges: IExcerptTokenRange[] = [{ startIndex: 0, endIndex: 1 }]; + + condenseTokens(tokens, ranges); + + expect(tokens).toEqual([content('A'), content('BC')]); + expect(ranges).toEqual([{ startIndex: 0, endIndex: 1 }]); + }); + + it('does not merge a reference chain when the "." token is a range boundary', () => { + const tokens: IExcerptToken[] = [reference('A', 'a'), content('.'), reference('B', 'b')]; + // The "." token (index 1) is the start of a range, so the reference merge must not occur. + const ranges: IExcerptTokenRange[] = [{ startIndex: 1, endIndex: 3 }]; + + condenseTokens(tokens, ranges); + + expect(tokens).toEqual([reference('A', 'a'), content('.'), reference('B', 'b')]); + expect(ranges).toEqual([{ startIndex: 1, endIndex: 3 }]); + }); + + it('handles a "." token that is itself a merged whitespace + dot content token', () => { + // The leading whitespace content token and the "." content token merge into " .", which then + // participates in the reference merge. The surviving "." token originates from the whitespace + // token's index, which is what protects the range remapping. + const tokens: IExcerptToken[] = [reference('A', 'a'), content(' '), content('.'), reference('B', 'b')]; + const ranges: IExcerptTokenRange[] = []; + + condenseTokens(tokens, ranges); + + expect(tokens).toEqual([reference('A .B', 'b')]); + }); + + it('remaps an exclusive endIndex equal to the token count', () => { + const tokens: IExcerptToken[] = [content('a'), content('b'), content('c')]; + const ranges: IExcerptTokenRange[] = [{ startIndex: 0, endIndex: 3 }]; + + condenseTokens(tokens, ranges); + + expect(tokens).toEqual([content('abc')]); + expect(ranges).toEqual([{ startIndex: 0, endIndex: 1 }]); + }); + + it('handles empty input', () => { + const tokens: IExcerptToken[] = []; + const ranges: IExcerptTokenRange[] = [{ startIndex: 0, endIndex: 0 }]; + + condenseTokens(tokens, ranges); + + expect(tokens).toEqual([]); + expect(ranges).toEqual([{ startIndex: 0, endIndex: 0 }]); + }); + + it('matches the reference implementation across many randomized inputs', () => { + let seed: number = 0x1234abcd; + // A small deterministic PRNG so the test is reproducible. + const nextRandom = (): number => { + seed ^= seed << 13; + seed ^= seed >>> 17; + seed ^= seed << 5; + return ((seed >>> 0) % 100000) / 100000; + }; + + const makeToken = (index: number): IExcerptToken => { + const roll: number = nextRandom(); + if (roll < 0.4) { + return reference('Ref' + index, 'cr' + index); + } + if (roll < 0.6) { + return content('.'); + } + if (roll < 0.78) { + return content(' '); + } + return content('txt' + index); + }; + + for (let iteration: number = 0; iteration < 5000; ++iteration) { + const tokenCount: number = Math.floor(nextRandom() * 10); + const baseTokens: IExcerptToken[] = []; + for (let i: number = 0; i < tokenCount; ++i) { + baseTokens.push(makeToken(i)); + } + + const rangeCount: number = Math.floor(nextRandom() * 3); + const baseRanges: IExcerptTokenRange[] = []; + for (let r: number = 0; r < rangeCount; ++r) { + const startIndex: number = Math.floor(nextRandom() * (tokenCount + 1)); + const endIndex: number = Math.min( + tokenCount, + startIndex + Math.floor(nextRandom() * (tokenCount + 1)) + ); + baseRanges.push({ startIndex, endIndex }); + } + + const actualTokens: IExcerptToken[] = baseTokens.map((token) => ({ ...token })); + const actualRanges: IExcerptTokenRange[] = baseRanges.map((range) => ({ ...range })); + condenseTokens(actualTokens, actualRanges); + + const expectedTokens: IExcerptToken[] = baseTokens.map((token) => ({ ...token })); + const expectedRanges: IExcerptTokenRange[] = baseRanges.map((range) => ({ ...range })); + condenseTokensReference(expectedTokens, expectedRanges); + + expect(actualTokens).toEqual(expectedTokens); + expect(actualRanges).toEqual(expectedRanges); + } + }); +}); diff --git a/common/changes/@microsoft/api-extractor/condense-tokens-optimization_2026-07-16-23-06-45.json b/common/changes/@microsoft/api-extractor/condense-tokens-optimization_2026-07-16-23-06-45.json new file mode 100644 index 0000000000..5d562b7281 --- /dev/null +++ b/common/changes/@microsoft/api-extractor/condense-tokens-optimization_2026-07-16-23-06-45.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "comment": "Improve the performance of the internal excerpt token condensing algorithm from O(n^2) to O(n) by eliminating repeated array splicing and per-merge bookkeeping.", + "type": "patch", + "packageName": "@microsoft/api-extractor" + } + ], + "packageName": "@microsoft/api-extractor", + "email": "dmichon-msft@users.noreply.github.com" +}