diff --git a/schema/src/classes/bundle.ts b/schema/src/classes/bundle.ts index fb3ab69..83f630f 100644 --- a/schema/src/classes/bundle.ts +++ b/schema/src/classes/bundle.ts @@ -1,6 +1,7 @@ import { IBundle, IStrand, Meta, Source, IVersioning, Searchable, SearchOptions, SearchResult, Identifier, Symbol, splitPath } from '../interfaces'; -import { Function, MethodMap, EnumStruct, Constant, Define, Enumeration, TypeDefinition, TypeSet } from './symbol'; +import { Function, MethodMap, EnumStruct, Constant, Define, Enumeration, TypeDefinition, TypeSet, Declaration, SearchEntry } from './symbol'; import { ClassSymbol } from './symbol/types'; +import { getSearchIndex } from './search_index'; export class Bundle implements IBundle, Searchable { /** @@ -35,17 +36,19 @@ export class Bundle implements IBundle, Searchable { this.version = bundle.version; } + /** + * @brief Symbols scoring above MIN_SCORE across all strands + * @note Indexes are built on first search, changes to strands afterwards are not searched + */ public async search(needle: string, options: SearchOptions): Promise { - const ret: Promise[] = []; - - for (const [include, strand] of Object.entries(this.strands)) { - ret.push(strand.search(needle, { - ...options, - parents: [...options.parents, include], - })); - } + return getSearchIndex(this, options, o => this.searchEntries(o)).search(needle, options.parents); + } - return (await Promise.all(ret)).flat(); + public searchEntries(options: Readonly): SearchEntry[] { + return Object.entries(this.strands).flatMap(([include, strand]) => strand.searchEntries({ + ...options, + parents: [...options.parents, include], + })); } public getSymbolByPath(p: readonly string[]): ClassSymbol { @@ -86,26 +89,25 @@ export class Strand implements IStrand, Searchable { this.typedefs = Strand.mapFibers(strand.typedefs, TypeDefinition); } + /** + * @brief Symbols scoring above MIN_SCORE + * @note Indexes are built on first search, changes to the strand afterwards are not searched + */ public async search(needle: string, options: Readonly): Promise { - const ret: Promise[] = []; - - const searchSymbolType = (member: Record) => { - for (const f of Object.values(member)) { - ret.push(f.search(needle, options)); - } - } + return getSearchIndex(this, options, o => this.searchEntries(o)).search(needle, options.parents); + } - searchSymbolType(this.functions); - searchSymbolType(this.methodmaps); - searchSymbolType(this.enumstructs); - searchSymbolType(this.constants); - searchSymbolType(this.defines); - searchSymbolType(this.enums); - searchSymbolType(this.typesets); - searchSymbolType(this.typedefs); - - // Return at least somewhat similar results - return (await Promise.all(ret)).flat().filter(e => e.score > 0.5); + public searchEntries(options: Readonly): SearchEntry[] { + return [ + this.functions, + this.methodmaps, + this.enumstructs, + this.constants, + this.defines, + this.enums, + this.typesets, + this.typedefs, + ].flatMap(member => Object.values(member).flatMap((symbol: Declaration) => symbol.searchEntries(options))); } public getSymbolByPath(p: readonly string[]): ClassSymbol { diff --git a/schema/src/classes/index.ts b/schema/src/classes/index.ts index 9f1fed5..7e07d4b 100644 --- a/schema/src/classes/index.ts +++ b/schema/src/classes/index.ts @@ -1,2 +1,3 @@ export * from './symbol'; export * from './bundle'; +export { SearchIndex, MIN_SCORE } from './search_index'; diff --git a/schema/src/classes/search_index.ts b/schema/src/classes/search_index.ts new file mode 100644 index 0000000..2e8ff3c --- /dev/null +++ b/schema/src/classes/search_index.ts @@ -0,0 +1,279 @@ +import { SearchOptions, SearchResult } from '../interfaces'; +import { SearchEntry, calculateScore, toSearchResult, trackedEntries, recordRecentAdditions } from './symbol/base'; + +/** + * @brief Bundle and strand searches only return results scoring above this + */ +export const MIN_SCORE = 0.5; + +// Slack for float rounding when pruning, candidates are rescored exactly +const EPSILON = 1e-9; + +/** + * @brief Inverted index over search entries + * + * calculateScore ranks a term by exact match, case-insensitive match, + * case-insensitive substring, and otherwise by Sørensen-Dice similarity of + * bigrams. A term can only clear MIN_SCORE by containing the needle or by + * sharing enough bigrams with it, so rather than scoring every entry: + * + * - Entries are grouped by term, as many share one (int, void, Handle, ...), + * and each distinct term is scored at most once per search. + * - Substring candidates are read off the posting list of the needle's rarest + * lowercase bigram, since a term containing the needle contains all of them. + * - Dice candidates are found by summing shared bigrams over the needle's + * posting lists, which yields the exact coefficient, and keeping the terms + * that could still clear MIN_SCORE with the largest bonus in the index. + * + * Candidates are then scored with calculateScore, so results are identical + * to scoring every entry. + */ +export class SearchIndex { + private readonly entries: SearchEntry[]; + + // Term id of each entry + private readonly entryTerms: number[] = []; + + private readonly terms: string[] = []; + + private readonly lowerTerms: string[] = []; + + // Entry indices of each term + private readonly termEntries: number[][] = []; + + // Bigram count of each term with whitespace removed + private readonly bigramCounts: number[] = []; + + // Case-sensitive bigrams of terms with whitespace removed, as compared by Dice + private readonly bigrams = new Map(); + + // Distinct lowercase bigrams of terms, as compared by substring + private readonly lowerBigrams = new Map(); + + // Largest boost + weight of any entry + private readonly maxBonus: number; + + // Entries counting towards recent additions, in processing order + private readonly tracked: number[]; + + // Scratch space for shared bigram counts, zeroed after each search + private readonly shared: Int32Array; + + public constructor(entries: SearchEntry[]) { + this.entries = entries; + + const termIds = new Map(); + + let maxBonus = 0; + + entries.forEach((entry, i) => { + let id = termIds.get(entry.term); + + if (id === undefined) { + id = this.addTerm(entry.term); + termIds.set(entry.term, id); + } + + this.entryTerms.push(id); + this.termEntries[id].push(i); + maxBonus = Math.max(maxBonus, entry.boost + entry.weight); + }); + + this.maxBonus = maxBonus; + this.tracked = trackedEntries(entries); + this.shared = new Int32Array(this.terms.length); + } + + /** + * @brief Search entries scoring above MIN_SCORE, in entry order + * + * @param parents Prepended to the path of each result + */ + public search(needle: string, parents: readonly string[] = []): SearchResult[] { + const termScores = this.scoreTerms(needle); + + const matches: number[] = []; + + for (const [id, termScore] of termScores) { + for (const i of this.termEntries[id]) { + const entry = this.entries[i]; + + // Same expression as toSearchResult + if (termScore + entry.boost + entry.weight > MIN_SCORE) { + matches.push(i); + } + } + } + + matches.sort((a, b) => a - b); + + const termScore = (i: number) => { + const id = this.entryTerms[i]; + + return termScores.get(id) ?? calculateScore(this.terms[id], needle); + }; + + recordRecentAdditions(this.entries, this.tracked, i => toSearchResult(this.entries[i], termScore(i), parents)); + + return matches.map(i => toSearchResult(this.entries[i], termScore(i), parents)); + } + + /** + * @brief calculateScore of every term that may clear MIN_SCORE, by term id + */ + private scoreTerms(needle: string): Map { + const ret = new Map(); + + const score = (id: number) => { + if (!ret.has(id)) { + ret.set(id, calculateScore(this.terms[id], needle)); + } + }; + + const minTermScore = MIN_SCORE - this.maxBonus - EPSILON; + + const lower = needle.toLowerCase(); + const stripped = needle.replace(/\s+/g, ''); + + // Nothing to narrow down with, score everything + if (minTermScore <= 0 || lower.length < 2 || stripped.length < 2) { + this.terms.forEach((_, id) => score(id)); + + return ret; + } + + // Terms containing the needle + let rarest: number[] | undefined; + + for (const bigram of countBigrams(lower).keys()) { + const posting = this.lowerBigrams.get(bigram) ?? []; + + if (rarest === undefined || posting.length < rarest.length) { + rarest = posting; + } + } + + for (const id of rarest!) { + if (this.lowerTerms[id].includes(lower)) { + score(id); + } + } + + // Terms similar enough to the needle + const touched: number[] = []; + + for (const [bigram, count] of countBigrams(stripped)) { + const posting = this.bigrams.get(bigram); + + if (posting === undefined) { + continue; + } + + for (let j = 0; j < posting.terms.length; j++) { + const id = posting.terms[j]; + + if (this.shared[id] === 0) { + touched.push(id); + } + + this.shared[id] += Math.min(count, posting.counts[j]); + } + } + + const needleBigrams = stripped.length - 1; + + for (const id of touched) { + if ((2.0 * this.shared[id]) / (this.bigramCounts[id] + needleBigrams) > minTermScore) { + score(id); + } + + this.shared[id] = 0; + } + + return ret; + } + + private addTerm(term: string): number { + const id = this.terms.length; + + const lower = term.toLowerCase(); + const stripped = term.replace(/\s+/g, ''); + + this.terms.push(term); + this.lowerTerms.push(lower); + this.termEntries.push([]); + this.bigramCounts.push(Math.max(stripped.length - 1, 0)); + + for (const [bigram, count] of countBigrams(stripped)) { + let posting = this.bigrams.get(bigram); + + if (posting === undefined) { + posting = { terms: [], counts: [] }; + this.bigrams.set(bigram, posting); + } + + posting.terms.push(id); + posting.counts.push(count); + } + + for (const bigram of countBigrams(lower).keys()) { + let posting = this.lowerBigrams.get(bigram); + + if (posting === undefined) { + posting = []; + this.lowerBigrams.set(bigram, posting); + } + + posting.push(id); + } + + return id; + } +} + +/** + * @brief Occurrences of each bigram, keyed by its two UTF-16 code units + */ +function countBigrams(s: string): Map { + const ret = new Map(); + + for (let i = 0; i < s.length - 1; i++) { + const bigram = s.charCodeAt(i) * 0x10000 + s.charCodeAt(i + 1); + + ret.set(bigram, (ret.get(bigram) ?? 0) + 1); + } + + return ret; +} + +const indexes = new WeakMap>(); + +/** + * @brief Index of owner's entries for the given options, built on first use + * + * Indexes are cached per owner and per combination of options that affects + * entries or scores. Parents are applied per search and don't need their own. + */ +export function getSearchIndex( + owner: object, + options: Readonly, + collect: (options: Readonly) => SearchEntry[], +): SearchIndex { + let cache = indexes.get(owner); + + if (cache === undefined) { + cache = new Map(); + indexes.set(owner, cache); + } + + const key = JSON.stringify([options.weighted !== false, options.l1Only === true, options.identifier || null]); + + let index = cache.get(key); + + if (index === undefined) { + index = new SearchIndex(collect({ ...options, parents: [] })); + cache.set(key, index); + } + + return index; +} diff --git a/schema/src/classes/symbol/base.ts b/schema/src/classes/symbol/base.ts index d0c160b..133a21b 100644 --- a/schema/src/classes/symbol/base.ts +++ b/schema/src/classes/symbol/base.ts @@ -61,36 +61,145 @@ export class Declaration implements IDeclaration, Searchable { } public async search(needle: string, options: Readonly): Promise { - const localOptions = JSON.parse(JSON.stringify(options)); - - const path = [...localOptions.parents, `${this.identifier}.${this.name}`]; + return scoreEntries(this.searchEntries(options), needle); + } - const ret = { + /** + * @brief Everything a search can match on this symbol, without scores + */ + public searchEntries(options: Readonly): SearchEntry[] { + const entry: SearchEntry = { name: this.name, + term: this.name, identifier: this.identifier, part: Part.Name, - path, - score: calculateScore(this.name, needle), + path: [...options.parents, `${this.identifier}.${this.name}`], + boost: 0, + weight: 0, }; if (this.metadata !== null && this.metadata.created !== null) { - processAddition(this.metadata, ret); + entry.metadata = this.metadata; + entry.trackOrder = 0; } - return [ret]; + return [entry]; } } -async function processAddition(metadata: Metadata, sr: SearchResult) { +/** + * @brief Unscored search candidate + * + * Final score is `calculateScore(term, needle) + boost + weight`. + */ +export interface SearchEntry { + /** + * @brief Name reported in the search result + */ + name: string; + + /** + * @brief String the needle is scored against, usually the same as name + */ + term: string; + + identifier: Identifier; + + part: Part; + + path: string[]; + + /** + * @brief Bonus applied before the identifier weight + */ + boost: number; + + /** + * @brief Identifier weight, 0 if unweighted + */ + weight: number; + + /** + * @brief Set on declaration names that count towards recent additions + */ + metadata?: Metadata; + + /** + * @brief Order recent additions are processed in, lower first + */ + trackOrder?: number; +} + +/** + * @param termScore calculateScore of the entry term against the needle + * @param parents Prepended to the entry path + */ +export function toSearchResult(entry: SearchEntry, termScore: number, parents: readonly string[] = []): SearchResult { + return { + name: entry.name, + identifier: entry.identifier, + part: entry.part, + path: [...parents, ...entry.path], + score: termScore + entry.boost + entry.weight, + }; +} + +function scoreEntries(entries: SearchEntry[], needle: string): SearchResult[] { + const ret = entries.map(e => toSearchResult(e, calculateScore(e.term, needle))); + + recordRecentAdditions(entries, trackedEntries(entries), i => ret[i]); + + return ret; +} + +/** + * @param tracked Entry indices from trackedEntries + * @param result Search result of the entry at the given index + */ +export function recordRecentAdditions( + entries: readonly SearchEntry[], + tracked: readonly number[], + result: (i: number) => SearchResult, +) { + if (RecentFinalized) + return; + + for (const i of tracked) { + processAddition(entries[i].metadata!, result(i)); + } +} + +/** + * @brief Indices of entries that count towards recent additions, in processing order + * + * Declarations are processed first, then nested methods round-robin across + * their parents. This mirrors the order the previous async search visited them. + */ +export function trackedEntries(entries: readonly SearchEntry[]): number[] { + const ret: number[] = []; + + entries.forEach((e, i) => { + if (e.metadata !== undefined) { + ret.push(i); + } + }); + + // Array.prototype.sort is stable + return ret.sort((a, b) => entries[a].trackOrder! - entries[b].trackOrder!); +} + +function processAddition(metadata: Metadata, sr: SearchResult) { if (RecentFinalized) return; // Keep only the 20 highest created timestamp metadata if (RecentAdditions.length < 20) { - return RecentAdditions.push({ + RecentAdditions.push({ sr, metadata, }); + + return; } const sorted = RecentAdditions.sort((a, b) => { diff --git a/schema/src/classes/symbol/enum_struct.ts b/schema/src/classes/symbol/enum_struct.ts index 9813d22..15b2985 100644 --- a/schema/src/classes/symbol/enum_struct.ts +++ b/schema/src/classes/symbol/enum_struct.ts @@ -1,6 +1,6 @@ -import { Declaration, calculateScore } from './base'; +import { Declaration, SearchEntry } from './base'; import { Function } from './function'; -import { IEnumStruct, IField, Searchable, SearchResult, SearchOptions, Identifier, IdentifierWeights, Part } from '../../interfaces'; +import { IEnumStruct, IField, Searchable, SearchOptions, Identifier, IdentifierWeights, Part } from '../../interfaces'; export class EnumStruct extends Declaration implements IEnumStruct, Searchable { /** @@ -33,43 +33,42 @@ export class EnumStruct extends Declaration implements IEnumStruct, Searchable { }, {} as Record); } - public async search(needle: string, options: Readonly): Promise { - const localOptions = JSON.parse(JSON.stringify(options)); + public searchEntries(options: Readonly): SearchEntry[] { + const ret = super.searchEntries(options); - let ret = [ - ...await super.search(needle, localOptions), - ]; + ret[0].boost += 0.01; - ret[0].score += 0.01; + const parents = [...options.parents, `${this.identifier}.${this.name}`]; - localOptions.parents.push(`${this.identifier}.${this.name}`); - - if (localOptions.l1Only !== true) { - for (const method of Object.values(this.methods)) { - ret.push(...await method.search(needle, { - ...localOptions, + if (options.l1Only !== true) { + Object.values(this.methods).forEach((method, i) => { + const entries = method.searchEntries({ + ...options, + parents, weighted: false, identifier: Identifier.EnumStructMethod, - })); - } + }); + + entries[0].trackOrder = i + 1; + + ret.push(...entries); + }); for (const field of Object.values(this.fields)) { ret.push({ name: field.name, + term: field.name, identifier: Identifier.EnumStructField, part: Part.Name, - path: [...localOptions.parents, `${Identifier.EnumStructField}.${field.name}`], - score: calculateScore(field.name, needle), + path: [...parents, `${Identifier.EnumStructField}.${field.name}`], + boost: 0, + weight: 0, }); } } - if (localOptions.weighted !== false) { - ret = ret.map(e => { - e.score += IdentifierWeights.EnumStruct; - - return e; - }); + if (options.weighted !== false) { + ret.forEach(e => e.weight += IdentifierWeights.EnumStruct); } return ret; diff --git a/schema/src/classes/symbol/enumeration.ts b/schema/src/classes/symbol/enumeration.ts index f23b09f..cc097fd 100644 --- a/schema/src/classes/symbol/enumeration.ts +++ b/schema/src/classes/symbol/enumeration.ts @@ -1,5 +1,5 @@ -import { Declaration, calculateScore } from './base'; -import { IEnumeration, IEntry, Searchable, Identifier, IdentifierWeights, SearchOptions, SearchResult, Part } from '../../interfaces'; +import { Declaration, SearchEntry } from './base'; +import { IEnumeration, IEntry, Searchable, Identifier, IdentifierWeights, SearchOptions, Part } from '../../interfaces'; export class Enumeration extends Declaration implements IEnumeration, Searchable { /** @@ -20,35 +20,29 @@ export class Enumeration extends Declaration implements IEnumeration, Searchable }, {} as Record); } - public async search(needle: string, options: Readonly): Promise { - const localOptions = JSON.parse(JSON.stringify(options)); + public searchEntries(options: Readonly): SearchEntry[] { + const ret = super.searchEntries(options); - let ret = [ - ...await super.search(needle, localOptions), - ]; + ret[0].boost += 0.01; - ret[0].score += 0.01; + const parents = [...options.parents, `${this.identifier}.${this.name}`]; - localOptions.parents.push(`${this.identifier}.${this.name}`); - - if (localOptions.l1Only !== true) { + if (options.l1Only !== true) { for (const entry of Object.values(this.entries)) { ret.push({ name: entry.name, + term: entry.name, identifier: Identifier.EnumerationEntry, part: Part.Name, - path: [...localOptions.parents, `${Identifier.EnumerationEntry}.${entry.name}`], - score: calculateScore(entry.name, needle), + path: [...parents, `${Identifier.EnumerationEntry}.${entry.name}`], + boost: 0, + weight: 0, }); } } - if (localOptions.weighted !== false) { - ret = ret.map(e => { - e.score += IdentifierWeights.Enumeration; - - return e; - }); + if (options.weighted !== false) { + ret.forEach(e => e.weight += IdentifierWeights.Enumeration); } return ret; diff --git a/schema/src/classes/symbol/function.ts b/schema/src/classes/symbol/function.ts index 6c09b87..dc06c3e 100644 --- a/schema/src/classes/symbol/function.ts +++ b/schema/src/classes/symbol/function.ts @@ -1,5 +1,5 @@ -import { IFunction, FunctionKind, IArgument, Identifier, Part, SearchResult, Searchable, SearchOptions, IdentifierWeights } from '../../interfaces'; -import { Declaration, calculateScore } from './base'; +import { IFunction, FunctionKind, IArgument, Identifier, Part, Searchable, SearchOptions, IdentifierWeights } from '../../interfaces'; +import { Declaration, SearchEntry } from './base'; export class Function extends Declaration implements IFunction, Searchable { /** @@ -34,41 +34,37 @@ export class Function extends Declaration implements IFunction, Searchable { } } - public async search(needle: string, options: Readonly): Promise { - const localOptions = JSON.parse(JSON.stringify(options)); + public searchEntries(options: Readonly): SearchEntry[] { + const identifier: Identifier = options.identifier ? options.identifier : this.identifier; - const identifier: Identifier = localOptions.identifier ? localOptions.identifier : this.identifier; + const ret = super.searchEntries(options); - let ret: SearchResult[] = [ - ...await super.search(needle, localOptions), - ]; - - localOptions.parents.push(`${identifier}.${this.name}`); + const parents = [...options.parents, `${identifier}.${this.name}`]; for (const arg of this.arguments) { ret.push({ name: arg.type, + term: arg.type, identifier, part: Part.Parameter, - path: [...localOptions.parents, `${Identifier.Argument}.${arg.name}`], - score: calculateScore(arg.type, needle), + path: [...parents, `${Identifier.Argument}.${arg.name}`], + boost: 0, + weight: 0, }); } ret.push({ name: this.returnType, + term: this.returnType, identifier, part: Part.Return, - path: [...localOptions.parents, `${Identifier.Return}.${this.returnType}`], - score: calculateScore(this.returnType, needle), + path: [...parents, `${Identifier.Return}.${this.returnType}`], + boost: 0, + weight: 0, }); - if (localOptions.weighted !== false) { - ret = ret.map(e => { - e.score += IdentifierWeights.Function; - - return e; - }); + if (options.weighted !== false) { + ret.forEach(e => e.weight += IdentifierWeights.Function); } return ret; diff --git a/schema/src/classes/symbol/method_map.ts b/schema/src/classes/symbol/method_map.ts index 35beab8..06c7f7d 100644 --- a/schema/src/classes/symbol/method_map.ts +++ b/schema/src/classes/symbol/method_map.ts @@ -1,5 +1,5 @@ -import { IMethodMap, IProperty, SearchOptions, SearchResult, Searchable, Identifier, IdentifierWeights, Part } from '../../interfaces'; -import { Declaration, calculateScore } from './base'; +import { IMethodMap, IProperty, SearchOptions, Searchable, Identifier, IdentifierWeights, Part } from '../../interfaces'; +import { Declaration, SearchEntry } from './base'; import { Function } from './function'; export class MethodMap extends Declaration implements IMethodMap, Searchable { @@ -41,43 +41,42 @@ export class MethodMap extends Declaration implements IMethodMap, Searchable { }, {} as Record); } - public async search(needle: string, options: Readonly): Promise { - const localOptions = JSON.parse(JSON.stringify(options)); + public searchEntries(options: Readonly): SearchEntry[] { + const ret = super.searchEntries(options); - let ret = [ - ...await super.search(needle, localOptions), - ]; + ret[0].boost += 0.01; - ret[0].score += 0.01; + const parents = [...options.parents, `${this.identifier}.${this.name}`]; - localOptions.parents.push(`${this.identifier}.${this.name}`); - - if (localOptions.l1Only !== true) { - for (const method of Object.values(this.methods)) { - ret.push(...await method.search(needle, { - ...localOptions, + if (options.l1Only !== true) { + Object.values(this.methods).forEach((method, i) => { + const entries = method.searchEntries({ + ...options, + parents, weighted: false, identifier: Identifier.MethodMapMethod, - })); - } - + }); + + entries[0].trackOrder = i + 1; + + ret.push(...entries); + }); + for (const property of Object.values(this.properties)) { ret.push({ name: property.name, + term: property.name, identifier: Identifier.MethodMapProperty, - path: [...localOptions.parents, `${Identifier.MethodMapProperty}.${property.name}`], + path: [...parents, `${Identifier.MethodMapProperty}.${property.name}`], part: Part.Name, - score: calculateScore(property.name, needle), + boost: 0, + weight: 0, }); } } - if (localOptions.weighted !== false) { - ret = ret.map(e => { - e.score += IdentifierWeights.MethodMap; - - return e; - }); + if (options.weighted !== false) { + ret.forEach(e => e.weight += IdentifierWeights.MethodMap); } return ret; diff --git a/schema/src/classes/symbol/type_def.ts b/schema/src/classes/symbol/type_def.ts index d09e43f..bc2db77 100644 --- a/schema/src/classes/symbol/type_def.ts +++ b/schema/src/classes/symbol/type_def.ts @@ -1,5 +1,5 @@ -import { ITypeDefinition, ITypeSignature, SearchOptions, SearchResult, Searchable, Identifier, IdentifierWeights, Part } from '../../interfaces'; -import { Declaration, calculateScore } from './base'; +import { ITypeDefinition, ITypeSignature, SearchOptions, Searchable, Identifier, IdentifierWeights, Part } from '../../interfaces'; +import { Declaration, SearchEntry } from './base'; export class TypeDefinition extends Declaration implements ITypeDefinition, Searchable { /** @@ -23,41 +23,37 @@ export class TypeDefinition extends Declaration implements ITypeDefinition, Sear this.parsedSignature = typeDef.parsedSignature; } - public async search(needle: string, options: Readonly): Promise { - const localOptions = JSON.parse(JSON.stringify(options)); + public searchEntries(options: Readonly): SearchEntry[] { + const ret = super.searchEntries(options); - let ret = [ - ...await super.search(needle, localOptions), - ]; + const parents = [...options.parents, `${this.identifier}.${this.name}`]; - localOptions.parents.push(`${this.identifier}.${this.name}`); - - if (localOptions.l1Only !== true) { + if (options.l1Only !== true) { for (const arg of this.parsedSignature.arguments) { ret.push({ name: arg.type, + term: arg.type, identifier: Identifier.Argument, part: Part.Parameter, - path: [...localOptions.parents, `${Identifier.Argument}.${arg.name}`], - score: calculateScore(arg.type, needle), + path: [...parents, `${Identifier.Argument}.${arg.name}`], + boost: 0, + weight: 0, }); } ret.push({ name: this.parsedSignature.returnType, + term: this.parsedSignature.returnType, identifier: Identifier.Return, part: Part.Return, - path: [...localOptions.parents, `${Identifier.Return}.${this.parsedSignature.returnType}`], - score: calculateScore(this.parsedSignature.returnType, needle), + path: [...parents, `${Identifier.Return}.${this.parsedSignature.returnType}`], + boost: 0, + weight: 0, }); } - if (localOptions.weighted !== false) { - ret = ret.map(e => { - e.score += IdentifierWeights.TypeDefinition; - - return e; - }); + if (options.weighted !== false) { + ret.forEach(e => e.weight += IdentifierWeights.TypeDefinition); } return ret; diff --git a/schema/src/classes/symbol/type_set.ts b/schema/src/classes/symbol/type_set.ts index 52d89bc..7dda3ff 100644 --- a/schema/src/classes/symbol/type_set.ts +++ b/schema/src/classes/symbol/type_set.ts @@ -1,5 +1,5 @@ -import { ITypeSet, IType, SearchOptions, SearchResult, Searchable, Identifier, IdentifierWeights, Part } from '../../interfaces'; -import { Declaration, calculateScore } from './base'; +import { ITypeSet, IType, SearchOptions, Searchable, Identifier, IdentifierWeights, Part } from '../../interfaces'; +import { Declaration, SearchEntry } from './base'; export class TypeSet extends Declaration implements ITypeSet, Searchable { /** @@ -16,43 +16,39 @@ export class TypeSet extends Declaration implements ITypeSet, Searchable { this.types = typeSet.types; } - public async search(needle: string, options: Readonly): Promise { - const localOptions = JSON.parse(JSON.stringify(options)); + public searchEntries(options: Readonly): SearchEntry[] { + const ret = super.searchEntries(options); - let ret: SearchResult[] = [ - ...await super.search(needle, localOptions), - ]; + const parents = [...options.parents, `${this.identifier}.${this.name}`]; - localOptions.parents.push(`${this.identifier}.${this.name}`); - - if (localOptions.l1Only !== true) { + if (options.l1Only !== true) { for (const type of Object.values(this.types)) { for (const arg of type.parsedSignature.arguments) { ret.push({ name: arg.name, + term: arg.type, identifier: Identifier.Argument, part: Part.Parameter, - path: [...localOptions.parents, `${Identifier.Entry}.${type.type}`, `${Identifier.Argument}.${arg.name}`], - score: calculateScore(arg.type, needle), + path: [...parents, `${Identifier.Entry}.${type.type}`, `${Identifier.Argument}.${arg.name}`], + boost: 0, + weight: 0, }); } ret.push({ name: type.parsedSignature.returnType, + term: type.parsedSignature.returnType, identifier: Identifier.Return, part: Part.Return, - path: [...localOptions.parents, `${Identifier.Return}.${type.parsedSignature.returnType}`], - score: calculateScore(type.parsedSignature.returnType, needle), + path: [...parents, `${Identifier.Return}.${type.parsedSignature.returnType}`], + boost: 0, + weight: 0, }); } } - if (localOptions.weighted !== false) { - ret = ret.map(e => { - e.score += IdentifierWeights.TypeSet; - - return e; - }); + if (options.weighted !== false) { + ret.forEach(e => e.weight += IdentifierWeights.TypeSet); } return ret; diff --git a/schema/src/tests/search_index.test.ts b/schema/src/tests/search_index.test.ts new file mode 100644 index 0000000..39c0d51 --- /dev/null +++ b/schema/src/tests/search_index.test.ts @@ -0,0 +1,185 @@ +import { Bundle, IBundle, Identifier, SearchOptions, SearchResult, Declaration, MIN_SCORE } from '..'; + +const decl = (name: string) => ({ + name, + refLine: 1, + docStart: 0, + docEnd: 0, + docs: null, + metadata: null, +}); + +const fn = (name: string, returnType: string, args: [string, string][]) => ({ + ...decl(name), + kind: 'native', + returnType, + arguments: args.map(([name, type]) => ({ name, type, decl: '', default: null })), +}); + +const sig = (returnType: string, args: [string, string][]) => ({ + returnType, + arguments: args.map(([name, type]) => ({ name, type })), +}); + +const data = { + meta: { name: 'test', description: '', author: '' }, + source: { type: 'git' }, + version: null, + strands: { + 'adt_array.inc': { + functions: { + CreateArray: fn('CreateArray', 'Handle', [['blocksize', 'int'], ['startsize', 'int']]), + }, + methodmaps: { + ArrayList: { + ...decl('ArrayList'), + parent: 'Handle', + methods: { + ArrayList: fn('ArrayList', 'ArrayList', [['blocksize', 'int']]), + GetString: fn('GetString', 'int', [['index', 'int'], ['buffer', 'char[]'], ['maxlength', 'int']]), + PushArray: fn('PushArray', 'int', [['values', 'const any[]']]), + }, + properties: { + Length: { ...decl('Length'), type: 'int', getter: true, setter: false }, + }, + }, + }, + enumstructs: {}, + constants: {}, + defines: { + _adt_array_included: decl('_adt_array_included'), + }, + enums: {}, + typesets: {}, + typedefs: {}, + }, + 'clients.inc': { + functions: { + GetClientName: fn('GetClientName', 'bool', [['client', 'int'], ['name', 'char[]'], ['maxlen', 'int']]), + IsClientInGame: fn('IsClientInGame', 'bool', [['client', 'int']]), + a: fn('a', 'void', []), + }, + methodmaps: {}, + enumstructs: { + Player: { + ...decl('Player'), + methods: { + GetName: fn('GetName', 'void', [['buffer', 'char[]']]), + }, + fields: { + userid: { ...decl('userid'), type: 'int' }, + }, + }, + }, + constants: { + MAXPLAYERS: decl('MAXPLAYERS'), + }, + defines: { + MAX_NAME_LENGTH: decl('MAX_NAME_LENGTH'), + }, + enums: { + NetFlow: { + ...decl('NetFlow'), + entries: { + NetFlow_Outgoing: { ...decl('NetFlow_Outgoing') }, + NetFlow_Incoming: { ...decl('NetFlow_Incoming') }, + }, + }, + }, + typesets: { + SQLQueryCallback: { + ...decl('SQLQueryCallback'), + types: { + a: { ...decl(''), type: 'function void (Database db)', parsedSignature: sig('void', [['db', 'Database']]) }, + }, + }, + }, + typedefs: { + ListenCB: { + ...decl('ListenCB'), + type: 'function Action (int client, const char[] command)', + parsedSignature: sig('Action', [['client', 'int'], ['command', 'const char[]']]), + }, + }, + }, + }, +} as unknown as IBundle; + +const needles = [ + '', 'a', 'A', ' ', 'int', 'INT', 'void', 'char[]', 'const char[]', 'constchar[]', 'const char[]', + 'ArrayList', 'arraylist', 'Array List', 'Array', 'GetClientName', 'getclientname', 'GetClient', + 'ClientName', 'GetClinetName', 'NetFlow', 'netflow_in', 'MAX', 'max_name', 'userid', 'Player', + 'Database', 'Action', 'Length', 'xyzzy', 'Q', +]; + +// Score every entry, as search did before indexing +async function linearSearch(bundle: Bundle, needle: string, options: SearchOptions): Promise { + const ret: SearchResult[] = []; + + for (const [include, strand] of Object.entries(bundle.strands)) { + for (const member of [ + strand.functions, + strand.methodmaps, + strand.enumstructs, + strand.constants, + strand.defines, + strand.enums, + strand.typesets, + strand.typedefs, + ]) { + for (const symbol of Object.values(member) as Declaration[]) { + ret.push(...await symbol.search(needle, { ...options, parents: [...options.parents, include] })); + } + } + } + + return ret.filter(e => e.score > MIN_SCORE); +} + +describe('indexed search', () => { + const bundle = new Bundle(data); + + const optionSets: SearchOptions[] = [ + { parents: [] }, + { parents: ['core'] }, + { parents: [], weighted: false }, + { parents: [], l1Only: true }, + { parents: [], identifier: Identifier.MethodMapMethod }, + ]; + + for (const options of optionSets) { + test(`matches linear search with ${JSON.stringify(options)}`, async () => { + for (const needle of needles) { + expect(await bundle.search(needle, options)).toEqual(await linearSearch(bundle, needle, options)); + } + }); + } + + test('strand search matches linear search', async () => { + for (const needle of needles) { + const expected = (await linearSearch(bundle, needle, { parents: [] })) + .filter(e => e.path[0] === 'clients.inc'); + + expect(await bundle.strands['clients.inc'].search(needle, { parents: ['clients.inc'] })).toEqual(expected); + } + }); + + test('finds exact, substring and fuzzy matches', async () => { + const names = async (needle: string) => (await bundle.search(needle, { parents: [] })).map(e => e.name); + + expect(await names('GetClientName')).toContain('GetClientName'); + expect(await names('clientname')).toContain('GetClientName'); + expect(await names('GetClinetName')).toContain('GetClientName'); + expect(await names('xyzzy')).toEqual([]); + }); + + test('results do not share paths', async () => { + const [first] = await bundle.search('ArrayList', { parents: [] }); + + first.path.push('mutated'); + + const [second] = await bundle.search('ArrayList', { parents: [] }); + + expect(second.path).not.toContain('mutated'); + }); +});