Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -589,6 +589,15 @@ site binds to is still an answer, and says why it is empty:
}
```

An answer that may be missing something says what, in a `coverage` field
(`callers`, `callers --raw` and `refs`, CLI and MCP): `callCapped` lists files
that call the name but reached the per-file call-site cap (`--max-calls`,
default 8192), `notIndexed` the files left out for size (`--max-bytes`, default
1 MiB) whose text names it, and `hint` the flag that brings them in. An
answer nothing limited carries no `coverage` field. `outline`, `symbol-at` and
`complexity` on a file left out for size say so instead of "no such file",
and `find` writes the same note to stderr when it finds nothing.

`find`, `refs` and `outline` print the same answers as MCP `find_symbol`,
`find_references` and `symbols_overview`: each declaration's complete
signature, doc comment, parent and line span, which `symbols` leaves out.
Expand Down
12 changes: 11 additions & 1 deletion scripts/engine.browser.d.mts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
declare const ENGINE_VERSION = "2.31.7";
declare const SCHEMA_VERSION = 5;
declare const EXTRACTOR_VERSION = 16;
declare const EXTRACTOR_VERSION = 17;
type FileKind = "code" | "doc" | "config" | "asset" | "other";
type EdgeKind = "contains" | "doc-link" | "import" | "call" | "extends" | "implements" | "use" | "mention";
type Tier = 0 | 1 | 2;
Expand Down Expand Up @@ -77,6 +77,7 @@ interface FileRecord {
importedNames?: string[];
importAliases?: ImportAlias[];
truncated?: true;
callsCapped?: true;
astFallback?: true;
generated?: "minified" | "bundle";
relations?: RawRelation[];
Expand Down Expand Up @@ -268,6 +269,10 @@ interface WalkResult {
files: WalkedFile[];
capped: boolean;
excluded: number;
oversized?: {
rel: string;
size: number;
}[];
}
declare const DEFAULT_MAX_FILES = 20000;
declare function walk(root: string, opts?: WalkOptions): WalkResult;
Expand All @@ -283,6 +288,10 @@ interface RepoScan {
mtimes: Map<string, number>;
capped: boolean;
excluded: number;
oversized?: {
rel: string;
size: number;
}[];
contentUnchanged: boolean;
cacheDirty: boolean;
}
Expand Down Expand Up @@ -504,6 +513,7 @@ interface CodeInfo {
symbols: CodeSymbol[];
summary?: string;
truncated?: true;
callsCapped?: true;
astFallback?: true;
generated?: GeneratedKind;
refs: RawRef[];
Expand Down
294 changes: 148 additions & 146 deletions scripts/engine.browser.mjs

Large diffs are not rendered by default.

12 changes: 11 additions & 1 deletion scripts/engine.d.mts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
declare const ENGINE_VERSION = "2.31.7";
declare const SCHEMA_VERSION = 5;
declare const EXTRACTOR_VERSION = 16;
declare const EXTRACTOR_VERSION = 17;
type FileKind = "code" | "doc" | "config" | "asset" | "other";
type EdgeKind = "contains" | "doc-link" | "import" | "call" | "extends" | "implements" | "use" | "mention";
type Tier = 0 | 1 | 2;
Expand Down Expand Up @@ -77,6 +77,7 @@ interface FileRecord {
importedNames?: string[];
importAliases?: ImportAlias[];
truncated?: true;
callsCapped?: true;
astFallback?: true;
generated?: "minified" | "bundle";
relations?: RawRelation[];
Expand Down Expand Up @@ -268,6 +269,10 @@ interface WalkResult {
files: WalkedFile[];
capped: boolean;
excluded: number;
oversized?: {
rel: string;
size: number;
}[];
}
declare const DEFAULT_MAX_FILES = 20000;
declare function walk(root: string, opts?: WalkOptions): WalkResult;
Expand All @@ -283,6 +288,10 @@ interface RepoScan {
mtimes: Map<string, number>;
capped: boolean;
excluded: number;
oversized?: {
rel: string;
size: number;
}[];
contentUnchanged: boolean;
cacheDirty: boolean;
}
Expand Down Expand Up @@ -504,6 +513,7 @@ interface CodeInfo {
symbols: CodeSymbol[];
summary?: string;
truncated?: true;
callsCapped?: true;
astFallback?: true;
generated?: GeneratedKind;
refs: RawRef[];
Expand Down
336 changes: 242 additions & 94 deletions scripts/engine.mjs

Large diffs are not rendered by default.

22 changes: 15 additions & 7 deletions src/ast/extract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,12 @@ export interface AstResult {
}

const MAX_REF_IDENTS = 512;
const MAX_CALLS = 512;
// The per-file call-site cap, on both tiers. It was 512, and a test file that
// calls the function under test hundreds of times lost the rest without a word:
// vite's `callers resolveConfig` answered 56 of 68 sites, terraform's
// `NewContext` 53 of 78. At 8192 the TypeScript repo's index grows by 1% and
// takes no longer; a file the cap still cuts is flagged (FileRecord.callsCapped).
export const DEFAULT_MAX_CALLS = 8192;
const MAX_IMPORTED_NAMES = 256;
const MAX_SYMBOLS = 2000;
const MAX_RELATIONS = 256;
Expand Down Expand Up @@ -156,7 +161,8 @@ function isStatementString(node: TSNode): boolean {
return parent !== null && parent.type === "expression_statement" && parent.namedChildCount === 1;
}

// The call sites a file keeps, sorted by name then line; `sites` is in source
// The call sites a file keeps, sorted by name, line, then receiver (one line may
// call the same name on two receivers); `sites` is in source
// order. Past the cap the survivors are chosen BEFORE that sort: slicing the
// sorted list kept whatever sorts first, and code-unit order puts every
// uppercase name ahead of every lowercase one — tsgo's parser.go kept 512 of its
Expand All @@ -165,7 +171,7 @@ function isStatementString(node: TSNode): boolean {
// order, so every name a file calls stays bindable by the caller index, the call
// edges and dead code; the rest of the budget then goes in source order. Shared
// with the regex tier, so both keep the same sites.
export function capCallSites<T extends { name: string; line: number }>(sites: T[], max: number): T[] {
export function capCallSites<T extends { name: string; line: number; receiver?: string }>(sites: T[], max: number): T[] {
let kept = sites.slice();
if (sites.length > max) {
const keep = new Uint8Array(sites.length);
Expand All @@ -185,7 +191,7 @@ export function capCallSites<T extends { name: string; line: number }>(sites: T[
}
kept = sites.filter((_, i) => keep[i] === 1);
}
return kept.sort((a, b) => byStr(a.name, b.name) || a.line - b.line);
return kept.sort((a, b) => byStr(a.name, b.name) || a.line - b.line || byStr(a.receiver ?? "", b.receiver ?? ""));
}

function collectAll(
Expand All @@ -207,7 +213,9 @@ function collectAll(
const addCall = (name: string | undefined, node: TSNode, receiver?: string): void => {
if (!name || name === "_" || !/^[A-Za-z_]\w*$/.test(name)) return;
const line = node.startPosition.row + 1;
const key = `${name} ${line}`;
// The receiver is part of the key: `incremental.NewProgram(compiler.NewProgram())`
// is two calls to two packages, and keeping one hid the other's caller.
const key = `${name} ${line} ${receiver ?? ""}`;
if (callSeen.has(key)) return;
callSeen.add(key);
calls.push(receiver ? { name, line, receiver } : { name, line });
Expand Down Expand Up @@ -436,7 +444,7 @@ interface WalkCtx {
// undefined when no grammar is loaded for the extension (caller falls back to the
// regex extractor). Walks top-level declarations, type members, and declarations
// nested up to MAX_FUNC_DEPTH function bodies deep.
// `opts.maxCalls` overrides the per-file call-site cap (default MAX_CALLS).
// `opts.maxCalls` overrides the per-file call-site cap (default DEFAULT_MAX_CALLS).
// `opts.imports` (default true) fills `refs`/`pkg` from extract/imports.ts — the
// very scan the index runs, so this API and the graph agree on every file;
// extractCode passes false because it runs that scan itself.
Expand Down Expand Up @@ -970,7 +978,7 @@ export function extractAst(
root,
spec,
new Set(symbols.map((s) => s.name)),
opts.maxCalls ?? MAX_CALLS,
opts.maxCalls ?? DEFAULT_MAX_CALLS,
);
const wantImports = opts.imports !== false;
const refs = wantImports ? extractImports(ext, content) : [];
Expand Down
3 changes: 3 additions & 0 deletions src/ast/node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,9 @@ export function readReceiver(node: TSNode | null): string | undefined {
node.childForFieldName("argument") ??
node.childForFieldName("receiver") ??
node.childForFieldName("table");
// Rust's `crate::f()` and `super::f()` name a module by keyword, a leaf
// readName does not take for a name; the binder resolves both to a file.
if (obj && (obj.type === "crate" || obj.type === "super")) return obj.type;
const name = obj ? readName(obj) : undefined;
return name && /^[A-Za-z_]\w*$/.test(name) ? name : undefined;
}
Expand Down
78 changes: 69 additions & 9 deletions src/bind.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,18 @@ const PACKAGE_DIR = new Set(["go", "java"]);
// mates (gin calls `debugPrint` from 14 files); Python's leading underscore is
// a convention, so an import reaches them (`self._find_error_handler` in a
// subclass file).
const PRIVATE_REACH: Record<string, "package" | "import"> = { go: "package", java: "package", python: "import" };
// Rust's private items reach a module's descendants, which name them by path:
// `crate::helper()` from anywhere in the crate, `super::f()` from a child
// module — never by a bare name or a `use` of another module (see rustPath).
const PRIVATE_REACH: Record<string, "package" | "import" | "path"> = { go: "package", java: "package", python: "import", rust: "path" };

// Declarations no call can reach: a module or package is named, never called.
// Rust's `mod search;` beside `fn search` took every `search()` call in the file.
const NOT_CALLABLE = new Set(["mod", "module", "package", "namespace"]);

// The receiver a Rust `super::` takes inside an inline `mod { … }` block, where
// it names the enclosing file rather than its parent module.
const INLINE_SUPER = "super\u0000inline";

// Receivers that name the enclosing object. JS/TS/Java `this` and Rust `self`
// never get here (the extractor reads no receiver from them); PHP's `$this`
Expand Down Expand Up @@ -496,6 +507,38 @@ export function createCallBinder(scan: RepoScan, pairs: Set<string>, opts: CallB
fnParentsMemo.set(rel, (parents = set ?? noParents));
return parents;
};
// The file a Rust `crate::` or `super::` path names, read off the module
// tree's layout: the crate root is the nearest `lib.rs`/`main.rs` above the
// file (ripgrep's is `crates/core/main.rs`), a module's parent the
// `mod.rs`, `<dir>.rs` or crate root of the directory above it. Undefined
// when the layout says nothing, which leaves the call to the ordinary rules.
const indexed = (rel: string): boolean => (byRel ??= new Map(scan.files.map((f) => [f.rel, f]))).has(rel);
const firstIndexed = (dir: string, names: string[]): string | undefined =>
names.map((n) => (dir ? `${dir}/${n}` : n)).find(indexed);
// Cargo also makes every file directly under a target directory a crate of
// its own: `src/bin/tool.rs`, `examples/demo.rs`, `tests/it.rs`, `benches/b.rs`.
const crateRoot = (rel: string, base: string): boolean => {
if (base === "lib.rs" || base === "main.rs") return true;
const dir = dirOf(rel);
const parent = dir.slice(dir.lastIndexOf("/") + 1);
return parent === "examples" || parent === "tests" || parent === "benches" || (parent === "bin" && /(^|\/)src\/bin$/.test(dir));
};
const rustPath = (rel: string, receiver: string): string[] | undefined => {
const base = rel.slice(rel.lastIndexOf("/") + 1);
let found: string | undefined;
if (crateRoot(rel, base)) {
if (receiver === "crate") found = rel;
} else if (receiver === "crate") {
for (let dir = dirOf(rel); !found; dir = dirOf(dir)) {
found = firstIndexed(dir, ["lib.rs", "main.rs"]);
if (!dir) break;
}
} else {
const parent = dirOf(base === "mod.rs" ? dirOf(rel) : rel.slice(0, -".rs".length));
found = firstIndexed(parent, ["mod.rs", "lib.rs", "main.rs"]) ?? (parent && indexed(`${parent}.rs`) ? `${parent}.rs` : undefined);
}
return found === undefined ? undefined : [found];
};
const inFunction = (s: CodeSymbol): boolean => s.parent !== undefined && fnParentsOf(s.file).has(s.parent);
const isMember = (s: CodeSymbol): boolean => s.parent !== undefined && !fnParentsOf(s.file).has(s.parent);

Expand All @@ -508,7 +551,7 @@ export function createCallBinder(scan: RepoScan, pairs: Set<string>, opts: CallB
for (const f of scan.files) {
const reach = PRIVATE_REACH[familyOf(f.lang)] !== undefined;
for (const s of f.symbols) {
if (REFERENCE_KINDS.has(s.kind) || (!s.exported && !reach)) continue;
if (REFERENCE_KINDS.has(s.kind) || NOT_CALLABLE.has(s.kind) || (!s.exported && !reach)) continue;
if (only && !only.has(s.name)) continue;
const list = byName.get(s.name);
if (list) list.push(s);
Expand Down Expand Up @@ -567,7 +610,7 @@ export function createCallBinder(scan: RepoScan, pairs: Set<string>, opts: CallB
const fnParents = fnParentsOf(f.rel);
const isOwnMember = (s: CodeSymbol): boolean => s.parent !== undefined && !fnParents.has(s.parent);
for (const s of f.symbols) {
if (REFERENCE_KINDS.has(s.kind)) continue;
if (REFERENCE_KINDS.has(s.kind) || NOT_CALLABLE.has(s.kind)) continue;
if (!ownAny.has(s.name)) ownAny.set(s.name, s);
if (isOwnMember(s)) {
if (!ownMember.has(s.name)) ownMember.set(s.name, s);
Expand All @@ -581,7 +624,7 @@ export function createCallBinder(scan: RepoScan, pairs: Set<string>, opts: CallB
if (!ownMemberOf) {
ownMemberOf = new Map();
for (const s of f.symbols) {
if (REFERENCE_KINDS.has(s.kind) || !isOwnMember(s)) continue;
if (REFERENCE_KINDS.has(s.kind) || NOT_CALLABLE.has(s.kind) || !isOwnMember(s)) continue;
const key = `${s.parent}\u0000${s.name}`;
if (!ownMemberOf.has(key)) ownMemberOf.set(key, s);
}
Expand All @@ -600,12 +643,25 @@ export function createCallBinder(scan: RepoScan, pairs: Set<string>, opts: CallB
return ownTypeByLower.get(receiver.toLowerCase());
};
const aliases = scope.aliases(f);
// `crate` and `super` as this Rust file's path qualifiers, where they name a file.
let rustModules: Map<string, string[]> | undefined;
// The file's inline `mod name { … }` blocks: inside one, `super::` is the
// file itself (`#[cfg(test)] mod tests` calling `super::parse`).
let inlineMods: CodeSymbol[] | undefined;
if (family === "rust") {
for (const q of ["crate", "super"]) {
const files = rustPath(f.rel, q);
if (files) (rustModules ??= new Map()).set(q, files);
}
inlineMods = f.symbols.filter((s) => s.kind === "mod" && s.endLine !== undefined && s.endLine > s.line);
if (inlineMods.length) (rustModules ??= new Map()).set(INLINE_SUPER, [f.rel]);
}

// Cross-file defs of `name` this file reaches by stated evidence.
const corroborated = (name: string, pool: Pool): CodeSymbol[] => {
const out = scope.reached(exportedGroup(name, family, pool), f, name);
const priv = hiddenGroup(name, family, pool);
if (priv && hiddenReach) {
if (priv && (hiddenReach === "package" || hiddenReach === "import")) {
const seen = new Set(out.map((d) => d.file));
for (const d of hiddenReach === "package" ? scope.reached(priv, f, name) : importedDefs(priv, scope.targets(f.rel))) {
// A package-private def is visible to its own package only.
Expand Down Expand Up @@ -648,13 +704,15 @@ export function createCallBinder(scan: RepoScan, pairs: Set<string>, opts: CallB
const byReceiver = new Map<string, Reading>();
const byEnclosing = new Map<CodeSymbol | null, Map<string, Reading>>();
const read = (c: CallSite, enclosing: CodeSymbol | undefined): Reading => {
const receiver = c.receiver;
let receiver = c.receiver;
if (receiver === undefined) return BARE;
if (receiver === "super" && inlineMods?.some((m) => c.line > m.line && c.line <= m.endLine!)) receiver = INLINE_SUPER;
let r = byReceiver.get(receiver);
if (r) return r;
const self = (t: string | undefined): Reading => reading("self", receiver, t, t);
if (!SELF_NAMES.has(receiver)) {
if (aliases.modules.has(receiver)) r = reading("module", receiver);
if (rustModules?.has(receiver)) r = reading("module", receiver);
else if (aliases.modules.has(receiver)) r = reading("module", receiver);
else if (ownTypes.has(receiver)) r = self(receiver); // `Greeter.make()` next to Greeter
else if (!SHAPED.has(family)) {
// A variable named after a type this file declares: `store.get()`.
Expand Down Expand Up @@ -718,7 +776,9 @@ export function createCallBinder(scan: RepoScan, pairs: Set<string>, opts: CallB
if (shape === "bare") {
const renamed = aliases.names.get(name);
if (renamed) ({ name: callee, files: restrict } = renamed);
} else if (shape === "module") restrict = aliases.modules.get(receiver!);
} else if (shape === "module") restrict = rustModules?.get(receiver!) ?? aliases.modules.get(receiver!);
// A Rust path to an ancestor module reaches its private items too.
const viaPath = rustModules?.has(receiver!) === true && shape === "module";
if (only && !only.has(callee)) return null;
// What the call's shape can reach (SHAPED families): a bare call or a
// module qualifier a free function or type, a call on an object a
Expand All @@ -742,7 +802,7 @@ export function createCallBinder(scan: RepoScan, pairs: Set<string>, opts: CallB
if (!recall) return null;
} else {
const hits = scope.within(exportedGroup(callee, family, pool), restrict, callee);
if (hiddenReach === "import") {
if (hiddenReach === "import" || viaPath) {
for (const d of scope.within(hiddenGroup(callee, family, pool), restrict, callee)) if (!hits.includes(d)) hits.push(d);
}
const chosen = pickCandidate(f.rel, hits);
Expand Down
2 changes: 1 addition & 1 deletion src/cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ function record(v: unknown, rel: string): v is FileRecord {
strings(v.headings) && Array.isArray(v.symbols) && v.symbols.every((s) => symbol(s, rel)) &&
Array.isArray(v.refs) && v.refs.every(ref) &&
optionalArray(v.calls, call) && optionalArray(v.importAliases, importAlias) && optionalArray(v.relations, relation) && optionalArray(v.literals, literal) &&
(v.truncated === undefined || v.truncated === true) &&
(v.truncated === undefined || v.truncated === true) && (v.callsCapped === undefined || v.callsCapped === true) &&
(v.astFallback === undefined || v.astFallback === true) && (v.generated === undefined || v.generated === "minified" || v.generated === "bundle");
}

Expand Down
15 changes: 10 additions & 5 deletions src/callers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,11 @@ function indexCallers(scan: RepoScan, pairs: Set<string>, recall: boolean, only?
let entry = sites.get(key);
if (!entry) sites.set(key, (entry = { def: hit.def, callers: [] }));
else if (hit.def.line < entry.def.line) entry.def = hit.def;
// `foo(a.foo(1))` is two sites on one line that may reach the same
// declaration: the caller is that line, once. A file's sites come sorted
// by name then line, so the twin is the entry's last caller.
const last = entry.callers[entry.callers.length - 1];
if (last && last.file === f.rel && last.line === c.line) continue;
entry.callers.push(
recall
? { file: f.rel, line: c.line, confidence: hit.corroborated ? "corroborated" : "unique-name" }
Expand Down Expand Up @@ -186,11 +191,11 @@ export function enclosingAmong(symbols: CodeSymbol[], line: number): CodeSymbol
// buildCallerIndex answers "who calls this DEFINED, RESOLVED symbol",
// buildRawCallerIndex answers "where does this NAME get called", full stop.
//
// Bounded by the per-file call cap: FileRecord.calls truncates at 512
// call-sites per file, name+line deduped (src/types.ts, FileRecord.calls doc).
// A single generated/vendored file with more raw call sites than that loses
// sites silently upstream of this function, in extraction — not something
// buildRawCallerIndex can see or recover.
// Bounded by the per-file call cap: FileRecord.calls truncates at 8192
// call-sites per file, name+line+receiver deduped (src/types.ts, FileRecord.calls
// doc). A file with more raw call sites than that loses sites upstream of this
// function, in extraction; it carries `callsCapped`, which the CLI and MCP
// answers report (see coverage.ts).
export interface RawCallerSite {
file: string;
line: number; // 1-based line of the call site
Expand Down
Loading
Loading