diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba2a2cba2..f64f689e2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,6 +64,20 @@ jobs: # rather than on every target the matrix above cross-compiles for. - run: cargo xtask test-plugins + test_code_mode: + name: Test code-mode API + runs-on: ubuntu-22.04 + steps: + # v4.2.2 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: dtolnay/rust-toolchain@1.85.0 + - run: npm install --global bun@1.3.14 + - run: sudo apt-get update && sudo apt-get install -y ripgrep + - run: bun install --frozen-lockfile + - run: bun run build:api + - run: bun run typecheck + - run: bun run test:code-mode + test_mime_db: name: Test with MIME database runs-on: ubuntu-22.04 diff --git a/.gitignore b/.gitignore index 12465665a..645c7bc69 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,10 @@ plugin.wasm # Generated data left behind by the retired review web viewer. /examples/review/viewer/data/ + +# Node binding build and JS dependencies. +/bindings/node/*.node +/node_modules/ + +# Optional computed diff store. +/.cache/diffr/ diff --git a/Cargo.lock b/Cargo.lock index e978649b9..ed0445cdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -442,6 +442,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -657,6 +666,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" +dependencies = [ + "quote", + "syn 2.0.106", +] + [[package]] name = "data-encoding" version = "2.11.1" @@ -675,6 +694,17 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "diffr-node" +version = "0.1.0" +dependencies = [ + "difftastic", + "napi", + "napi-build", + "napi-derive", + "serde_json", +] + [[package]] name = "diffr-plugin-context" version = "0.1.0" @@ -780,12 +810,14 @@ dependencies = [ "rayon", "regex", "reqwest", + "rusqlite", "rustc-hash", "schemars", "serde", "serde_json", "serde_path_to_error", "serde_yaml_ng", + "sha2", "smallvec", "streaming-iterator", "strsim", @@ -1016,6 +1048,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fancy-regex" version = "0.19.2" @@ -1299,6 +1337,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -1320,6 +1367,15 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -1812,6 +1868,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.8" @@ -1827,6 +1893,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libz-sys" version = "1.1.29" @@ -1942,6 +2019,66 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "napi" +version = "2.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55740c4ae1d8696773c78fdafd5d0e5fe9bc9f1b071c7ba493ba5c413a9184f3" +dependencies = [ + "bitflags", + "ctor", + "napi-derive", + "napi-sys", + "once_cell", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "napi-build" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c0f5d67ee408a4685b61f5ab7e58605c8ae3f2b4189f0127d804ff13d5560a" + +[[package]] +name = "napi-derive" +version = "2.16.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbe2585d8ac223f7d34f13701434b9d5f4eb9c332cccce8dee57ea18ab8ab0c" +dependencies = [ + "cfg-if", + "convert_case", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "napi-derive-backend" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1639aaa9eeb76e91c6ae66da8ce3e89e921cd3885e99ec85f4abacae72fc91bf" +dependencies = [ + "convert_case", + "once_cell", + "proc-macro2", + "quote", + "regex", + "semver", + "syn 2.0.106", +] + +[[package]] +name = "napi-sys" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427802e8ec3a734331fec1035594a210ce1ff4dc5bc1950530920ab717964ea3" +dependencies = [ + "libloading", +] + [[package]] name = "nom" version = "8.0.0" @@ -2576,6 +2713,20 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-hash" version = "2.0.0" @@ -4054,6 +4205,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-xid" version = "0.2.6" diff --git a/Cargo.toml b/Cargo.toml index b1f934be5..faaecdd48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,9 @@ include = [ ] [dependencies] +sha2 = "0.10" +rusqlite = { version = "0.32", features = ["bundled"] } +tempfile = "3.27.0" git2 = { version = "0.20", default-features = false } regex = "1.10.4" clap = { version = "4.0.0", features = ["cargo", "env", "wrap_help", "string"] } @@ -164,7 +167,6 @@ assert_cmd = "2.0.17" predicates = "3.1.3" pretty_assertions = "1.3.0" -tempfile = "3.27.0" [build-dependencies] toml = "0.8" @@ -177,6 +179,7 @@ version_check = "0.9.4" [workspace] members = [ "xtask", + "crates/diffr-node", "crates/diffr-plugin-sdk", "plugins/context", "plugins/deleted-bodies", diff --git a/bindings/node/build.mjs b/bindings/node/build.mjs new file mode 100644 index 000000000..ab3527acb --- /dev/null +++ b/bindings/node/build.mjs @@ -0,0 +1,12 @@ +import { execFileSync } from "node:child_process"; +import { copyFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { join } from "node:path"; + +const cwd = fileURLToPath(new URL("../../", import.meta.url)); +execFileSync("cargo", ["build", "-p", "diffr-node"], { cwd, stdio: "inherit" }); +const metadata = JSON.parse(execFileSync("cargo", ["metadata", "--no-deps", "--format-version=1"], { cwd })); +const library = process.platform === "darwin" ? "libdiffr_node.dylib" + : process.platform === "win32" ? "diffr_node.dll" : "libdiffr_node.so"; +copyFileSync(join(metadata.target_directory, "debug", library), + new URL("./diffr.node", import.meta.url)); diff --git a/bindings/node/print.ts b/bindings/node/print.ts new file mode 100644 index 000000000..5e60abd95 --- /dev/null +++ b/bindings/node/print.ts @@ -0,0 +1,90 @@ +import type { RegionData as Region, SearchResultData, SourceData as Source } from "./types.ts"; + +type Row = { key: string; line: number; text: string; changed: boolean }; +type Fold = { key: string; first: number; last: number; id: number }; +type Item = Row | Fold; +const isFold = (item: Item): item is Fold => "id" in item; +const number = (line?: number) => line === undefined ? " " : String(line + 1).padStart(5); + +function visible(source: Source): Item[] { + const lines = source.text.split("\n").map(line => line.replace(/\r$/, "")); + const items: Item[] = []; + function visit(regions: Region[]) { + for (const region of regions) { + const first = region.start.line; + const end = region.end.line + Number(region.end.column > 0); + if (region.visibility?.collapsed) { + if (end > first) items.push({ key: `fold:${region.fold_state_id}`, id: region.fold_state_id, first, last: end - 1 }); + } else if (region.kind === "fold") { + visit(region.children); + } else { + for (let line = first; line < end; line++) { + items.push({ key: `row:${region.alignment_id}:${line - first}`, line, text: lines[line], + changed: (region.changed ?? []).some(span => span.line === line) }); + } + } + } + } + visit(source.regions); + return items; +} + +/** Merge ordered visibility streams by their shared alignment/fold identities. */ +function pair(left: Item[], right: Item[]): [Item | undefined, Item | undefined][] { + const rows: [Item | undefined, Item | undefined][] = []; + let l = 0, r = 0; + while (l < left.length) { + const matching = right.findIndex((item, index) => index >= r && item.key === left[l].key); + if (matching < 0) { rows.push([left[l++], undefined]); continue; } + while (r < matching) rows.push([undefined, right[r++]]); + rows.push([left[l++], right[r++]]); + } + while (r < right.length) rows.push([undefined, right[r++]]); + return rows; +} + +export function print(result: SearchResultData): string { + const lhs = result.display !== "rhs" ? result.sources.same ?? result.sources.lhs : undefined; + const rhs = result.display !== "lhs" ? result.sources.same ?? result.sources.rhs : undefined; + const leftPath = result.file.lhs?.path; + const rightPath = result.file.rhs?.path; + const name = leftPath && rightPath && leftPath !== rightPath ? `${leftPath} → ${rightPath}` : rightPath ?? leftPath; + const unchanged = Boolean(result.sources.same); + const combined = Boolean(lhs && rhs) && !unchanged; + const output = [`${name} — ${unchanged ? (lhs && rhs ? "base = head" : lhs ? "base" : "head") : combined ? "base → head" : lhs ? "base" : "head"}`, + combined ? " base head" : unchanged ? " line" : lhs ? " base" : " head"]; + let folded = false; + const range = (fold: Fold, side: string) => `${side} ${fold.first + 1}–${fold.last + 1}`; + const foldRow = (left?: Fold, right?: Fold) => { + folded = true; + const ranges = [left && range(left, "base"), right && range(right, "head")].filter(Boolean).join(" / "); + output.push(` … ${ranges} collapsed [fold_state_id=${(left ?? right)!.id}] …`); + }; + if (combined) { + for (const [left, right] of pair(visible(lhs!), visible(rhs!))) { + if ((left && isFold(left)) || (right && isFold(right))) { + foldRow(left && isFold(left) ? left : undefined, right && isFold(right) ? right : undefined); + continue; + } + const l = left as Row | undefined, r = right as Row | undefined; + if (l && r && !l.changed && !r.changed && l.text === r.text) { + output.push(`${number(l.line)} ${number(r.line)} ${r.text}`.trimEnd()); + } else { + if (l) output.push(`${number(l.line)} - ${l.text}`.trimEnd()); + if (r) output.push(` ${number(r.line)} + ${r.text}`.trimEnd()); + } + } + } else { + // Shared content has one tree; display determines its side labels. + const items = lhs && rhs ? pair(visible(lhs), visible(rhs)).map(([l,r]) => r ?? l!) : visible((lhs ?? rhs)!); + for (const item of items) { + if (isFold(item)) { + foldRow(lhs ? item : undefined, rhs ? item : undefined); + } else { + output.push(`${number(item.line)} ${unchanged ? " " : lhs ? "-" : "+"} ${item.text}`.trimEnd()); + } + } + } + if (folded) output.push("", "[More context: call result.setCollapsed(fold_state_id, false) with an indicated\nID, then print the result again. Full source text and region children are already\npresent; no read call is needed.]"); + return output.join("\n"); +} diff --git a/bindings/node/result.ts b/bindings/node/result.ts new file mode 100644 index 000000000..fb764acd7 --- /dev/null +++ b/bindings/node/result.ts @@ -0,0 +1,51 @@ +import { inspect } from "node:util"; +import { print } from "./print.ts"; +import type { Region, SearchResult, SearchResultData, Scope, Source } from "./api.ts"; + +function walk(regions: Region[], visit: (region: Region) => void): void { + for (const region of regions) { + visit(region); + if (region.kind === "fold") walk(region.children, visit); + } +} +export function bind(data: SearchResultData): SearchResult { + // Region methods are attached below before the bound result escapes. + const sources = data.sources as SearchResult["sources"]; + for (const source of Object.values(sources)) { + source.regions ??= []; + walk(source.regions, region => { + const anyLeaf = (predicate: (leaf: Extract) => boolean) => { + let found = false; + walk([region], child => { if (child.kind === "leaf") found ||= predicate(child); }); + return found; + }; + Object.defineProperties(region, { + hasChanges: { value: () => anyLeaf(leaf => Boolean(leaf.changed?.length)) }, + hasHighlights: { value: () => anyLeaf(leaf => Boolean(leaf.search_highlights?.length)) }, + hasChangedHighlights: { value: () => anyLeaf(leaf => (leaf.search_highlights ?? []).some(hit => + (leaf.changed ?? []).some(change => change.line === hit.line && change.start_column < hit.end_column && hit.start_column < change.end_column))) }, + }); + }); + } + return Object.assign(Object.create(Result.prototype), data); +} +class Result { + declare display: SearchResultData["display"]; + declare scope: Scope; + declare file: SearchResultData["file"]; + declare sources: SearchResult["sources"]; + setCollapsed(foldStateId: number, collapsed: boolean): void { + if (!Number.isInteger(foldStateId) || foldStateId < 0 || typeof collapsed !== "boolean") throw new TypeError("expected a fold-state ID and boolean"); + let found = false; + for (const source of Object.values(this.sources)) walk(source.regions, region => { + if (region.fold_state_id === foldStateId) { + region.visibility = { ...region.visibility, collapsed }; + found = true; + } + }); + if (!found) throw new RangeError(`No fold_state_id ${foldStateId} in this result`); + } + toString(): string { return print(this.toJSON()); } + [inspect.custom](): string { return this.toString(); } + toJSON(): SearchResultData { return { display: this.display, scope: this.scope, file: this.file, sources: this.sources } as SearchResultData; } +} diff --git a/bun.lock b/bun.lock new file mode 100644 index 000000000..2b1943d5c --- /dev/null +++ b/bun.lock @@ -0,0 +1,32 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "diffr", + "dependencies": { + "@typesafe-ai/sdk": "^0.6.0", + "zod": "4.4.3", + }, + "devDependencies": { + "@types/bun": "^1.3.10", + "typescript": "^5.9.3", + }, + }, + }, + "packages": { + "@types/bun": ["@types/bun@1.4.2", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="], + + "@types/node": ["@types/node@26.6.1", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-VqGJBMCtdhqkBUCcBLvywI0NJ+KLuVzgNnlBUNFOQjqVxzo2lxLUNg1DSey8+u2u6ktswSAxg+s68QLzWHNOuA=="], + + "@typesafe-ai/sdk": ["@typesafe-ai/sdk@0.6.0", "", {}, "sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw=="], + + "bun-types": ["bun-types@1.4.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], + + "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + } +} diff --git a/crates/diffr-node/Cargo.toml b/crates/diffr-node/Cargo.toml new file mode 100644 index 000000000..f95dfdfeb --- /dev/null +++ b/crates/diffr-node/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "diffr-node" +version = "0.1.0" +edition = "2021" +rust-version = "1.85.0" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +difftastic = { path = "../.." } +napi = { version = "2", default-features = false, features = ["napi6", "async", "serde-json"] } +napi-derive = "2" +serde_json = "1" + +[build-dependencies] +napi-build = "=2.1.3" diff --git a/crates/diffr-node/build.rs b/crates/diffr-node/build.rs new file mode 100644 index 000000000..0f1b01002 --- /dev/null +++ b/crates/diffr-node/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/crates/diffr-node/src/lib.rs b/crates/diffr-node/src/lib.rs new file mode 100644 index 000000000..6cf148c42 --- /dev/null +++ b/crates/diffr-node/src/lib.rs @@ -0,0 +1,45 @@ +//! JSON conversion lives at the binding boundary; Rust search uses typed inputs. +//! Synchronous diff/storage work runs on N-API's blocking worker pool. +use difftastic::search::{configured_session, Options}; +use napi::{bindgen_prelude::spawn_blocking, Error, Result}; +use napi_derive::napi; +use serde_json::Value; + +fn error(error: impl std::fmt::Display) -> Error { + Error::from_reason(error.to_string()) +} + +#[napi] +pub async fn hydrate(scope: Value, hits: Value) -> Result { + spawn_blocking(move || { + let scope = serde_json::from_value(scope).map_err(error)?; + let hits = serde_json::from_value(hits).map_err(error)?; + let mut session = + configured_session(scope, Options::default()).map_err(|e| error(format!("{e:#}")))?; + let results = session.hydrate(hits).map_err(|e| error(format!("{e:#}")))?; + serde_json::to_value(results).map_err(error) + }) + .await + .map_err(error)? +} + +#[napi] +pub async fn postprocess(scope: Value, selected: Value, options: Option) -> Result { + spawn_blocking(move || { + let scope = serde_json::from_value(scope).map_err(error)?; + let selected = serde_json::from_value(selected).map_err(error)?; + let options = options + .map(serde_json::from_value) + .transpose() + .map_err(error)? + .unwrap_or_default(); + let mut session = + configured_session(scope, options).map_err(|e| error(format!("{e:#}")))?; + let results = session + .postprocess(selected) + .map_err(|e| error(format!("{e:#}")))?; + serde_json::to_value(results).map_err(error) + }) + .await + .map_err(error)? +} diff --git a/src/cli.rs b/src/cli.rs index d7f313da4..c52fabb75 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -30,7 +30,7 @@ pub(crate) fn run() -> Result { paths }) .unwrap_or_default(); - let args = Command::new(env!("CARGO_BIN_NAME")) + let args = Command::new("diffr") .version(env!("CARGO_PKG_VERSION")) .about("Structural diffs with Git-style comparison inputs") .arg(Arg::new("repo").long("repo").default_value(".")) diff --git a/src/diff/sliders.rs b/src/diff/sliders.rs index 03b56c99d..281542c49 100644 --- a/src/diff/sliders.rs +++ b/src/diff/sliders.rs @@ -206,7 +206,7 @@ fn unchanged_descendants<'a>( /// Nested sliders require a single unchanged descendant whose /// delimiters we can slide. /// -/// ``` +/// ```text /// (old-1 (novel (old-2))) /// ``` /// diff --git a/src/diff/unchanged.rs b/src/diff/unchanged.rs index 33b85fb52..ec1d63879 100644 --- a/src/diff/unchanged.rs +++ b/src/diff/unchanged.rs @@ -195,7 +195,7 @@ fn is_mostly_unchanged_list(lhs: &Syntax, rhs: &Syntax) -> bool { /// This is important in cases where we have two adjacent lists that /// have a small number of changes. /// -/// ``` +/// ```text /// ; old /// (1 2 3 4) (a b c d) /// diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 000000000..11e0c3728 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,354 @@ +//! Difftastic is a syntactic diff tool. +//! +//! For usage instructions and advice on contributing, see [the +//! manual](http://difftastic.wilfred.me.uk/). +//! + +// I frequently develop difftastic on a newer rustc than the MSRV, so +// these two aren't relevant. +#![allow(renamed_and_removed_lints)] +// This tends to trigger on larger tuples of simple types, and naming +// them would probably be worse for readability. +#![allow(clippy::type_complexity)] +// == "" is often clearer when dealing with strings. +#![allow(clippy::comparison_to_empty)] +// It's common to have pairs foo_lhs and foo_rhs, leading to double +// the number of arguments and triggering this lint. +#![allow(clippy::too_many_arguments)] +// Has false positives on else if chains that sometimes have the same +// body for readability. +#![allow(clippy::if_same_then_else)] +// Good practice in general, but a necessary evil for Syntax. Its Hash +// implementation does not consider the mutable fields, so it is still +// correct. +#![allow(clippy::mutable_key_type)] +// manual_unwrap_or_default was added in Rust 1.79, so earlier versions of +// clippy complain about allowing it. +#![allow(unknown_lints)] +// It's sometimes more readable to explicitly create a vec than to use +// the Default trait. +#![allow(clippy::manual_unwrap_or_default)] +// I find the explicit arithmetic clearer sometimes. +#![allow(clippy::implicit_saturating_sub)] +// It's helpful being super explicit about byte length versus Unicode +// character point length sometimes. +#![allow(clippy::needless_as_bytes)] +// .to_owned() is more explicit on string references. +#![warn(clippy::str_to_string)] +// .to_string() on a String is clearer as .clone(). +#![warn(clippy::string_to_string)] +// Debugging features shouldn't be in checked-in code. +#![warn(clippy::todo)] +#![warn(clippy::dbg_macro)] + +mod cli; +mod config; +mod constants; +mod diff; +mod engine; +mod exit_codes; +use engine::diff_file_content; +mod files; +mod git; +mod gitattributes; +mod hash; +mod line_layout; +mod line_parser; +mod lines; +mod options; +mod pairing; +mod parse; +mod plugin; +pub(crate) mod protocol; +pub mod search; +pub mod storage; +mod summary; +mod tags; +mod version; +mod words; + +#[macro_use] +extern crate log; + +use crate::config::Params; + +use crate::exit_codes::EXIT_BAD_ARGUMENTS; +use crate::files::{guess_content, read_files_or_die, read_or_die, ProbableFileKind}; +use crate::gitattributes::{check_diff_attr, DiffAttribute}; +use crate::parse::guess_language::{ + guess, language_globs, language_name, Language, LanguageOverride, +}; +use crate::parse::syntax; + +/// The global allocator used by difftastic. +/// +/// Diffing allocates a large amount of memory, and both Jemalloc and +/// MiMalloc perform better than the system allocator. +/// +/// Some versions of MiMalloc (specifically libmimalloc-sys greater +/// than 0.1.24) handle very large, mostly unused allocations +/// badly. This makes large line-oriented diffs very slow, as +/// discussed in #297. +/// +/// MiMalloc is generally faster than Jemalloc, but older versions of +/// MiMalloc don't compile on GCC 15+, so use Jemalloc for now. See +/// #805. +/// +/// For reference, Jemalloc uses 10-20% more time (although up to 33% +/// more instructions) when testing on sample files. +#[cfg(not(any(windows, target_os = "illumos", target_os = "freebsd")))] +use tikv_jemallocator::Jemalloc; + +#[cfg(not(any(windows, target_os = "illumos", target_os = "freebsd")))] +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; + +use std::path::Path; + +use strum::IntoEnumIterator; +use typed_arena::Arena; + +use crate::engine::QueryConflict; +use crate::options::{DiffOptions, FileArgument, Mode}; +use crate::parse::folds::Conflict; +use crate::parse::syntax::init_all_info; +use crate::parse::tree_sitter_parser as tsp; +use crate::summary::{DiffResult, FileContent, FileFormat}; + +extern crate pretty_env_logger; + +/// Terminate the process if we get SIGPIPE. +#[cfg(unix)] +fn reset_sigpipe() { + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } +} + +#[cfg(not(unix))] +fn reset_sigpipe() { + // Do nothing. +} + +/// The entrypoint. +pub fn run_cli() { + pretty_env_logger::try_init_timed_custom_env("DFT_LOG") + .expect("The logger has not been previously initialized"); + reset_sigpipe(); + + let result = match std::env::args_os().nth(1).as_deref() { + Some(arg) if arg == "debug" => { + run_debug(); + return; + } + _ => cli::run(), + }; + match result { + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("{error}"); + std::process::exit(2); + } + } +} + +fn run_debug() { + let params = &Params::default(); + + match options::parse_args() { + Mode::DumpTreeSitter { + path, + language_overrides, + } => { + let path = Path::new(&path); + let bytes = read_or_die(path); + let src = String::from_utf8_lossy(&bytes).to_string(); + + let language = guess(path, &src, &language_overrides); + match language { + Some(lang) => { + let ts_lang = tsp::from_language(lang); + let tree = tsp::to_tree(&src, ts_lang); + tsp::print_tree(&src, &tree); + } + None => { + eprintln!("No tree-sitter parser for file: {:?}", path); + } + } + } + Mode::DumpSyntax { + path, + ignore_comments, + language_overrides, + } => { + let path = Path::new(&path); + let bytes = read_or_die(path); + let src = String::from_utf8_lossy(&bytes).to_string(); + + let language = guess(path, &src, &language_overrides); + match language { + Some(lang) => { + let ts_lang = params.language(lang); + let arena = Arena::new(); + let ast = conflict_or_die(tsp::parse(&arena, &src, ts_lang, ignore_comments)); + init_all_info(&ast, &[]); + println!("{:#?}", ast); + } + None => { + eprintln!("No tree-sitter parser for file: {:?}", path); + } + } + } + Mode::DumpSyntaxDot { + path, + ignore_comments, + language_overrides, + } => { + let path = Path::new(&path); + let bytes = read_or_die(path); + let src = String::from_utf8_lossy(&bytes).to_string(); + + let language = guess(path, &src, &language_overrides); + match language { + Some(lang) => { + let ts_lang = params.language(lang); + let arena = Arena::new(); + let ast = conflict_or_die(tsp::parse(&arena, &src, ts_lang, ignore_comments)); + init_all_info(&ast, &[]); + syntax::print_as_dot(&ast); + } + None => { + eprintln!("No tree-sitter parser for file: {:?}", path); + } + } + } + Mode::ListLanguages { language_overrides } => { + for (lang_override, globs) in language_overrides { + let name = match lang_override { + LanguageOverride::Language(lang) => language_name(lang), + LanguageOverride::PlainText => "Text", + }; + println!("{} (from override)", name); + for glob in globs { + print!(" {}", glob.as_str()); + } + println!(); + } + + for language in Language::iter() { + println!("{}", language_name(language)); + + for glob in language_globs(language) { + print!(" {}", glob.as_str()); + } + println!(); + } + } + }; +} + +/// Diff two files: `--no-index`. +fn diff_file( + params: &Params, + display_path: &str, + lhs_path: &FileArgument, + rhs_path: &FileArgument, + diff_options: &DiffOptions, + missing_as_empty: bool, + overrides: &[(LanguageOverride, Vec)], + binary_overrides: &[glob::Pattern], +) -> Result { + let (lhs_bytes, rhs_bytes) = read_files_or_die(lhs_path, rhs_path, missing_as_empty); + + let (mut lhs_src, mut rhs_src) = match ( + guess_content(&lhs_bytes, lhs_path, binary_overrides), + guess_content(&rhs_bytes, rhs_path, binary_overrides), + check_diff_attr(Path::new(display_path)), + ) { + (ProbableFileKind::Binary, _, _) + | (_, ProbableFileKind::Binary, _) + | (_, _, Some(DiffAttribute::AssumeBinary)) => { + return Ok(DiffResult { + file_format: FileFormat::Binary, + lhs_src: FileContent::Binary, + rhs_src: FileContent::Binary, + lhs_positions: vec![], + rhs_positions: vec![], + lhs_folds: vec![], + rhs_folds: vec![], + }); + } + (ProbableFileKind::Text(lhs_src), ProbableFileKind::Text(rhs_src), _) => (lhs_src, rhs_src), + }; + + // Ensure that lhs_src and rhs_src both have trailing + // newlines. + // + // This is important when textually diffing files that don't have + // a trailing newline, e.g. "foo\n\bar\n" versus "foo". We want to + // consider `foo` to be unchanged in this case. + // + // Theoretically a tree-sitter parser could change its AST due to + // the additional trailing newline, but it seems vanishingly + // unlikely. + if !lhs_src.is_empty() && !lhs_src.ends_with('\n') { + lhs_src.push('\n'); + } + if !rhs_src.is_empty() && !rhs_src.ends_with('\n') { + rhs_src.push('\n'); + } + + diff_file_content( + params, + display_path, + lhs_path, + rhs_path, + &lhs_src, + &rhs_src, + diff_options, + overrides, + ) +} + +/// The syntax dumps stop at a fold query conflict. +fn conflict_or_die(result: Result) -> T { + match result { + Ok(value) => value, + Err(conflict) => { + eprintln!( + "line {}: {} and {} capture the same {} with different fold ranges", + conflict.line + 1, + conflict.sources.0, + conflict.sources.1, + conflict.kind + ); + std::process::exit(EXIT_BAD_ARGUMENTS); + } + } +} + +#[cfg(test)] +mod tests { + use std::ffi::OsStr; + + use super::*; + + #[test] + fn test_diff_identical_content() { + let s = "foo"; + let res = diff_file_content( + &Params::default(), + "foo.el", + &FileArgument::from_path_argument(OsStr::new("foo.el")), + &FileArgument::from_path_argument(OsStr::new("foo.el")), + s, + s, + &DiffOptions::default(), + &[], + ) + .unwrap(); + + assert_eq!(res.lhs_positions, vec![]); + assert_eq!(res.rhs_positions, vec![]); + } +} diff --git a/src/main.rs b/src/main.rs index 6dd75d718..5ce9506df 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,352 +1,4 @@ -//! Difftastic is a syntactic diff tool. -//! -//! For usage instructions and advice on contributing, see [the -//! manual](http://difftastic.wilfred.me.uk/). -//! - -// I frequently develop difftastic on a newer rustc than the MSRV, so -// these two aren't relevant. -#![allow(renamed_and_removed_lints)] -// This tends to trigger on larger tuples of simple types, and naming -// them would probably be worse for readability. -#![allow(clippy::type_complexity)] -// == "" is often clearer when dealing with strings. -#![allow(clippy::comparison_to_empty)] -// It's common to have pairs foo_lhs and foo_rhs, leading to double -// the number of arguments and triggering this lint. -#![allow(clippy::too_many_arguments)] -// Has false positives on else if chains that sometimes have the same -// body for readability. -#![allow(clippy::if_same_then_else)] -// Good practice in general, but a necessary evil for Syntax. Its Hash -// implementation does not consider the mutable fields, so it is still -// correct. -#![allow(clippy::mutable_key_type)] -// manual_unwrap_or_default was added in Rust 1.79, so earlier versions of -// clippy complain about allowing it. -#![allow(unknown_lints)] -// It's sometimes more readable to explicitly create a vec than to use -// the Default trait. -#![allow(clippy::manual_unwrap_or_default)] -// I find the explicit arithmetic clearer sometimes. -#![allow(clippy::implicit_saturating_sub)] -// It's helpful being super explicit about byte length versus Unicode -// character point length sometimes. -#![allow(clippy::needless_as_bytes)] -// .to_owned() is more explicit on string references. -#![warn(clippy::str_to_string)] -// .to_string() on a String is clearer as .clone(). -#![warn(clippy::string_to_string)] -// Debugging features shouldn't be in checked-in code. -#![warn(clippy::todo)] -#![warn(clippy::dbg_macro)] - -mod cli; -mod config; -mod constants; -mod diff; -mod engine; -mod exit_codes; -use engine::diff_file_content; -mod files; -mod git; -mod gitattributes; -mod hash; -mod line_layout; -mod line_parser; -mod lines; -mod options; -mod pairing; -mod parse; -mod plugin; -pub(crate) mod protocol; -mod summary; -mod tags; -mod version; -mod words; - -#[macro_use] -extern crate log; - -use crate::config::Params; - -use crate::exit_codes::EXIT_BAD_ARGUMENTS; -use crate::files::{guess_content, read_files_or_die, read_or_die, ProbableFileKind}; -use crate::gitattributes::{check_diff_attr, DiffAttribute}; -use crate::parse::guess_language::{ - guess, language_globs, language_name, Language, LanguageOverride, -}; -use crate::parse::syntax; - -/// The global allocator used by difftastic. -/// -/// Diffing allocates a large amount of memory, and both Jemalloc and -/// MiMalloc perform better than the system allocator. -/// -/// Some versions of MiMalloc (specifically libmimalloc-sys greater -/// than 0.1.24) handle very large, mostly unused allocations -/// badly. This makes large line-oriented diffs very slow, as -/// discussed in #297. -/// -/// MiMalloc is generally faster than Jemalloc, but older versions of -/// MiMalloc don't compile on GCC 15+, so use Jemalloc for now. See -/// #805. -/// -/// For reference, Jemalloc uses 10-20% more time (although up to 33% -/// more instructions) when testing on sample files. -#[cfg(not(any(windows, target_os = "illumos", target_os = "freebsd")))] -use tikv_jemallocator::Jemalloc; - -#[cfg(not(any(windows, target_os = "illumos", target_os = "freebsd")))] -#[global_allocator] -static GLOBAL: Jemalloc = Jemalloc; - -use std::path::Path; - -use strum::IntoEnumIterator; -use typed_arena::Arena; - -use crate::engine::QueryConflict; -use crate::options::{DiffOptions, FileArgument, Mode}; -use crate::parse::folds::Conflict; -use crate::parse::syntax::init_all_info; -use crate::parse::tree_sitter_parser as tsp; -use crate::summary::{DiffResult, FileContent, FileFormat}; - -extern crate pretty_env_logger; - -/// Terminate the process if we get SIGPIPE. -#[cfg(unix)] -fn reset_sigpipe() { - unsafe { - libc::signal(libc::SIGPIPE, libc::SIG_DFL); - } -} - -#[cfg(not(unix))] -fn reset_sigpipe() { - // Do nothing. -} - -/// The entrypoint. +//! Command-line entry point; the shared engine also backs `diffr/api`. fn main() { - pretty_env_logger::try_init_timed_custom_env("DFT_LOG") - .expect("The logger has not been previously initialized"); - reset_sigpipe(); - - let result = match std::env::args_os().nth(1).as_deref() { - Some(arg) if arg == "debug" => { - run_debug(); - return; - } - _ => cli::run(), - }; - match result { - Ok(code) => std::process::exit(code), - Err(error) => { - eprintln!("{error}"); - std::process::exit(2); - } - } -} - -fn run_debug() { - let params = &Params::default(); - - match options::parse_args() { - Mode::DumpTreeSitter { - path, - language_overrides, - } => { - let path = Path::new(&path); - let bytes = read_or_die(path); - let src = String::from_utf8_lossy(&bytes).to_string(); - - let language = guess(path, &src, &language_overrides); - match language { - Some(lang) => { - let ts_lang = tsp::from_language(lang); - let tree = tsp::to_tree(&src, ts_lang); - tsp::print_tree(&src, &tree); - } - None => { - eprintln!("No tree-sitter parser for file: {:?}", path); - } - } - } - Mode::DumpSyntax { - path, - ignore_comments, - language_overrides, - } => { - let path = Path::new(&path); - let bytes = read_or_die(path); - let src = String::from_utf8_lossy(&bytes).to_string(); - - let language = guess(path, &src, &language_overrides); - match language { - Some(lang) => { - let ts_lang = params.language(lang); - let arena = Arena::new(); - let ast = conflict_or_die(tsp::parse(&arena, &src, ts_lang, ignore_comments)); - init_all_info(&ast, &[]); - println!("{:#?}", ast); - } - None => { - eprintln!("No tree-sitter parser for file: {:?}", path); - } - } - } - Mode::DumpSyntaxDot { - path, - ignore_comments, - language_overrides, - } => { - let path = Path::new(&path); - let bytes = read_or_die(path); - let src = String::from_utf8_lossy(&bytes).to_string(); - - let language = guess(path, &src, &language_overrides); - match language { - Some(lang) => { - let ts_lang = params.language(lang); - let arena = Arena::new(); - let ast = conflict_or_die(tsp::parse(&arena, &src, ts_lang, ignore_comments)); - init_all_info(&ast, &[]); - syntax::print_as_dot(&ast); - } - None => { - eprintln!("No tree-sitter parser for file: {:?}", path); - } - } - } - Mode::ListLanguages { language_overrides } => { - for (lang_override, globs) in language_overrides { - let name = match lang_override { - LanguageOverride::Language(lang) => language_name(lang), - LanguageOverride::PlainText => "Text", - }; - println!("{} (from override)", name); - for glob in globs { - print!(" {}", glob.as_str()); - } - println!(); - } - - for language in Language::iter() { - println!("{}", language_name(language)); - - for glob in language_globs(language) { - print!(" {}", glob.as_str()); - } - println!(); - } - } - }; -} - -/// Diff two files: `--no-index`. -fn diff_file( - params: &Params, - display_path: &str, - lhs_path: &FileArgument, - rhs_path: &FileArgument, - diff_options: &DiffOptions, - missing_as_empty: bool, - overrides: &[(LanguageOverride, Vec)], - binary_overrides: &[glob::Pattern], -) -> Result { - let (lhs_bytes, rhs_bytes) = read_files_or_die(lhs_path, rhs_path, missing_as_empty); - - let (mut lhs_src, mut rhs_src) = match ( - guess_content(&lhs_bytes, lhs_path, binary_overrides), - guess_content(&rhs_bytes, rhs_path, binary_overrides), - check_diff_attr(Path::new(display_path)), - ) { - (ProbableFileKind::Binary, _, _) - | (_, ProbableFileKind::Binary, _) - | (_, _, Some(DiffAttribute::AssumeBinary)) => { - return Ok(DiffResult { - file_format: FileFormat::Binary, - lhs_src: FileContent::Binary, - rhs_src: FileContent::Binary, - lhs_positions: vec![], - rhs_positions: vec![], - lhs_folds: vec![], - rhs_folds: vec![], - }); - } - (ProbableFileKind::Text(lhs_src), ProbableFileKind::Text(rhs_src), _) => (lhs_src, rhs_src), - }; - - // Ensure that lhs_src and rhs_src both have trailing - // newlines. - // - // This is important when textually diffing files that don't have - // a trailing newline, e.g. "foo\n\bar\n" versus "foo". We want to - // consider `foo` to be unchanged in this case. - // - // Theoretically a tree-sitter parser could change its AST due to - // the additional trailing newline, but it seems vanishingly - // unlikely. - if !lhs_src.is_empty() && !lhs_src.ends_with('\n') { - lhs_src.push('\n'); - } - if !rhs_src.is_empty() && !rhs_src.ends_with('\n') { - rhs_src.push('\n'); - } - - diff_file_content( - params, - display_path, - lhs_path, - rhs_path, - &lhs_src, - &rhs_src, - diff_options, - overrides, - ) -} - -/// The syntax dumps stop at a fold query conflict. -fn conflict_or_die(result: Result) -> T { - match result { - Ok(value) => value, - Err(conflict) => { - eprintln!( - "line {}: {} and {} capture the same {} with different fold ranges", - conflict.line + 1, - conflict.sources.0, - conflict.sources.1, - conflict.kind - ); - std::process::exit(EXIT_BAD_ARGUMENTS); - } - } -} - -#[cfg(test)] -mod tests { - use std::ffi::OsStr; - - use super::*; - - #[test] - fn test_diff_identical_content() { - let s = "foo"; - let res = diff_file_content( - &Params::default(), - "foo.el", - &FileArgument::from_path_argument(OsStr::new("foo.el")), - &FileArgument::from_path_argument(OsStr::new("foo.el")), - s, - s, - &DiffOptions::default(), - &[], - ) - .unwrap(); - - assert_eq!(res.lhs_positions, vec![]); - assert_eq!(res.rhs_positions, vec![]); - } + difftastic::run_cli(); } diff --git a/src/options.rs b/src/options.rs index 2186a78c8..d2fcf1fe5 100644 --- a/src/options.rs +++ b/src/options.rs @@ -18,7 +18,7 @@ pub(crate) const DEFAULT_BYTE_LIMIT: usize = 1_000_000; pub(crate) const DEFAULT_GRAPH_LIMIT: usize = 3_000_000; pub(crate) const DEFAULT_PARSE_ERROR_LIMIT: usize = 0; -pub(crate) const USAGE: &str = concat!(env!("CARGO_BIN_NAME"), " debug [OPTIONS]"); +pub(crate) const USAGE: &str = "diffr debug [OPTIONS]"; pub(crate) const DEFAULT_TERMINAL_WIDTH: usize = 80; @@ -93,13 +93,13 @@ fn app() -> clap::Command { .action(ArgAction::Append) .help(concat!("Associate this glob pattern with this language, overriding normal language detection. For example: -$ ", env!("CARGO_BIN_NAME"), " debug --override='*.c:C++' --dump-syntax file.c +$ ", "diffr", " debug --override='*.c:C++' --dump-syntax file.c See --list-languages for the list of language names. Language names are matched case insensitively. Overrides may also specify the language \"text\" to treat a file as plain text. This argument may be given more than once. For example: -$ ", env!("CARGO_BIN_NAME"), " debug --override='CustomFile:json' --override='*.c:text' --dump-syntax file.c +$ ", "diffr", " debug --override='CustomFile:json' --override='*.c:text' --dump-syntax file.c To configure multiple overrides using environment variables, difftastic also accepts DFT_OVERRIDE_1 up to DFT_OVERRIDE_9. diff --git a/src/search.rs b/src/search.rs index b173d60dc..b4fed8803 100644 --- a/src/search.rs +++ b/src/search.rs @@ -80,3 +80,799 @@ pub struct Options { plugins: Option, } +// The host keeps one active index. Only a storage backend/location change replaces +// it; scopes and analysis settings do not participate in its lifetime. +type ConfiguredIndex = (DiffKey, Arc>); +static INDEX: LazyLock>> = LazyLock::new(|| Mutex::new(None)); + +/// Create comparison-local analysis state using the host's shared computed index. +/// Storage remains entirely outside the serialized client API. +pub fn configured_session(scope: Scope, options: Options) -> anyhow::Result> { + let config = Config::load(None)?; + let index = index_for_config(&scope.repo, &config)?; + let mut session = Session::with_config(scope, index, config)?; + // Overrides select mutation behavior; hydration's indexed structure is stable. + if let Some(mut plugins) = options.plugins { + plugins.resolve(&session.scope.repo)?; + session.pipeline = Pipeline::from_config(&plugins, &session.scope.repo)?; + } + Ok(session) +} + +fn index_for_config(repository: &Path, config: &Config) -> anyhow::Result>> { + let repository = repository.canonicalize()?; + let directory = match config.storage.backend { + crate::storage::StoreBackend::Memory => None, + _ => Some(repository.join(&config.storage.path)), + }; + let key = DiffKey::new(&(config.storage.backend, directory))?; + let mut current = INDEX + .lock() + .map_err(|_| anyhow!("search index lock poisoned"))?; + if let Some((identity, index)) = &*current { + if identity == &key { + return Ok(index.clone()); + } + } + let index = Arc::new(Index::new(config.storage.open(&repository)?)); + *current = Some((key, index.clone())); + Ok(index) +} + +impl Session { + /// Create a comparison using an existing index and default analysis settings. + pub fn new(scope: Scope, index: Arc>) -> anyhow::Result { + Self::with_config(scope, index, Config::default()) + } + + fn with_config(scope: Scope, index: Arc>, config: Config) -> anyhow::Result { + validate(&scope)?; + let pipeline = Pipeline::from_config(&config.plugins, &scope.repo)?; + let queries = pipeline.queries()?; + let resolved: BTreeMap<_, Vec<_>> = crate::plugin::queries::assemble(&queries)? + .into_iter() + .map(|(language, sources)| { + ( + language, + sources + .into_iter() + .map(|source| (source.name, source.text)) + .collect(), + ) + }) + .collect(); + let analysis = DiffKey::new(&( + serde_json::to_value(&config.diff)?, + serde_json::to_value(&config.plugins)?, + resolved, + ))? + .as_str() + .to_owned(); + let params = Arc::new(config.compile_queries(queries)?); + let session = git::DiffSession::open( + &scope.repo, + git::Comparison { + before: git::Operand::revision(&scope.base_worktree.commit_id), + after: git::Operand::revision(&scope.head_worktree.commit_id), + }, + params.clone(), + &git::FileParams::default(), + &pipeline, + ) + .map_err(|e| anyhow!("{e}"))?; + Ok(Self { + scope, + index, + params, + pipeline, + manifest: session + .file_manifest() + .iter() + .map(|f| f.manifest_entry()) + .collect(), + analysis, + }) + } + /// Hydrate line hits against pinned Git blobs. Cached structural trees never + /// contain query-specific spans or mutable presentation state. + pub fn hydrate(&mut self, hits: Vec) -> anyhow::Result> { + validate(&self.scope)?; + let scope = self.scope.clone(); + let mut selected: BTreeMap = BTreeMap::new(); + for hit in hits { + ensure!(hit.file.is_absolute(), "hit paths must be absolute"); + let path = hit.file.canonicalize().context("hit path does not exist")?; + let mut located = None; + for (right, worktree) in [(false, &scope.base_worktree), (true, &scope.head_worktree)] { + if let Ok(relative) = path.strip_prefix(worktree.path.canonicalize()?) { + located = Some(( + right, + relative.to_str().context("non-UTF8 hit path")?.to_owned(), + )); + } + } + let (right, path) = located.context("hit path is outside scoped worktrees")?; + let indexed = self.index.resolve(self, right, &path)?; + let key = key(&indexed.entry.file); + let result = selected.entry(key).or_insert_with(|| SearchResult { + display: Display::Both, + scope: scope.clone(), + comparison: indexed.comparison(), + }); + let output = result + .comparison + .source_mut(right) + .context("hit side absent")?; + for line in hit.lines { + ensure!(line.line > 0, "hit line numbers are 1-based"); + let text = output + .text + .split_terminator('\n') + .nth((line.line - 1) as usize) + .context("hit line is out of bounds")?; + let text = text.strip_suffix('\r').unwrap_or(text); + ensure!( + text == line.text, + "hit text differs from pinned source at {path}:{}", + line.line + ); + let span = Span { + line: line.line - 1, + start_column: 0, + end_column: u32::try_from(text.len())?, + }; + let mut spans = Vec::new(); + collect(&output.regions, &mut spans); + spans.push(span); + spans.sort_by_key(|s| (s.line, s.start_column, s.end_column)); + spans.dedup(); + attach(&mut output.regions, &spans); + } + } + for result in selected.values_mut() { + for source in result.comparison.sources_mut() { + evidence_visibility(&mut source.regions); + } + } + Ok(selected.into_values().collect()) + } + + /// Validate complete hydrated comparisons and process them without projecting sides. + pub fn postprocess( + &mut self, + selected: Vec, + ) -> anyhow::Result> { + validate(&self.scope)?; + let mut groups: BTreeMap = BTreeMap::new(); + for mut result in selected { + ensure!( + result.scope == self.scope, + "selected result belongs to another scope" + ); + ensure!( + match result.display { + Display::Both => true, + Display::Lhs => result.comparison.source(false).is_some(), + Display::Rhs => result.comparison.source(true).is_some(), + }, + "display requests an absent side" + ); + let files = result.comparison.files(); + let (right, file) = sides(&files)[0]; + let indexed = self.index.resolve(self, right, &file.path)?; + let mut original = indexed.comparison(); + let mut submitted = result.comparison.clone(); + // Visibility is presentation state, highlights are query state. All other + // tree data must still be the indexed computation supplied by hydration. + for source in submitted.sources_mut() { + clear_query(&mut source.regions); + } + for source in original.sources_mut() { + clear_query(&mut source.regions); + } + ensure!( + submitted == original, + "hydrated comparison differs from index" + ); + for (source, baseline) in result + .comparison + .sources_mut() + .into_iter() + .zip(indexed.comparison().sources_mut()) + { + let mut spans = Vec::new(); + collect(&source.regions, &mut spans); + spans.sort_by_key(|s| (s.line, s.start_column, s.end_column)); + spans.dedup(); + for span in &spans { + let line = source + .text + .split_terminator('\n') + .nth(span.line as usize) + .context("highlight line out of bounds")?; + ensure!( + span.start_column <= span.end_column + && span.end_column as usize <= line.len() + && line.is_char_boundary(span.start_column as usize) + && line.is_char_boundary(span.end_column as usize), + "invalid highlight coordinates" + ); + } + reset_visibility(&mut source.regions, &baseline.regions); + attach(&mut source.regions, &spans); + } + let group_key = serde_json::to_string(&(result.display, result.comparison.files()))?; + if let Some((_, group)) = groups.get_mut(&group_key) { + // Only selected evidence is combined; the complete trees are retained. + for (target, incoming) in group + .comparison + .sources_mut() + .into_iter() + .zip(result.comparison.sources()) + { + let mut spans = Vec::new(); + collect(&target.regions, &mut spans); + collect(&incoming.regions, &mut spans); + spans.sort_by_key(|s| (s.line, s.start_column, s.end_column)); + spans.dedup(); + attach(&mut target.regions, &spans); + } + } else { + groups.insert(group_key, (indexed.entry, result)); + } + } + let mut processed: BTreeMap<_, Comparison> = BTreeMap::new(); + let mut results = Vec::new(); + for (_, (entry, mut result)) in groups { + let selection: Vec<_> = result + .comparison + .sources() + .into_iter() + .map(|source| { + let mut spans = Vec::new(); + collect(&source.regions, &mut spans); + spans + .iter() + .map(|s| (s.line, s.start_column, s.end_column)) + .collect::>() + }) + .collect(); + // Reuse processing across display choices, keyed by file identity and hits. + let identity = (key(&entry.file), selection); + if let Some(comparison) = processed.get(&identity) { + result.comparison = comparison.clone(); + } else { + self.pipeline.run(&entry, &mut result.comparison)?; + processed.insert(identity, result.comparison.clone()); + } + results.push(result); + } + Ok(results) + } +} + +impl StoredDiff { + fn comparison(&self) -> Comparison { + match (&self.entry.file, &self.sources) { + ( + Pairing::Both { + lhs: lhs_file, + rhs: rhs_file, + }, + Pairing::Both { lhs, rhs }, + ) if lhs.text == rhs.text => Comparison::Same { + lhs_file: lhs_file.clone(), + rhs_file: rhs_file.clone(), + source: rhs.clone(), + }, + _ => Comparison::from_sides(self.entry.file.clone(), self.sources.clone()) + .expect("indexed sides agree"), + } + } +} +fn clear_query(regions: &mut [Region]) { + for region in regions { + region.visibility = protocol::Visibility::default(); + match &mut region.node { + Node::Leaf { + search_highlights, .. + } => search_highlights.clear(), + Node::Fold { children } => clear_query(children), + } + } +} +fn reset_visibility(regions: &mut [Region], baseline: &[Region]) { + for (region, original) in regions.iter_mut().zip(baseline) { + region.visibility = original.visibility.clone(); + if let (Node::Fold { children }, Node::Fold { children: original }) = + (&mut region.node, &original.node) + { + reset_visibility(children, original); + } + } +} + +impl Index { + pub fn new(store: Arc) -> Self { + Self { store } + } + + fn resolve(&self, session: &Session, right: bool, path: &str) -> anyhow::Result { + let scope = &session.scope; + let entry = if let Some(entry) = session.manifest.iter().find(|entry| { + sides(&entry.file) + .iter() + .any(|(side, file)| *side == right && file.path == path) + }) { + entry.clone() + } else { + let repo = Repository::open(&scope.repo)?; + let file = |worktree: &Worktree| -> anyhow::Result { + let tree = repo + .find_commit(Oid::from_str(&worktree.commit_id)?)? + .tree()?; + let entry = tree + .get_path(Path::new(path)) + .context("hit does not resolve to an indexed blob")?; + ensure!( + entry.kind() == Some(git2::ObjectType::Blob) + && entry.filemode() & 0o170000 == 0o100000, + "search requires regular text files" + ); + Ok(FileRef { + path: path.to_owned(), + oid: entry.id().to_string(), + mode: format!("{:o}", entry.filemode()), + }) + }; + let lhs = file(&scope.base_worktree)?; + let rhs = file(&scope.head_worktree)?; + ensure!( + lhs.oid == rhs.oid, + "changed file missing from comparison manifest" + ); + let mut entry = FileChange { + file: Pairing::Both { lhs, rhs }, + status: FileStatus::Unchanged, + tags: crate::tags::from_path(path) + .into_iter() + .map(str::to_owned) + .collect(), + }; + entry.tags = session.pipeline.classify(&entry)?; + entry + }; + let cache_key = DiffKey::new(&(&session.analysis, path, &entry))?; + if let Some(diff) = self.store.get(&cache_key)? { + return Ok(diff); + } + let repo = Repository::open(&scope.repo)?; + let read = |file: &FileRef| -> anyhow::Result { + ensure!( + file.mode == "100644" || file.mode == "100755", + "search requires regular text files" + ); + let blob = repo.find_blob(Oid::from_str(&file.oid)?)?; + ensure!(!blob.is_binary(), "search requires text files"); + Ok(std::str::from_utf8(blob.content())?.to_owned()) + }; + let mut text = [String::new(), String::new()]; + for (right, file) in sides(&entry.file) { + text[usize::from(right)] = read(file)?; + } + let options = crate::options::DiffOptions { + generated: entry.tags.iter().any(|tag| tag == "generated"), + ..session.params.diff.options(false) + }; + let mut diff = DiffResult::from_sources_with_options( + path, + &text[0], + &text[1], + &session.params, + &options, + )?; + // Parse an identical blob once for search context, independently of diffing. + if text[0] == text[1] && !options.generated && text[0].len() <= options.byte_limit { + if let Some(language) = + crate::parse::guess_language::guess(Path::new(path), &text[0], &[]) + { + let config = session.params.language(language); + let tree = crate::parse::tree_sitter_parser::to_tree(&text[0], config.parser); + let arena = typed_arena::Arena::new(); + let (nodes, _) = crate::parse::tree_sitter_parser::to_syntax( + &tree, &text[0], &arena, config, false, + ) + .map_err(|e| anyhow!("fold query conflict: {e:?}"))?; + crate::parse::folds::unmatched(&nodes, &mut diff.lhs_folds); + let count = diff.lhs_folds.len(); + for (i, fold) in diff.lhs_folds.iter_mut().enumerate() { + let lhs = std::num::NonZeroU32::new(i as u32 + 1).unwrap(); + let rhs = std::num::NonZeroU32::new((count + i) as u32 + 1).unwrap(); + fold.syntax_id = lhs; + fold.match_kind = crate::parse::folds::FoldMatch::Matched { opposite: rhs }; + diff.rhs_folds.push(crate::parse::folds::Fold { + tags: fold.tags.clone(), + range: fold.range, + syntax_id: rhs, + match_kind: crate::parse::folds::FoldMatch::Matched { opposite: lhs }, + placeholder: String::new(), + }); + } + } + } + let syntax = |source: &str| { + crate::parse::guess_language::guess(Path::new(path), source, &[]) + .map(|language| { + protocol::project::syntax_spans( + source, + session.params.language(language).parser, + ) + }) + .unwrap_or_default() + }; + let protocol::Diff::Text { sides, .. } = protocol::project::diff( + &diff, + protocol::project::Inputs { + file: &entry.file, + sizes: (text[0].len() as u64, text[1].len() as u64), + syntax: (syntax(&text[0]), syntax(&text[1])), + }, + ) else { + bail!("search requires text files") + }; + let stored = StoredDiff { + entry, + sources: sides, + }; + self.store.put(&cache_key, &stored)?; + Ok(stored) + } +} + +fn sides(pair: &Pairing) -> Vec<(bool, &T)> { + match pair { + Pairing::Both { lhs, rhs } => vec![(false, lhs), (true, rhs)], + Pairing::LeftOnly { lhs } => vec![(false, lhs)], + Pairing::RightOnly { rhs } => vec![(true, rhs)], + } +} +fn key(file: &Pairing) -> String { + serde_json::to_string(file).expect("file refs serialize") +} +fn validate(scope: &Scope) -> anyhow::Result<()> { + let repo = Repository::open(&scope.repo)?; + for worktree in [&scope.base_worktree, &scope.head_worktree] { + ensure!( + worktree.path.is_absolute(), + "worktree paths must be absolute" + ); + let pinned = Oid::from_str(&worktree.commit_id)?; + repo.find_commit(pinned)?; + let checkout = Repository::open(&worktree.path)?; + ensure!( + checkout.head()?.target() == Some(pinned), + "worktree HEAD does not match its commitId" + ); + ensure!( + checkout.commondir().canonicalize()? == repo.commondir().canonicalize()?, + "worktree belongs to another repository" + ); + } + ensure!( + scope.base_worktree.path.canonicalize()? != scope.head_worktree.path.canonicalize()?, + "worktrees must be distinct" + ); + Ok(()) +} +fn attach(regions: &mut [Region], spans: &[Span]) { + for region in regions { + match &mut region.node { + Node::Leaf { + search_highlights, .. + } => { + *search_highlights = spans + .iter() + .copied() + .filter(|s| region.range.lines().contains(&s.line)) + .collect(); + } + Node::Fold { children } => attach(children, spans), + } + } +} +fn collect(regions: &[Region], spans: &mut Vec) { + for region in regions { + match ®ion.node { + Node::Leaf { + search_highlights, .. + } => spans.extend(search_highlights), + Node::Fold { children } => collect(children, spans), + } + } +} +fn evidence_visibility(regions: &mut [Region]) -> bool { + let mut any = false; + for region in regions { + let highlighted = match &mut region.node { + Node::Leaf { + search_highlights, .. + } => !search_highlights.is_empty(), + Node::Fold { children } => evidence_visibility(children), + }; + region.visibility.collapsed = !highlighted; + any |= highlighted; + } + any +} + +#[cfg(test)] +mod storage_tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct ReadOnlyStore { + store: Arc, + hits: AtomicUsize, + } + impl Store for ReadOnlyStore { + fn get(&self, key: &DiffKey) -> anyhow::Result> { + let stored = self.store.get(key)?; + if let Some(diff) = &stored { + self.hits.fetch_add(1, Ordering::SeqCst); + let text = serde_json::to_string(diff)?; + assert!(!text.contains("search_highlights")); + assert!(!text.contains("\"collapsed\":true")); + } + Ok(stored) + } + fn put(&self, _: &DiffKey, _: &StoredDiff) -> anyhow::Result<()> { + bail!("unexpected recomputation on a warm store") + } + fn remove(&self, key: &DiffKey) -> anyhow::Result<()> { + self.store.remove(key) + } + } + fn fixture() -> (tempfile::TempDir, Scope, Vec) { + let temporary = tempfile::tempdir().unwrap(); + let repo = temporary.path().join("repo"); + std::fs::create_dir(&repo).unwrap(); + let git = |args: &[&str]| { + let output = std::process::Command::new("git") + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgSign=false", + "-c", + "core.hooksPath=/dev/null", + ]) + .args(args) + .current_dir(&repo) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().to_owned() + }; + git(&["init", "--quiet"]); + std::fs::write( + repo.join("retry.js"), + "function retry() {\n return 1; // token\n}\nfunction helper() {\n setup();\n work();\n finish();\n}\n", + ) + .unwrap(); + std::fs::write( + repo.join("same.js"), + "function same() {\n return 'token';\n}\n", + ) + .unwrap(); + git(&["add", "."]); + git(&["commit", "--quiet", "-m", "base"]); + let base = git(&["rev-parse", "HEAD"]); + std::fs::write( + repo.join("retry.js"), + "function retry() {\n return 2; // token\n}\nfunction helper() {\n setup();\n work();\n finish();\n}\n", + ) + .unwrap(); + git(&["commit", "--quiet", "-am", "head"]); + let head = git(&["rev-parse", "HEAD"]); + let base_path = temporary.path().join("base"); + let head_path = temporary.path().join("head"); + git(&[ + "worktree", + "add", + "--quiet", + "--detach", + base_path.to_str().unwrap(), + &base, + ]); + git(&[ + "worktree", + "add", + "--quiet", + "--detach", + head_path.to_str().unwrap(), + &head, + ]); + let scope = Scope { + repo, + base_worktree: Worktree { + path: base_path, + commit_id: base, + }, + head_worktree: Worktree { + path: head_path.clone(), + commit_id: head, + }, + }; + let hits = vec![ + Hit { + file: head_path.join("retry.js"), + lines: vec![HitLine { + line: 2, + text: " return 2; // token".into(), + }], + }, + Hit { + file: head_path.join("same.js"), + lines: vec![HitLine { + line: 2, + text: " return 'token';".into(), + }], + }, + ]; + (temporary, scope, hits) + } + #[test] + fn side_selection_preserves_original_pairing_and_unchanged_sources() { + let (_temporary, scope, hits) = fixture(); + let config = Config::from_toml( + r#" +[plugins.bundled.summarize] +enabled = true +api_key = "test" +endpoint = "http://127.0.0.1:9" +min_lines = 1 +retries = 0 +"#, + ) + .unwrap(); + let mut session = Session::with_config( + scope.clone(), + Arc::new(Index::new(Arc::new(crate::storage::MemoryStore::default()))), + config, + ) + .unwrap(); + // No summary request is valid: retry contains the hit, helper is paired, + // and same.js is unchanged. An erroneous request fails on the closed endpoint. + let hydrated = session.hydrate(hits).unwrap(); + for result in hydrated { + for display in [Display::Lhs, Display::Rhs] { + let mut selected = result.clone(); + selected.display = display; + let results = session.postprocess(vec![result.clone(), selected]).unwrap(); + assert_eq!( + serde_json::to_value(&results[0].comparison).unwrap(), + serde_json::to_value(&results[1].comparison).unwrap() + ); + } + } + } + + #[test] + fn separate_comparisons_reuse_the_same_index_and_clean_computed_trees() { + let (_temporary, scope, hits) = fixture(); + let memory = Arc::new(crate::storage::MemoryStore::default()); + let mut warm = Session::new(scope.clone(), Arc::new(Index::new(memory.clone()))).unwrap(); + warm.hydrate(hits.clone()).unwrap(); + + // From here on, any cache miss fails instead of recomputing a diff. + let reader = Arc::new(ReadOnlyStore { + store: memory, + hits: AtomicUsize::new(0), + }); + let index = Arc::new(Index::new(reader.clone())); + let mut first = Session::new(scope.clone(), index.clone()).unwrap(); + let first_results = first.hydrate(hits).unwrap(); + first.postprocess(first_results).unwrap(); + drop(first); + + let reversed = Scope { + repo: scope.repo.clone(), + base_worktree: scope.head_worktree.clone(), + head_worktree: scope.base_worktree.clone(), + }; + let mut second = Session::new(reversed, index).unwrap(); + let results = second + .hydrate(vec![Hit { + file: scope.base_worktree.path.join("same.js"), + lines: vec![HitLine { + line: 1, + text: "function same() {".into(), + }], + }]) + .unwrap(); + second.postprocess(results).unwrap(); + assert_eq!(reader.hits.load(Ordering::SeqCst), 6); + } + + #[test] + fn config_selects_storage_and_reopened_stores_do_not_recompute() { + let (temporary, scope, hits) = fixture(); + let repository = &scope.repo; + let mut expected = None; + for backend in ["memory", "file", "sqlite"] { + let config_path = temporary.path().join("config.toml"); + std::fs::write( + &config_path, + format!("[storage]\nbackend = '{backend}'\npath = '.cache/diffr'\n"), + ) + .unwrap(); + let config = Config::load(Some(&config_path)).unwrap(); + let index = index_for_config(repository, &config).unwrap(); + let mut changed_analysis = config.clone(); + changed_analysis.plugins = Config::from_toml("[plugins]\norder = []").unwrap().plugins; + assert!(Arc::ptr_eq( + &index, + &index_for_config(repository, &changed_analysis).unwrap() + )); + let mut session = Session::with_config(scope.clone(), index, config.clone()).unwrap(); + let hydrated = session.hydrate(hits.clone()).unwrap(); + let result = serde_json::to_value(session.postprocess(hydrated).unwrap()).unwrap(); + if let Some(expected) = &expected { + assert_eq!(&result, expected); + } else { + expected = Some(result.clone()); + } + if backend == "memory" { + continue; + } + let reader = Arc::new(ReadOnlyStore { + store: config.storage.open(repository).unwrap(), + hits: AtomicUsize::new(0), + }); + let mut reopened = + Session::with_config(scope.clone(), Arc::new(Index::new(reader.clone())), config) + .unwrap(); + let hydrated = reopened.hydrate(hits.clone()).unwrap(); + assert_eq!( + serde_json::to_value(reopened.postprocess(hydrated).unwrap()).unwrap(), + result + ); + assert!(reader.hits.load(Ordering::SeqCst) >= 4); + // Different hit locations reuse clean trees rather than the old query's spans. + let alternate = vec![Hit { + file: scope.head_worktree.path.join("same.js"), + lines: vec![HitLine { + line: 1, + text: "function same() {".into(), + }], + }]; + let mut fresh = Session::new( + scope.clone(), + Arc::new(Index::new(Arc::new(crate::storage::MemoryStore::default()))), + ) + .unwrap(); + assert_eq!( + serde_json::to_value(reopened.hydrate(alternate.clone()).unwrap()).unwrap(), + serde_json::to_value(fresh.hydrate(alternate).unwrap()).unwrap() + ); + // Analysis changes must miss, even when the blob identities are unchanged. + let other_config = Config::from_toml("[plugins]\norder = []").unwrap(); + let mut other = + Session::with_config(scope.clone(), Arc::new(Index::new(reader)), other_config) + .unwrap(); + assert!(other + .hydrate(hits.clone()) + .unwrap_err() + .to_string() + .contains("unexpected recomputation")); + } + assert!(repository.join(".cache/diffr/diffs.sqlite").exists()); + assert!(!temporary.path().join(".cache").exists()); + } +}