From 6bbb45b97f12ae1628ddc82d6047beb3efa46315 Mon Sep 17 00:00:00 2001 From: Robert DeLanghe <1240090+bdelanghe@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:03:07 -0400 Subject: [PATCH 1/3] chore(deps): upgrade tree-sitter stack to 0.26 / rust 0.24 / html 0.23 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tree-sitter 0.22 → 0.26.10 tree-sitter-rust 0.21 → 0.24.2 tree-sitter-html 0.20 → 0.23.2 Both grammar call sites updated: language() fn → LANGUAGE.into(), which is the canonical API since the 0.23 grammar series. All 97 unit tests and 5 cucumber tests pass unchanged. Co-Authored-By: Claude Sonnet 4.6 --- Cargo.lock | 55 ++++++++++++++++++++++++++++++++++++++++++-------- Cargo.toml | 6 +++--- src/html.rs | 2 +- src/printer.rs | 2 +- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89240f4..9989f5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -278,6 +278,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -453,6 +459,12 @@ dependencies = [ "walkdir", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.4.1" @@ -487,6 +499,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "inflections" version = "1.1.1" @@ -803,6 +825,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -839,6 +862,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "strsim" version = "0.11.1" @@ -966,32 +995,42 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.22.6" +version = "0.26.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df7cc499ceadd4dcdf7ec6d4cbc34ece92c3fa07821e287aedecd4416c516dca" +checksum = "3c343ed63e3f5c64d1acdecb5d2c13d4e169cb5fde0052106ebaa6c6f27f9e55" dependencies = [ "cc", "regex", + "regex-syntax 0.8.11", + "serde_json", + "streaming-iterator", + "tree-sitter-language", ] [[package]] name = "tree-sitter-html" -version = "0.20.4" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8766b5ad3721517f8259e6394aefda9c686aebf7a8c74ab8624f2c3b46902fd5" +checksum = "261b708e5d92061ede329babaaa427b819329a9d427a1d710abb0f67bbef63ee" dependencies = [ "cc", - "tree-sitter", + "tree-sitter-language", ] +[[package]] +name = "tree-sitter-language" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" + [[package]] name = "tree-sitter-rust" -version = "0.21.2" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "277690f420bf90741dea984f3da038ace46c4fe6047cba57a66822226cde1c93" +checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" dependencies = [ "cc", - "tree-sitter", + "tree-sitter-language", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4bd1bad..dc08fb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,9 +13,9 @@ repository = "https://github.com/bounded-systems/git-ast" # its `Map` is a `BTreeMap` (keys sorted) with deterministic scalar formatting — # exactly the value-level normalization a canonical form needs. [dependencies] -tree-sitter = "0.22" -tree-sitter-rust = "0.21" -tree-sitter-html = "0.20" +tree-sitter = "0.26" +tree-sitter-rust = "0.24" +tree-sitter-html = "0.23" serde_json = "1" # The Gherkin suite (tests/features/*.feature) executes the README's claims diff --git a/src/html.rs b/src/html.rs index 4daf1ec..af36c4e 100644 --- a/src/html.rs +++ b/src/html.rs @@ -25,7 +25,7 @@ use crate::Error; fn parse(source: &[u8]) -> Result { let mut parser = Parser::new(); parser - .set_language(&tree_sitter_html::language()) + .set_language(&tree_sitter_html::LANGUAGE.into()) .map_err(|e| Error::Parsing(format!("loading HTML grammar: {e}")))?; parser .parse(source, None) diff --git a/src/printer.rs b/src/printer.rs index f2d355f..1008eb3 100644 --- a/src/printer.rs +++ b/src/printer.rs @@ -66,7 +66,7 @@ pub fn canonicalize(source: &[u8]) -> Result, Error> { fn parse(source: &[u8]) -> Result { let mut parser = Parser::new(); parser - .set_language(&tree_sitter_rust::language()) + .set_language(&tree_sitter_rust::LANGUAGE.into()) .map_err(|e| Error::Parsing(format!("loading Rust grammar: {e}")))?; let tree = parser .parse(source, None) From f85f36e4c9d7e1db7391f6778e92be6eae83aa53 Mon Sep 17 00:00:00 2001 From: Robert DeLanghe <1240090+bdelanghe@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:08:47 -0400 Subject: [PATCH 2/3] style: rustfmt html.rs (CI fmt check) --- src/html.rs | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/html.rs b/src/html.rs index af36c4e..65796fb 100644 --- a/src/html.rs +++ b/src/html.rs @@ -175,7 +175,8 @@ fn extract_attr_value(node: Node, src: &[u8]) -> String { // use `node` again in the fallback path. let found: Option = { let mut c = node.walk(); - let x = node.named_children(&mut c) + let x = node + .named_children(&mut c) .find(|n| n.kind() == "attribute_value") .and_then(|n| n.utf8_text(src).ok()) .map(str::to_string); @@ -425,8 +426,7 @@ fn element_to_def(node: Node, src: &[u8]) -> Option { let mut cursor = node.walk(); let children: Vec = node.children(&mut cursor).collect(); drop(cursor); - let (tag_node, has_body) = if let Some(&st) = - children.iter().find(|n| n.kind() == "start_tag") + let (tag_node, has_body) = if let Some(&st) = children.iter().find(|n| n.kind() == "start_tag") { (st, true) } else if let Some(&sc) = children.iter().find(|n| n.kind() == "self_closing_tag") { @@ -443,7 +443,11 @@ fn element_to_def(node: Node, src: &[u8]) -> Option { .get("role") .and_then(|r| aria_kind(r)) .or_else(|| implicit_role(&tag_name))?; - let text = if has_body { collect_text(node, src) } else { String::new() }; + let text = if has_body { + collect_text(node, src) + } else { + String::new() + }; let name = accessible_name(&attrs, &text).unwrap_or_default(); Some(make_def(kind, &name, &tag_name, &attrs)) } @@ -488,12 +492,7 @@ fn self_closing_to_def(node: Node, src: &[u8]) -> Option { /// so two elements that differ only in label share the same shape. const NAME_ATTRS: &[&str] = &["aria-label", "title", "alt"]; -fn make_def( - kind: &'static str, - name: &str, - tag: &str, - attrs: &BTreeMap, -) -> Def { +fn make_def(kind: &'static str, name: &str, tag: &str, attrs: &BTreeMap) -> Def { let attr_str: String = attrs .iter() .map(|(k, v)| format!("{k}={v}")) @@ -571,7 +570,10 @@ mod tests { #[test] fn values_double_quoted() { let out = canon(r#"link"#); - assert!(out.contains(r#"href="/page""#), "single quotes not converted: {out}"); + assert!( + out.contains(r#"href="/page""#), + "single quotes not converted: {out}" + ); } #[test] @@ -599,7 +601,10 @@ mod tests { fn script_and_style_preserved_verbatim() { let src = ""; let out = canon(src); - assert!(out.contains("var x={b:1,a:2};"), "script was modified: {out}"); + assert!( + out.contains("var x={b:1,a:2};"), + "script was modified: {out}" + ); } // ── inspect ─────────────────────────────────────────────────────────────── @@ -704,7 +709,11 @@ mod tests { fn subtree_hashes_nonempty_for_semantic_elements() { let ds = defs(r#""#); for d in &ds { - assert!(!d.subtree_hashes.is_empty(), "no subtree hashes for {}", d.kind); + assert!( + !d.subtree_hashes.is_empty(), + "no subtree hashes for {}", + d.kind + ); } } } From 15d5784ed744c7afb7a0142383178a0f0ae1d056 Mon Sep 17 00:00:00 2001 From: Robert DeLanghe <1240090+bdelanghe@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:06:41 -0400 Subject: [PATCH 3/3] =?UTF-8?q?feat(clones):=20add=20git-ast=20clones=20?= =?UTF-8?q?=E2=80=94=20structural=20clone=20detection=20via=20identity=20i?= =?UTF-8?q?ndex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First concrete step of prx-nf1f (identity-index). The subtree_hashes and shape_hash produced by inspect are computed per-file and then discarded. This commit accumulates them across a file set and surfaces equivalence classes — definitions with identical bodies under the canonical form, regardless of declared name. Design: - src/clones.rs: find_clones(&[(&str, &[Def])]) → Vec, keyed by shape_hash (BTreeMap for deterministic output). Groups with >1 occurrence are structural clones; single occurrences are filtered out. - Exact by construction: equal shape_hash ⟹ same canonical body. No threshold, no heuristics — that layer (MinHash+LSH over subtree_hashes) is explicitly deferred to the research note. - Routes by file extension (html/htm → html::inspect, else printer::inspect) matching the existing inspect/match/blame dispatch. CLI: git-ast clones FILE... Output: shape_hash N×kind\n file name\n per occurrence, one block per group. 11 new unit tests (108 total), all passing. Smoke-tested against the repo's own source (no false clones) and against two files with a known shared body (correctly groups 3 occurrences under one shape hash). Co-Authored-By: Claude Sonnet 4.6 --- src/clones.rs | 215 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 50 +++++++++++- 3 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 src/clones.rs diff --git a/src/clones.rs b/src/clones.rs new file mode 100644 index 0000000..e532648 --- /dev/null +++ b/src/clones.rs @@ -0,0 +1,215 @@ +//! Structural clone detection via the identity index. +//! +//! The `subtree_hashes` and `shape_hash` produced by [`crate::printer::inspect`] +//! and [`crate::html::inspect`] are computed per-file and then discarded. This +//! module accumulates them across a set of files and surfaces **equivalence +//! classes**: groups of definitions that are structurally identical (same body, +//! possibly different names) — Type-1 clones by construction, zero heuristics. +//! +//! The identity-index research note (`docs/research/identity-index.md`) describes +//! why this falls out for free: equal `shape_hash` ⟹ same trie node in the +//! Merkle DAG. This module is the first concrete step toward that index — an +//! in-memory occurrence map over a provided file set, exposing the equivalence +//! class query that the index would answer in O(1) against a persisted store. + +use std::collections::BTreeMap; + +use crate::printer::Def; + +/// A group of structurally identical definitions found across the scanned files. +/// +/// All members share the same `shape_hash`, meaning their bodies are identical +/// under git-ast's canonical form — they differ at most by their declared names. +/// This is an exact equivalence class by construction (hash collision ⟹ same +/// canonical body), not a heuristic similarity score. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CloneGroup { + /// The shared shape hash that identifies this equivalence class. + pub shape_hash: String, + /// The ARIA role / Rust kind shared by all members (`fn`, `navigation`, …). + pub kind: &'static str, + /// All occurrences, in file-then-definition order. + pub occurrences: Vec, +} + +/// One occurrence of a cloned definition. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Occurrence { + /// The file path the definition was found in. + pub file: String, + /// The declared name (accessible name for HTML, identifier for Rust). + pub name: String, +} + +/// Build clone groups from an already-inspected set of files. +/// +/// `file_defs` is a slice of `(path, defs)` pairs, typically produced by +/// calling `inspect` on each file. Returns only groups with **two or more** +/// occurrences, sorted deterministically by `shape_hash`. +/// +/// # Note on exactness +/// +/// The grouping is **exact**: equal `shape_hash` means the canonical forms are +/// identical modulo the declared name. No threshold, no approximation. The +/// fuzzy / near-clone layer (MinHash + LSH over `subtree_hashes`) is a +/// separate concern described in the research note. +pub fn find_clones<'a>(file_defs: &[(&'a str, &'a [Def])]) -> Vec { + // shape_hash → (kind, Vec) + // BTreeMap gives deterministic iteration order in the output. + let mut index: BTreeMap)> = BTreeMap::new(); + + for &(file, defs) in file_defs { + for def in defs { + let entry = index + .entry(def.shape_hash.clone()) + .or_insert_with(|| (def.kind, Vec::new())); + entry.1.push(Occurrence { + file: file.to_string(), + name: def.name.clone(), + }); + } + } + + index + .into_iter() + .filter(|(_, (_, occs))| occs.len() > 1) + .map(|(shape_hash, (kind, occurrences))| CloneGroup { + shape_hash, + kind, + occurrences, + }) + .collect() +} + +// ─── Tests ─────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + fn def(kind: &'static str, name: &str, content_hash: &str, shape_hash: &str) -> Def { + Def { + kind, + name: name.to_string(), + content_hash: content_hash.to_string(), + shape_hash: shape_hash.to_string(), + subtree_hashes: vec![], + } + } + + #[test] + fn no_files_gives_no_groups() { + assert!(find_clones(&[]).is_empty()); + } + + #[test] + fn single_file_no_duplicates_gives_no_groups() { + let defs = vec![ + def("fn", "foo", "aaa", "bbb"), + def("fn", "bar", "ccc", "ddd"), + ]; + let groups = find_clones(&[("a.rs", &defs)]); + assert!(groups.is_empty()); + } + + #[test] + fn same_shape_in_one_file_is_a_clone() { + let defs = vec![ + def("fn", "foo", "aaa", "shared"), + def("fn", "bar", "bbb", "shared"), + ]; + let groups = find_clones(&[("a.rs", &defs)]); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].shape_hash, "shared"); + assert_eq!(groups[0].occurrences.len(), 2); + assert_eq!(groups[0].occurrences[0].name, "foo"); + assert_eq!(groups[0].occurrences[1].name, "bar"); + } + + #[test] + fn same_shape_across_two_files_is_a_clone() { + let a = vec![def("fn", "foo", "aaa", "shared")]; + let b = vec![def("fn", "bar", "bbb", "shared")]; + let groups = find_clones(&[("a.rs", &a), ("b.rs", &b)]); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].occurrences[0].file, "a.rs"); + assert_eq!(groups[0].occurrences[1].file, "b.rs"); + } + + #[test] + fn content_hash_exact_clone_also_groups_by_shape() { + // Two fns with identical name AND body → same content_hash AND shape_hash. + let a = vec![def("fn", "foo", "same", "same")]; + let b = vec![def("fn", "foo", "same", "same")]; + let groups = find_clones(&[("a.rs", &a), ("b.rs", &b)]); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].occurrences.len(), 2); + } + + #[test] + fn three_clones_form_one_group() { + let a = vec![def("fn", "foo", "a1", "s1")]; + let b = vec![def("fn", "bar", "b1", "s1")]; + let c = vec![def("fn", "baz", "c1", "s1")]; + let groups = find_clones(&[("a.rs", &a), ("b.rs", &b), ("c.rs", &c)]); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].occurrences.len(), 3); + } + + #[test] + fn different_shapes_produce_separate_groups() { + let defs = vec![ + def("fn", "foo", "a", "s1"), + def("fn", "bar", "b", "s1"), + def("fn", "baz", "c", "s2"), + def("fn", "qux", "d", "s2"), + ]; + let groups = find_clones(&[("x.rs", &defs)]); + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].shape_hash, "s1"); + assert_eq!(groups[1].shape_hash, "s2"); + } + + #[test] + fn unique_definitions_are_not_reported() { + let a = vec![def("fn", "foo", "unique1", "unique1")]; + let b = vec![def("fn", "bar", "unique2", "unique2")]; + let groups = find_clones(&[("a.rs", &a), ("b.rs", &b)]); + assert!(groups.is_empty()); + } + + #[test] + fn kind_is_preserved_from_first_occurrence() { + let a = vec![def("navigation", "Main nav", "h1", "s1")]; + let b = vec![def("navigation", "Site nav", "h2", "s1")]; + let groups = find_clones(&[("a.html", &a), ("b.html", &b)]); + assert_eq!(groups[0].kind, "navigation"); + } + + #[test] + fn output_is_sorted_by_shape_hash() { + let defs = vec![ + def("fn", "a", "x", "zzz"), + def("fn", "b", "y", "zzz"), + def("fn", "c", "p", "aaa"), + def("fn", "d", "q", "aaa"), + ]; + let groups = find_clones(&[("f.rs", &defs)]); + assert_eq!(groups[0].shape_hash, "aaa"); + assert_eq!(groups[1].shape_hash, "zzz"); + } + + #[test] + fn round_trip_through_real_rust_source() { + use crate::printer; + let src_a = b"fn helper() -> u32 { 42 }"; + let src_b = b"fn aux() -> u32 { 42 }"; + let defs_a = printer::inspect(src_a).unwrap(); + let defs_b = printer::inspect(src_b).unwrap(); + // Same body, different names → identical shape_hash. + assert_eq!(defs_a[0].shape_hash, defs_b[0].shape_hash); + let groups = find_clones(&[("a.rs", &defs_a), ("b.rs", &defs_b)]); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].kind, "fn"); + } +} diff --git a/src/lib.rs b/src/lib.rs index 8cf4020..0b26fa2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,7 @@ //! Configuration is read via [`config`]. pub mod blame; +pub mod clones; pub mod config; pub mod diff; pub mod drivers; diff --git a/src/main.rs b/src/main.rs index 5e7781b..6f11b7b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,7 +7,7 @@ use std::io::Read; use std::process::ExitCode; -use git_ast::{blame, drivers, filters, html, identity, printer, setup, Error}; +use git_ast::{blame, clones, drivers, filters, html, identity, printer, setup, Error}; fn main() -> ExitCode { let args: Vec = std::env::args().skip(1).collect(); @@ -21,6 +21,7 @@ fn main() -> ExitCode { "inspect" => run_inspect(rest), "match" => run_match(rest), "blame" => run_blame(rest), + "clones" => run_clones(rest), "filter-process" => filters::run_long_running_filter().map(|()| 0u8), "diff-driver" => drivers::run_diff_driver(rest).map(|()| 0u8), "merge-driver" => drivers::run_merge_driver(rest).map(|()| 0u8), @@ -116,6 +117,52 @@ fn run_match(args: &[String]) -> Result { Ok(0) } +/// The `clones` read verb: detect structurally identical definitions across files. +/// +/// Groups top-level definitions by `shape_hash` (body-only identity, name-blind). +/// Any group with two or more members is a structural clone: same canonical body, +/// possibly different declared names. This is the first concrete step of the +/// identity index described in `docs/research/identity-index.md` — an in-memory +/// occurrence map, exact by construction (no similarity threshold). +fn run_clones(args: &[String]) -> Result { + if args.is_empty() { + return Err(Error::Config( + "clones expects: git-ast clones ...".to_string(), + )); + } + let mut file_defs: Vec<(String, Vec)> = Vec::new(); + for path in args { + let source = std::fs::read(path)?; + let ext = std::path::Path::new(path) + .extension() + .and_then(|e| e.to_str()); + let defs = match ext { + Some("html") | Some("htm") => html::inspect(&source)?, + _ => printer::inspect(&source)?, + }; + if !defs.is_empty() { + file_defs.push((path.clone(), defs)); + } + } + let pairs: Vec<(&str, &[printer::Def])> = file_defs + .iter() + .map(|(p, d)| (p.as_str(), d.as_slice())) + .collect(); + let groups = clones::find_clones(&pairs); + for group in &groups { + println!( + "{} {}×{}", + group.shape_hash, + group.occurrences.len(), + group.kind + ); + for occ in &group.occurrences { + println!(" {} {}", occ.file, occ.name); + } + } + Ok(0) +} + /// The `blame` verb: refactor-aware, per-definition blame /// (`git-ast blame `). For each top-level item, prints the commit that last /// changed it — following it through renames. @@ -153,6 +200,7 @@ fn print_help() { inspect [FILE] List top-level defs with a formatting-invariant hash\n \ match OLD NEW Correspond defs across two versions (rename/move/edit)\n \ blame FILE Per-def blame, following items through renames\n \ + clones FILE... Detect structurally identical defs across files\n \ filter-process Clean/smudge long-running filter (Rust + JSON)\n \ diff-driver Structural diff (JSON); text diff otherwise\n \ merge-driver Structural 3-way merge (JSON)\n \