diff --git a/.gitignore b/.gitignore index b46291876..9cf547af8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ sample_files/compare.result notes.md perf_baseline_*.txt +__pycache__/ +.env diff --git a/.typos.toml b/.typos.toml index 5bf9439f4..011989477 100644 --- a/.typos.toml +++ b/.typos.toml @@ -5,6 +5,8 @@ extend-exclude = [ "homepage/home_img", "demo_files/", "difft.1", + # Git index lines carry blob hashes that can spell words. + "examples/review/real/*/change.patch", ] [default.extend-words] diff --git a/Cargo.lock b/Cargo.lock index 9d280c07e..1cba9554e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,17 +82,40 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" -version = "2.5.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bstr" @@ -111,6 +134,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.2.54" @@ -123,6 +152,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.0" @@ -163,6 +198,32 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -215,6 +276,7 @@ dependencies = [ "humansize", "ignore", "imara-diff", + "jsonrpsee", "lazy_static", "libc", "line-numbers", @@ -235,6 +297,7 @@ dependencies = [ "strum", "terminal_size", "tikv-jemallocator", + "tokio", "toml", "tree-sitter", "tree-sitter-ada", @@ -384,6 +447,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -405,6 +474,57 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "git2" version = "0.20.4" @@ -437,6 +557,25 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -469,6 +608,45 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "humansize" version = "2.1.3" @@ -484,6 +662,63 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "log", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -645,6 +880,50 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.106", +] + [[package]] name = "jobserver" version = "0.1.32" @@ -654,6 +933,89 @@ dependencies = [ "libc", ] +[[package]] +name = "jsonrpsee" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3f48dc3e6b8bd21e15436c1ddd0bc22a6a54e8ec46fedd6adf3425f396ec6a" +dependencies = [ + "jsonrpsee-core", + "jsonrpsee-http-client", + "jsonrpsee-proc-macros", + "jsonrpsee-types", + "tracing", +] + +[[package]] +name = "jsonrpsee-core" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "316c96719901f05d1137f19ba598b5fe9c9bc39f4335f67f6be8613921946480" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "jsonrpsee-types", + "pin-project", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "jsonrpsee-http-client" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790bedefcec85321e007ff3af84b4e417540d5c87b3c9779b9e247d1bcc3dab8" +dependencies = [ + "base64", + "http-body", + "hyper", + "hyper-rustls", + "hyper-util", + "jsonrpsee-core", + "jsonrpsee-types", + "rustls", + "rustls-platform-verifier", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tower", + "url", +] + +[[package]] +name = "jsonrpsee-proc-macros" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da3f8ab5ce1bb124b6d082e62dffe997578ceaf0aeb9f3174a214589dc00f07" +dependencies = [ + "heck", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "jsonrpsee-types" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc88ff4688e43cc3fa9883a8a95c6fa27aa2e76c96e610b737b6554d650d7fd5" +dependencies = [ + "http", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -726,6 +1088,17 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "nom" version = "8.0.0" @@ -750,6 +1123,18 @@ dependencies = [ "autocfg", ] +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "owo-colors" version = "4.2.3" @@ -773,6 +1158,32 @@ dependencies = [ "indexmap", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.34" @@ -838,6 +1249,15 @@ dependencies = [ "log", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.14+spec-1.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.101" @@ -911,6 +1331,20 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-hash" version = "2.0.0" @@ -930,6 +1364,80 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.23.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs 0.26.11", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.17" @@ -945,6 +1453,38 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.228" @@ -1004,12 +1544,28 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1049,6 +1605,12 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.106" @@ -1071,6 +1633,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.2" @@ -1107,6 +1675,46 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "tikv-jemalloc-sys" version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" @@ -1137,6 +1745,44 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.8.23" @@ -1145,8 +1791,8 @@ checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", - "toml_datetime", - "toml_edit", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", ] [[package]] @@ -1158,6 +1804,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -1167,8 +1822,86 @@ dependencies = [ "indexmap", "serde", "serde_spanned", - "toml_datetime", - "winnow", + "toml_datetime 0.6.11", + "winnow 0.7.15", +] + +[[package]] +name = "toml_edit" +version = "0.25.14+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2195eec204e2764644a4ea619704f9fbe5e0673038eded55ad9956f24fca0cc" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", ] [[package]] @@ -1772,6 +2505,12 @@ dependencies = [ "petgraph", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "ts-parser-perl" version = "1.2.1" @@ -1800,6 +2539,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -1855,6 +2600,39 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "webpki-root-certs" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" +dependencies = [ + "webpki-root-certs 1.0.9", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi-util" version = "0.1.9" @@ -1864,13 +2642,28 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1879,7 +2672,31 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -1888,28 +2705,46 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1922,24 +2757,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -1955,6 +2814,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "writeable" version = "0.6.4" @@ -2011,6 +2879,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 97176af68..c1605e89a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,8 @@ ignore = ">= 0.4, < 0.4.30" owo-colors = "4.2.3" imara-diff = { version = "0.2.0", default-features = false } rayon = "1.7.0" +jsonrpsee = { version = "0.26", default-features = false, features = ["http-client", "macros"] } +tokio = { version = "1", default-features = false, features = ["rt-multi-thread", "time"] } tree_magic_mini = "3.2.0" bumpalo = "3.18.0" diff --git a/README.md b/README.md index 9436d2af5..8781fdfe2 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ diffr main...HEAD -- src/ # merge-base comparison diffr main HEAD --format ndjson ``` -See the [CLI reference](docs/cli.md), [streaming API](docs/streaming.md), -and [fixture viewer](examples/review/viewer/README.md). +See the [CLI reference](docs/cli.md), [streaming API](docs/streaming.md) +including fold hooks, and [fixture viewer](examples/review/viewer/README.md). The executable is `diffr`; the Cargo package remains `difftastic`. The upstream installation commands below install upstream Difftastic, not this fork. diff --git a/docs/cli.md b/docs/cli.md index 2853828e1..cfe066595 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -28,7 +28,9 @@ for changes; ordinary output exits 0. Errors exit 2. `--no-index` supports two files, implies change exit status, and does not yet support metadata options. Structural output uses the existing terminal renderer. `--format json` emits -one domain object per line. `--format ndjson` emits a buffered event stream; `--format snapshot` is the fixture text adapter. +one domain object per line. `--format ndjson` emits a buffered event stream, +diffing `--jobs N` files at once (default 16) and emitting each as it finishes; +`--format snapshot` is the fixture text adapter. `-U N` selects ordinary context padding. Matching limits, `--ignore-comments`, color, width and inline/split display remain configurable; see `--help`. diff --git a/docs/streaming.md b/docs/streaming.md index 13560a51d..2a0062344 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -43,8 +43,8 @@ pipe reads can split records or contain several. | Event | Fields | Consumer action | | --- | --- | --- | -| `start` | `version: 1`, `before`, `after`, `total` | Initialize progress. | -| `file` | `file`, `diff` | Render a result. | +| `start` | `version: 1`, `before`, `after`, `total`, `files` | Lay out every file up front. | +| `file` | `file`, `diff`, optional `hook_error` | Render a result. | | `file_error` | `file`, `message` | Report failure and keep reading. | | `complete` | `succeeded`, `failed` | Mark complete, including partial failures. | @@ -58,6 +58,10 @@ The file descriptor contains nullable `old_path`, `new_path`, `class`, and `stat domain JSON: complete sources, token correspondence, folds and context hunks. There is no display layout in the response. +`files` lists every selected file descriptor in priority order. File results +arrive in completion order, not manifest order, since files are diffed +concurrently (`--jobs`, default 16). Match results to the manifest by identity. + At completion, `succeeded + failed == total`; every selected file has one result or file error. EOF without `complete` means interrupted/incomplete output. Setup failures write to stderr and exit 2 before producing any records. @@ -72,15 +76,73 @@ fail discovery. ## Computation and output Discovery and rename detection finish before `start`; syntax matching is lazy. -A producer thread consumes the file iterator. The calling thread serializes, -writes and flushes each event. A bounded queue holds one ready event, allowing -computation to overlap slow writes without collecting the entire comparison. -When the queue is full, the producer waits. This bounds the number of in-flight -files, not their individual size. - -Closing stdout stops production when its next send fails; an already running file -may finish. Terminate the process to cancel immediately. The CLI also retains its -normal SIGPIPE behavior on Unix. +A pool of `--jobs` workers pulls files from the iterator: each worker reads the +next file's sources under a lock, then diffs them while other workers pull +further files. The calling thread serializes, writes and flushes each event. +A bounded queue holds one ready event, so computation overlaps slow writes +without collecting the entire comparison. In-flight files are bounded by the +pool size, not their individual size. `--jobs 1` restores priority order. + +Closing stdout stops production once the files in flight finish. Terminate the +process to cancel immediately. The CLI also retains its normal SIGPIPE behavior +on Unix. + +## Fold hooks + +A trusted hook can replace fold placeholders with richer text, such as +pseudocode, before each `file` event is emitted. A hook is a JSON-RPC 2.0 +server over HTTP that diffr starts once per invocation and calls on loopback: + +```toml +[folds.hook] +command = ["uv", "run", "--script", "examples/hooks/summarize.py"] +tags = ["body"] # optional; any listed tag qualifies. Omit to send every fold. +min_lines = 12 # optional; default 0 +timeout_ms = 5000 # optional; per call +startup_timeout_ms = 30000 # optional; time allowed to start listening +``` + +The command starts with the caller's environment plus `DIFFR_HOOK_PORT`, the +loopback port it must listen on, and `DIFFR_WORKSPACE`, the diffed repository's +root. It runs in the directory containing the config file, so relative paths in +`command` resolve against the config wherever it lives, including one given by +`--config` outside the repository. Its stdout is discarded because diffr's own +stdout carries the event stream; log to stderr. diffr polls the port until the +hook accepts connections, exits 2 before `start` if the hook exits or misses +`startup_timeout_ms`, and kills the hook when the comparison ends. + +Only novel folds on the after side qualify: bodies that exist in the after +source with no counterpart in the before source. Files with no qualifying fold +never reach the hook. Streaming is the only output mode that runs hooks; the +terminal frontend streams, so it does too. + +One call per file, method `summarize`, params by name. The worker diffing that +file blocks on the reply; other workers keep calling, so a hook must serve +requests concurrently rather than one at a time. + +```jsonc +// diffr -> hook POST / with a JSON-RPC 2.0 request +{"jsonrpc": "2.0", "id": 7, "method": "summarize", "params": { + "path": "src/auth.py", "language": "Python", "src": "", + "folds": [{"id": 0, "range": {"start": {"line": 40, "byte_column": 0}, "end": {"line": 88, "byte_column": 1}}, + "tags": ["body"], "placeholder": "Body"}]}} +// hook -> diffr +{"jsonrpc": "2.0", "id": 7, "result": {"0": "def refresh_token(session):\n ..."}} +{"jsonrpc": "2.0", "id": 8, "error": {"code": -32000, "message": "rate limited"}} +``` + +`language` is null for plain text. A fold `id` indexes `rhs_folds` in that file's +`diff`; the matching fold gains a non-null `summary` while `placeholder` is +unchanged. Folds missing from the result keep a null `summary`. An error +object, a timeout, an unknown fold id, or an invalid response leaves every +summary in that file null and adds `hook_error` to its `file` event. + +`examples/hooks/summarize.py` is a reference hook: an aiohttp server that hands +each request to jsonrpcserver and asks Gemini 3.8 Flash, with thinking disabled, +for Python-style pseudocode. It needs `GOOGLE_API_KEY` and answers up to 16 +files at once on one asyncio loop with a shared httpx client. `uv run --script` +installs its dependencies on first use. `tests/hooks/rpc_server.py` is a +dependency-free hook used by the tests. ## Fixture viewer diff --git a/examples/hooks/summarize.py b/examples/hooks/summarize.py new file mode 100755 index 000000000..6821423e7 --- /dev/null +++ b/examples/hooks/summarize.py @@ -0,0 +1,121 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = ["aiohttp>=3.9", "httpx>=0.27", "jsonrpcserver>=5"] +# /// +"""Reference diffr fold hook: rewrite large novel folds as Python-style pseudocode. + +diffr starts this server once per invocation with the port in DIFFR_HOOK_PORT and +calls the JSON-RPC 2.0 method `summarize` once per file, concurrently. Each call +awaits one Gemini request on a shared httpx client. Log to stderr; stdout is +discarded by diffr. + +Environment: + DIFFR_HOOK_PORT set by diffr + GOOGLE_API_KEY required + DIFFR_SUMMARY_MODEL default gemini-3.8-flash + DIFFR_SUMMARY_WORKERS concurrent model requests, default 16 +""" + +import asyncio +import json +import os + +import httpx +from aiohttp import web +from jsonrpcserver import Error, Result, Success, async_dispatch, method + +API_KEY = os.environ["GOOGLE_API_KEY"] +PORT = int(os.environ["DIFFR_HOOK_PORT"]) +MODEL = os.environ.get("DIFFR_SUMMARY_MODEL", "gemini-3.8-flash") +WORKERS = int(os.environ.get("DIFFR_SUMMARY_WORKERS", "16")) +URL = f"https://generativelanguage.googleapis.com/v1beta/models/{MODEL}:generateContent" + +SYSTEM = ( + "You rewrite regions of a source file as terse Python-style pseudocode for a diff " + "viewer that shows the pseudocode in place of the collapsed region. The user supplies " + "one numbered source file and a list of folds, each with an id and 1-based line range. " + "For each fold, write pseudocode covering only that fold's lines: keep the control flow " + "and the names that matter, drop types, error plumbing and boilerplate. Aim for about one " + "pseudocode line per five source lines, between one and eight lines per fold. Reply with " + "one {id, pseudocode} object per fold." +) +SCHEMA = { + "type": "ARRAY", + "items": { + "type": "OBJECT", + "properties": {"id": {"type": "INTEGER"}, "pseudocode": {"type": "STRING"}}, + "required": ["id", "pseudocode"], + }, +} + +client = httpx.AsyncClient(headers={"x-goog-api-key": API_KEY}, timeout=60) +limit = asyncio.Semaphore(WORKERS) + + +def prompt(path, language, src, folds): + numbered = "\n".join( + f"{n:5d} | {line}" for n, line in enumerate(src.splitlines(), 1) + ) + ranges = "\n".join( + f"- fold {fold['id']}: lines {fold['range']['start']['line'] + 1}-" + f"{fold['range']['end']['line'] + 1}" + for fold in folds + ) + return f"File {path} ({language or 'unknown language'}):\n\n{numbered}\n\nFolds:\n{ranges}" + + +async def complete(path, language, src, folds): + body = { + "systemInstruction": {"parts": [{"text": SYSTEM}]}, + "contents": [ + {"role": "user", "parts": [{"text": prompt(path, language, src, folds)}]} + ], + "generationConfig": { + "temperature": 0, + "maxOutputTokens": 160 * len(folds) + 100, + "thinkingConfig": {"thinkingBudget": 0}, + "responseMimeType": "application/json", + "responseSchema": SCHEMA, + }, + } + async with limit: + response = await client.post(URL, json=body) + response.raise_for_status() + content = response.json()["candidates"][0]["content"]["parts"][-1]["text"] + expected = {fold["id"] for fold in folds} + texts = {} + for item in json.loads(content): + if item["id"] not in expected: + raise ValueError( + f"model answered for unknown fold {item['id']}: {content[:200]}" + ) + if item["pseudocode"].strip(): + texts[str(item["id"])] = item["pseudocode"].strip() + return texts + + +@method +async def summarize(path, language, src, folds) -> Result: + try: + return Success(await complete(path, language, src, folds)) + except httpx.HTTPStatusError as error: + return Error( + -32000, + f"{MODEL}: HTTP {error.response.status_code} {error.response.text[:200]}", + ) + except (httpx.HTTPError, ValueError, KeyError) as error: + return Error(-32000, f"{MODEL}: {error}") + + +async def handle(request: web.Request) -> web.Response: + return web.Response( + text=await async_dispatch(await request.text()), content_type="application/json" + ) + + +app = web.Application() +app.router.add_post("/", handle) + +if __name__ == "__main__": + web.run_app(app, host="127.0.0.1", port=PORT, print=None, access_log=None) diff --git a/examples/review/README.md b/examples/review/README.md index e3a39bbb7..b017e1a34 100644 --- a/examples/review/README.md +++ b/examples/review/README.md @@ -148,7 +148,7 @@ remain outside this v0. - Fold candidates are generated but ignored by this printer. Visible import lines stay expanded; no placeholders are emitted. - Left/right line numbers, grouped deletions/additions and explicit gaps. `~` marks matched syntax whose indentation differs; both exact source lines are retained in text snapshots. This - is a review snapshot, not an applyable Git patch. Token-level colors and split + is a review snapshot, not an applicable Git patch. Token-level colors and split view are not implemented. - Unsupported languages or Difftastic limit fallbacks retain a textual diff without syntax annotations. The context rules currently target the corpus's diff --git a/src/cli.rs b/src/cli.rs index 1d2d8e053..c67e44727 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,6 +1,7 @@ //! Git-style CLI input; rendering and NDJSON remain adapters over the same engine. use crate::config::Config; use crate::git::{Comparison, DiffSession, FileParams, Operand, Result}; +use crate::hook::Hook; use crate::options::{DiffOptions, DisplayMode, DisplayOptions}; use clap::{Arg, ArgAction, ArgGroup, ArgMatches, Command}; use git2::{DiffStatsFormat, Repository}; @@ -34,6 +35,14 @@ pub(crate) fn run() -> Result { .about("Structural diffs with Git-style comparison inputs") .arg(Arg::new("repo").long("repo").default_value(".")) .arg(Arg::new("config").long("config")) + .arg( + Arg::new("jobs") + .long("jobs") + .short('j') + .value_parser(clap::value_parser!(usize)) + .default_value("16") + .help("Concurrent file diffs for --format ndjson; results are emitted as each finishes"), + ) .arg(Arg::new("order").long("order").value_delimiter(',').action(ArgAction::Append).help("File class priority from diffr-classify attributes")) .arg(flag("cached").visible_alias("staged")) .arg(flag("merge-base")) @@ -157,12 +166,17 @@ pub(crate) fn run() -> Result { let params = Arc::new( Config::load(workspace, args.get_one::("config").map(Path::new))?.compile()?, ); + let hook = fold_hook(¶ms, workspace)?; let mut session = DiffSession::open(workspace, comparison, params, &files)?; session.context_lines = display.num_context_lines; session.diff_options = diff_options; let changed = session.remaining() > 0; if streaming { - let failed = crate::stream::write(session, &mut io::stdout().lock())?; + let jobs = *args.get_one::("jobs").unwrap(); + if jobs == 0 { + return Err("--jobs must be at least 1".into()); + } + let failed = crate::stream::write(session, jobs, hook, &mut io::stdout().lock())?; return Ok(if failed { 2 } else { @@ -447,10 +461,12 @@ fn no_index( ) }; if args.get_one::("format").map(String::as_str) == Some("ndjson") { + let hook = fold_hook(&config, Path::new(args.get_one::("repo").unwrap()))?; crate::stream::write_file( &paths[0].to_string_lossy(), &paths[1].to_string_lossy(), compute, + hook.as_deref(), &mut io::stdout().lock(), )?; Ok(i32::from(changed && args.get_flag("exit-code"))) @@ -460,6 +476,15 @@ fn no_index( } } +/// Streaming output summarizes large novel folds through the configured hook. +fn fold_hook(params: &crate::config::Params, workspace: &Path) -> Result>> { + params + .hook + .as_ref() + .map(|config| Hook::spawn(config, workspace).map(Arc::new)) + .transpose() +} + /// Explicit machine/text modes and redirected output must never enter the alternate screen. fn opens_tui(explicit_format: bool, metadata_or_quiet: bool, terminal: bool) -> bool { terminal && !explicit_format && !metadata_or_quiet diff --git a/src/config.rs b/src/config.rs index 05b4ce2bf..fe29ee81c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,7 +5,7 @@ use crate::parse::{guess_language::Language, tree_sitter_parser}; use query::AnnotationQuery; use serde::Deserialize; use std::collections::BTreeMap; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, OnceLock}; use strum::IntoEnumIterator; @@ -13,6 +13,42 @@ use strum::IntoEnumIterator; #[serde(default, deny_unknown_fields)] pub(crate) struct Config { pub(crate) languages: BTreeMap, + pub(crate) folds: FoldsConfig, +} + +#[derive(Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub(crate) struct FoldsConfig { + pub(crate) hook: Option, +} + +/// A trusted subprocess that supplies summaries for large novel folds. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct HookConfig { + /// Relative command paths resolve against the config file, wherever it lives. + #[serde(skip)] + pub(crate) dir: PathBuf, + pub(crate) command: Vec, + /// None sends every tagged fold; otherwise a fold needs one of these tags. + #[serde(default)] + pub(crate) tags: Option>, + #[serde(default)] + pub(crate) min_lines: usize, + /// Per-call limit once the hook is listening. + #[serde(default = "default_timeout_ms")] + pub(crate) timeout_ms: u64, + /// How long the hook may take to start listening on its port. + #[serde(default = "default_startup_timeout_ms")] + pub(crate) startup_timeout_ms: u64, +} + +fn default_timeout_ms() -> u64 { + 5000 +} + +fn default_startup_timeout_ms() -> u64 { + 30_000 } #[derive(Default, Deserialize)] @@ -34,6 +70,7 @@ impl std::error::Error for ConfigError {} pub(crate) struct Params { languages: DftHashMap>>, + pub(crate) hook: Option, } pub(crate) struct LanguageParams { @@ -65,7 +102,16 @@ impl Config { .map(Path::to_path_buf) .unwrap_or_else(|| workspace.join("diffr.toml")); match std::fs::read_to_string(&path) { - Ok(source) => Self::from_toml(&source), + Ok(source) => { + let mut config = Self::from_toml(&source)?; + if let Some(hook) = &mut config.folds.hook { + hook.dir = path + .parent() + .expect("config file has a parent") + .to_path_buf(); + } + Ok(config) + } Err(error) if explicit.is_none() && error.kind() == std::io::ErrorKind::NotFound => { Ok(Self::default()) } @@ -78,6 +124,14 @@ impl Config { } pub(crate) fn compile(self) -> Result { + if let Some(hook) = &self.folds.hook { + if hook.command.is_empty() { + return Err(ConfigError("folds.hook.command must not be empty".into())); + } + if hook.timeout_ms == 0 || hook.startup_timeout_ms == 0 { + return Err(ConfigError("folds.hook timeouts must be positive".into())); + } + } let defaults = Self::from_toml(include_str!("config/defaults.toml"))?; let mut resolved = defaults.languages; for (name, overrides) in self.languages { @@ -111,7 +165,10 @@ impl Config { })), ); } - Ok(Params { languages }) + Ok(Params { + languages, + hook: self.folds.hook, + }) } } @@ -231,6 +288,39 @@ mod tests { assert_eq!(result.rhs_folds[0].tags, ["embedded"]); } + #[test] + fn parses_fold_hook_settings() { + let params = Config::from_toml( + "[folds.hook]\ncommand = ['uv', 'run', 'summarize.py']\ntags = ['body']\nmin_lines = 30", + ) + .unwrap() + .compile() + .unwrap(); + let hook = params.hook.unwrap(); + assert_eq!(hook.command, ["uv", "run", "summarize.py"]); + assert_eq!(hook.tags.as_deref(), Some(&["body".to_owned()][..])); + assert_eq!( + (hook.min_lines, hook.timeout_ms, hook.startup_timeout_ms), + (30, 5000, 30_000) + ); + assert!(Config::from_toml("") + .unwrap() + .compile() + .unwrap() + .hook + .is_none()); + for input in [ + "[folds.hook]\ncommand = []", + "[folds.hook]\ncommand = ['x']\ntimeout_ms = 0", + ] { + assert!( + Config::from_toml(input).unwrap().compile().is_err(), + "{input}" + ); + } + assert!(Config::from_toml("[folds.hook]\ncommand = ['x']\nunknown = 1").is_err()); + } + #[test] fn rejects_unknown_settings_languages_and_invalid_queries() { assert!(Config::from_toml("typo = true").is_err()); diff --git a/src/git.rs b/src/git.rs index 24f0ed50d..2d4c9ec6c 100644 --- a/src/git.rs +++ b/src/git.rs @@ -329,28 +329,57 @@ impl DiffSession { } } -impl Iterator for DiffSession { - type Item = (FileChange, Result); - fn next(&mut self) -> Option { +/// Sources read on the session thread; diffing needs no repository access. +pub(crate) struct LoadedFile { + pub(crate) file: FileChange, + before: String, + after: String, + params: Arc, + context_lines: u32, + diff_options: crate::options::DiffOptions, +} + +impl LoadedFile { + pub(crate) fn diff(&self) -> DiffResult { + DiffResult::from_sources_with_options( + self.file.path(), + &self.before, + &self.after, + &self.params, + &crate::options::DisplayOptions { + num_context_lines: self.context_lines, + ..Default::default() + }, + &self.diff_options, + ) + } +} + +impl DiffSession { + /// Read the next file's sources without diffing them. + pub(crate) fn load(&mut self) -> Option<(FileChange, Result)> { let pending = self.files.next()?; let result = (|| { if matches!(pending.file.status, FileStatus::Conflicted) { return Err("unmerged index entry: resolve the conflict before requesting a structural diff".into()); } - let before = pending.before.read(&self.repo)?; - let after = pending.after.read(&self.repo)?; - Ok(DiffResult::from_sources_with_options( - pending.file.path(), - &before, - &after, - &self.params, - &crate::options::DisplayOptions { - num_context_lines: self.context_lines, - ..Default::default() - }, - &self.diff_options, - )) + Ok(LoadedFile { + before: pending.before.read(&self.repo)?, + after: pending.after.read(&self.repo)?, + file: pending.file.clone(), + params: Arc::clone(&self.params), + context_lines: self.context_lines, + diff_options: self.diff_options.clone(), + }) })(); Some((pending.file, result)) } } + +impl Iterator for DiffSession { + type Item = (FileChange, Result); + fn next(&mut self) -> Option { + let (file, loaded) = self.load()?; + Some((file, loaded.map(|loaded| loaded.diff()))) + } +} diff --git a/src/hook.rs b/src/hook.rs new file mode 100644 index 000000000..742caca34 --- /dev/null +++ b/src/hook.rs @@ -0,0 +1,305 @@ +//! Trusted fold-summary hook: a JSON-RPC 2.0 server that diffr starts once per +//! session and calls over loopback HTTP. +//! +//! The hook receives its port in `DIFFR_HOOK_PORT` and the diffed repository in +//! `DIFFR_WORKSPACE`. Each call carries one file and its large novel folds on +//! the after side. Workers block on their own call while the client multiplexes +//! every in-flight request on a small tokio runtime, so files still stream out +//! as each worker finishes. +use crate::config::HookConfig; +use crate::parse::folds::FoldMatch; +use crate::review::wire; +use crate::summary::{DiffResult, FileContent, FileFormat}; +use jsonrpsee::core::ClientError; +use jsonrpsee::http_client::{HttpClient, HttpClientBuilder}; +use jsonrpsee::proc_macros::rpc; +use serde::Serialize; +use serde_json::Value; +use std::collections::BTreeMap; +use std::net::{TcpListener, TcpStream}; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +#[derive(Serialize)] +pub(crate) struct RequestFold { + /// Index into the file's rhs folds. + id: usize, + range: Value, + tags: Vec, + placeholder: String, +} + +/// The interface every hook implements. Params are sent by name; the result +/// maps fold ids, as strings, to replacement text. +#[rpc(client)] +trait FoldHook { + #[method(name = "summarize", param_kind = map)] + async fn summarize( + &self, + path: String, + language: Option, + src: String, + folds: Vec, + ) -> jsonrpsee::core::RpcResult>; +} + +pub(crate) struct Hook { + config: HookConfig, + child: Mutex, + runtime: tokio::runtime::Runtime, + client: HttpClient, +} + +impl Hook { + pub(crate) fn spawn(config: &HookConfig, workspace: &Path) -> crate::git::Result { + let port = free_port()?; + let mut child = Command::new(&config.command[0]) + .args(&config.command[1..]) + .current_dir(&config.dir) + .env("DIFFR_HOOK_PORT", port.to_string()) + .env("DIFFR_WORKSPACE", workspace) + .stdin(Stdio::null()) + // Stdout belongs to diffr's own stream; hooks log to stderr. + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .map_err(|error| format!("could not start fold hook {:?}: {error}", config.command))?; + if let Err(error) = await_listening(&mut child, port, config.startup_timeout_ms) { + let _ = child.kill(); + let _ = child.wait(); + return Err(error.into()); + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_name("diffr-hook-client") + .enable_all() + .build()?; + let client = runtime.block_on(async { + HttpClientBuilder::default() + .request_timeout(Duration::from_millis(config.timeout_ms)) + .build(format!("http://127.0.0.1:{port}")) + })?; + Ok(Self { + config: config.clone(), + child: Mutex::new(child), + runtime, + client, + }) + } + + /// Fill in summaries for this file's qualifying folds, blocking on the hook. + /// Files without qualifying folds never reach the hook. + pub(crate) fn summarize(&self, diff: &mut DiffResult) -> Result<(), String> { + let selected: Vec = diff + .rhs_folds + .iter() + .enumerate() + .filter(|(_, fold)| self.qualifies(fold)) + .map(|(index, _)| index) + .collect(); + if selected.is_empty() { + return Ok(()); + } + let FileContent::Text(src) = &diff.rhs_src else { + return Ok(()); + }; + let language = match &diff.file_format { + FileFormat::SupportedLanguage(language) => { + Some(crate::parse::guess_language::language_name(*language).to_owned()) + } + _ => None, + }; + let folds = selected + .iter() + .map(|&index| { + let fold = &diff.rhs_folds[index]; + RequestFold { + id: index, + range: wire::range(&fold.range), + tags: fold.tags.clone(), + placeholder: fold.placeholder.clone(), + } + }) + .collect(); + let texts = self + .runtime + .block_on(FoldHookClient::summarize( + &self.client, + diff.display_path.clone(), + language, + src.clone(), + folds, + )) + .map_err(|error| match error { + ClientError::Call(error) => format!("fold hook reported: {}", error.message()), + ClientError::RequestTimeout => { + format!("fold hook timed out after {}ms", self.config.timeout_ms) + } + other => format!("fold hook: {other}"), + })?; + for (key, text) in texts { + let index: usize = key + .parse() + .ok() + .filter(|index| selected.contains(index)) + .ok_or_else(|| format!("fold hook answered for unknown fold {key:?}"))?; + diff.rhs_folds[index].summary = Some(text); + } + Ok(()) + } + + fn qualifies(&self, fold: &crate::parse::folds::Fold) -> bool { + if !matches!(fold.match_kind, FoldMatch::Novel) { + return false; + } + let lines = (fold.range.end.line.0 - fold.range.start.line.0 + 1) as usize; + if lines < self.config.min_lines { + return false; + } + match &self.config.tags { + Some(tags) => fold.tags.iter().any(|tag| tags.contains(tag)), + None => true, + } + } +} + +/// Reserve a loopback port for the hook. The listener is released before the +/// hook starts, which is the usual small race on a single machine. +fn free_port() -> std::io::Result { + Ok(TcpListener::bind(("127.0.0.1", 0))?.local_addr()?.port()) +} + +/// Poll until the hook accepts connections, or fail early if it exits. +fn await_listening(child: &mut Child, port: u16, startup_timeout_ms: u64) -> Result<(), String> { + let deadline = Instant::now() + Duration::from_millis(startup_timeout_ms); + let address = std::net::SocketAddr::from(([127, 0, 0, 1], port)); + loop { + if TcpStream::connect_timeout(&address, Duration::from_millis(100)).is_ok() { + return Ok(()); + } + if let Some(status) = child.try_wait().map_err(|error| error.to_string())? { + return Err(format!("fold hook exited during startup with {status}")); + } + if Instant::now() >= deadline { + return Err(format!( + "fold hook did not listen on port {port} within {startup_timeout_ms}ms" + )); + } + std::thread::sleep(Duration::from_millis(20)); + } +} + +impl Drop for Hook { + fn drop(&mut self) { + let mut child = self.child.lock().unwrap(); + let _ = child.kill(); + let _ = child.wait(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + use std::sync::Arc; + + fn hook(mode: &str, timeout_ms: u64) -> crate::git::Result { + let config = HookConfig { + dir: env!("CARGO_MANIFEST_DIR").into(), + command: vec![ + "python3".into(), + "tests/hooks/rpc_server.py".into(), + mode.into(), + ], + tags: Some(vec!["body".into()]), + min_lines: 2, + timeout_ms, + startup_timeout_ms: 10_000, + }; + Hook::spawn(&config, Path::new(".")) + } + + fn diff(rhs: &str) -> DiffResult { + let params = Config::from_toml("").unwrap().compile().unwrap(); + DiffResult::from_sources_with_options( + "file.py", + "", + rhs, + ¶ms, + &crate::options::DisplayOptions::default(), + &crate::options::DiffOptions::default(), + ) + } + + const LARGE: &str = "def f():\n a()\n b()\n c()\n\ndef g():\n d()\n"; + + #[test] + fn summaries_land_on_selected_folds_only() { + let hook = hook("first", 5000).unwrap(); + let mut result = diff(LARGE); + assert_eq!(result.rhs_folds.len(), 2); + hook.summarize(&mut result).unwrap(); + assert_eq!(result.rhs_folds[0].summary.as_deref(), Some("summary of f")); + assert_eq!(result.rhs_folds[1].summary, None); + assert!(result.lhs_folds.iter().all(|fold| fold.summary.is_none())); + } + + #[test] + fn concurrent_calls_share_one_client() { + let hook = Arc::new(hook("echo", 5000).unwrap()); + let workers: Vec<_> = (0..8) + .map(|_| { + let hook = Arc::clone(&hook); + std::thread::spawn(move || { + let mut result = diff(LARGE); + hook.summarize(&mut result).unwrap(); + result.rhs_folds[0].summary.clone() + }) + }) + .collect(); + for worker in workers { + assert_eq!(worker.join().unwrap().as_deref(), Some("pseudo Body")); + } + } + + #[test] + fn small_or_untagged_folds_never_reach_the_hook() { + let hook = hook("error", 5000).unwrap(); + let mut result = diff("import os\nimport sys\n"); + hook.summarize(&mut result).unwrap(); + let mut result = diff("def f():\n a()\n"); + hook.summarize(&mut result).unwrap(); + } + + #[test] + fn startup_failures_are_reported() { + let error = hook("exit", 5000) + .err() + .expect("exit must fail") + .to_string(); + assert!(error.contains("exited during startup"), "{error}"); + } + + #[test] + fn call_failures_are_reported_without_losing_the_diff() { + let mut result = diff(LARGE); + let error = hook("slow", 200) + .unwrap() + .summarize(&mut result) + .unwrap_err(); + assert!(error.contains("timed out"), "{error}"); + let error = hook("error", 5000) + .unwrap() + .summarize(&mut result) + .unwrap_err(); + assert!(error.contains("declined"), "{error}"); + let error = hook("bad", 5000) + .unwrap() + .summarize(&mut result) + .unwrap_err(); + assert!(error.starts_with("fold hook:"), "{error}"); + assert!(result.rhs_folds.iter().all(|fold| fold.summary.is_none())); + } +} diff --git a/src/main.rs b/src/main.rs index 7f9fa18ab..c1576de45 100644 --- a/src/main.rs +++ b/src/main.rs @@ -54,6 +54,7 @@ mod files; mod git; mod gitattributes; mod hash; +mod hook; mod line_parser; mod lines; mod options; diff --git a/src/parse/folds.rs b/src/parse/folds.rs index e4b128322..8714add33 100644 --- a/src/parse/folds.rs +++ b/src/parse/folds.rs @@ -16,6 +16,8 @@ pub(crate) struct Fold { pub(crate) match_kind: FoldMatch, /// Text shown in place of the source, including supplied pseudocode. pub(crate) placeholder: String, + /// Longer replacement text supplied by a configured fold hook. + pub(crate) summary: Option, } #[derive(Debug, Clone)] @@ -171,5 +173,6 @@ pub(crate) fn project(node: &Syntax<'_>, change: ChangeKind<'_>) -> Option .unwrap_or_default() }) .unwrap_or_else(|| "…".into()), + summary: None, }) } diff --git a/src/review/mod.rs b/src/review/mod.rs index 67fec8262..8ce09f194 100644 --- a/src/review/mod.rs +++ b/src/review/mod.rs @@ -3,4 +3,4 @@ mod render; #[cfg(test)] mod tests; -mod wire; +pub(crate) mod wire; diff --git a/src/review/wire.rs b/src/review/wire.rs index 1304205fc..ce3fc9780 100644 --- a/src/review/wire.rs +++ b/src/review/wire.rs @@ -51,7 +51,7 @@ fn position(position: &MatchedPos) -> Value { json!({"pos": span(&position.pos), "kind": kind}) } -fn range(range: &SourceRange) -> Value { +pub(crate) fn range(range: &SourceRange) -> Value { json!({ "start": { "line": range.start.line.0, @@ -74,6 +74,7 @@ fn fold(fold: &Fold) -> Value { "range": range(&fold.range), "match_kind": match_kind, "placeholder": fold.placeholder, + "summary": fold.summary, }) } diff --git a/src/stream.rs b/src/stream.rs index eacf341d2..4dd6d502e 100644 --- a/src/stream.rs +++ b/src/stream.rs @@ -1,9 +1,14 @@ //! Incremental stdout protocol over the shared file iterator. -use crate::git::{DiffSession, FileChange, Operand, Result}; +use crate::git::{DiffSession, FileChange, LoadedFile, Operand, Result}; +use crate::hook::Hook; +use crate::summary::DiffResult; +use rayon::iter::{ParallelBridge, ParallelIterator}; use serde::Serialize; use serde_json::Value; use std::io::{BufWriter, Write}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::mpsc::{sync_channel, SendError, SyncSender}; +use std::sync::Arc; use std::thread; #[derive(Serialize)] @@ -19,6 +24,9 @@ enum Event { File { file: FileChange, diff: Value, + /// The fold hook failed for this file; its folds keep their placeholders. + #[serde(skip_serializing_if = "Option::is_none")] + hook_error: Option, }, FileError { file: FileChange, @@ -30,13 +38,24 @@ enum Event { }, } -/// Returns whether any file failed. The queue holds at most one ready event; -/// computation can overlap output without retaining the whole diff. -pub(crate) fn write(session: DiffSession, output: &mut impl Write) -> Result { +/// Returns whether any file failed. Files are diffed on `jobs` workers and +/// emitted as they finish, so results arrive in completion order. The queue +/// holds at most one ready event; computation can overlap output without +/// retaining the whole diff. +pub(crate) fn write( + session: DiffSession, + jobs: usize, + hook: Option>, + output: &mut impl Write, +) -> Result { let (sender, receiver) = sync_channel(1); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(jobs) + .thread_name(|index| format!("diffr-worker-{index}")) + .build()?; let worker = thread::spawn(move || { - // A disconnected consumer cancels production after the current file. - let _ = produce(session, sender); + // A disconnected consumer cancels production after the files in flight. + let _ = produce(session, &pool, hook.as_deref(), sender); }); let mut output = BufWriter::new(output); let result: Result = (|| { @@ -61,6 +80,8 @@ pub(crate) fn write(session: DiffSession, output: &mut impl Write) -> Result, sender: SyncSender, ) -> std::result::Result<(), SendError> { sender.send(Event::Start { @@ -70,28 +91,64 @@ fn produce( total: session.remaining(), files: session.file_manifest(), })?; - let mut succeeded = 0; - let mut failed = 0; - for (file, result) in session { - let event = match result { - Ok(diff) => { - succeeded += 1; - Event::File { - file, - diff: diff.domain_json(), + let succeeded = AtomicUsize::new(0); + let failed = AtomicUsize::new(0); + let cancelled = Arc::new(AtomicBool::new(false)); + let loader = Loader { + session, + cancelled: Arc::clone(&cancelled), + }; + pool.install(|| { + loader.par_bridge().for_each(|(file, loaded)| { + let event = match loaded { + Ok(loaded) => { + succeeded.fetch_add(1, Ordering::Relaxed); + file_event(file, loaded.diff(), hook) } - } - Err(error) => { - failed += 1; - Event::FileError { - file, - message: error.to_string(), + Err(error) => { + failed.fetch_add(1, Ordering::Relaxed); + Event::FileError { + file, + message: error.to_string(), + } } + }; + if sender.send(event).is_err() { + cancelled.store(true, Ordering::Relaxed); } - }; - sender.send(event)?; + }); + }); + sender.send(Event::Complete { + succeeded: succeeded.into_inner(), + failed: failed.into_inner(), + }) +} + +/// Summaries are filled in before the event so clients never see a fold change. +fn file_event(file: FileChange, mut diff: DiffResult, hook: Option<&Hook>) -> Event { + let hook_error = hook.and_then(|hook| hook.summarize(&mut diff).err()); + Event::File { + file, + diff: diff.domain_json(), + hook_error, + } +} + +/// Reads sources serially on whichever worker pulls next; diffing then +/// proceeds on that worker while others pull further files. +struct Loader { + session: DiffSession, + cancelled: Arc, +} + +impl Iterator for Loader { + type Item = (FileChange, crate::git::Result); + fn next(&mut self) -> Option { + if self.cancelled.load(Ordering::Relaxed) { + return None; + } + self.session.load() } - sender.send(Event::Complete { succeeded, failed }) } /// Stream a standalone file comparison through the same file/completion events. @@ -99,7 +156,8 @@ fn produce( pub(crate) fn write_file( before: &str, after: &str, - compute: impl FnOnce() -> crate::summary::DiffResult, + compute: impl FnOnce() -> DiffResult, + hook: Option<&Hook>, output: &mut impl Write, ) -> Result<()> { let file = FileChange { @@ -125,14 +183,7 @@ pub(crate) fn write_file( )?; output.write_all(b"\n")?; output.flush()?; - let diff = compute(); - serde_json::to_writer( - &mut output, - &Event::File { - file, - diff: diff.domain_json(), - }, - )?; + serde_json::to_writer(&mut output, &file_event(file, compute(), hook))?; output.write_all(b"\n")?; serde_json::to_writer( &mut output, @@ -167,6 +218,7 @@ mod tests { "before.rs", "after.rs", || panic!("must not compute after manifest flush fails"), + None, &mut output, ); assert!(result.is_err()); diff --git a/tests/hooks/rpc_server.py b/tests/hooks/rpc_server.py new file mode 100755 index 000000000..668b33032 --- /dev/null +++ b/tests/hooks/rpc_server.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Minimal JSON-RPC 2.0 fold hook for tests, standard library only. + +Listens on 127.0.0.1:$DIFFR_HOOK_PORT. The first argument selects a behavior: + echo answer every fold with "pseudo " + first answer only fold 0 with "summary of f" + error return a JSON-RPC error for every request + slow never answer (sleeps inside the handler) + cwd answer fold 0 with the working directory and assert DIFFR_WORKSPACE + bad return a non-JSON body +""" + +import json +import os +import sys +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +MODE = sys.argv[1] + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + assert request["jsonrpc"] == "2.0" and request["method"] == "summarize", request + params = request["params"] + assert params["src"] and isinstance(params["folds"], list), params + if MODE == "slow": + time.sleep(30) + if MODE == "bad": + body = b"not json" + elif MODE == "error": + body = json.dumps( + { + "jsonrpc": "2.0", + "id": request["id"], + "error": {"code": -32000, "message": "declined"}, + } + ).encode() + else: + if MODE == "echo": + texts = { + str(f["id"]): "pseudo " + f["placeholder"] for f in params["folds"] + } + elif MODE == "first": + texts = {"0": "summary of f"} + elif MODE == "cwd": + assert os.environ["DIFFR_WORKSPACE"] == sys.argv[2], os.environ[ + "DIFFR_WORKSPACE" + ] + texts = {"0": os.getcwd()} + else: + raise SystemExit(f"unknown mode {MODE}") + body = json.dumps( + {"jsonrpc": "2.0", "id": request["id"], "result": texts} + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +if MODE == "exit": + raise SystemExit(3) +ThreadingHTTPServer( + ("127.0.0.1", int(os.environ["DIFFR_HOOK_PORT"])), Handler +).serve_forever() diff --git a/tests/streaming/check.py b/tests/streaming/check.py old mode 100644 new mode 100755 index 80ad5241b..db57aea86 --- a/tests/streaming/check.py +++ b/tests/streaming/check.py @@ -1,31 +1,57 @@ #!/usr/bin/env python3 """Exercise the CLI stream against real Git repositories and partial staging.""" + import json import os -from pathlib import Path +import shutil import subprocess +import sys import tempfile +from pathlib import Path ROOT = Path(__file__).resolve().parents[2] EXE = ROOT / "target/debug/diffr" -ENV = dict(os.environ, GIT_CONFIG_GLOBAL="/dev/null", GIT_CONFIG_NOSYSTEM="1", - GIT_AUTHOR_NAME="Test", GIT_AUTHOR_EMAIL="test@example.invalid", - GIT_COMMITTER_NAME="Test", GIT_COMMITTER_EMAIL="test@example.invalid") +ENV = dict( + os.environ, + GIT_CONFIG_GLOBAL="/dev/null", + GIT_CONFIG_NOSYSTEM="1", + GIT_AUTHOR_NAME="Test", + GIT_AUTHOR_EMAIL="test@example.invalid", + GIT_COMMITTER_NAME="Test", + GIT_COMMITTER_EMAIL="test@example.invalid", +) + def git(repo, *args): - return subprocess.check_output(["git", "-C", str(repo), *args], env=ENV).decode().strip() + return ( + subprocess.check_output(["git", "-C", str(repo), *args], env=ENV) + .decode() + .strip() + ) + def commit(repo, message): git(repo, "add", ".") git(repo, "commit", "-qm", message) return git(repo, "rev-parse", "HEAD") + def cli(repo, *args): - return subprocess.run([str(EXE), "--repo", str(repo), *args], capture_output=True, env=ENV) + return subprocess.run( + [str(EXE), "--repo", str(repo), *args], + capture_output=True, + env=ENV, + check=False, + ) + def stream(repo, *args, code=0): - with subprocess.Popen([str(EXE), "--repo", str(repo), "--format", "ndjson", *args], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=ENV) as process: + with subprocess.Popen( + [str(EXE), "--repo", str(repo), "--format", "ndjson", *args], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=ENV, + ) as process: first = json.loads(process.stdout.readline()) assert first["type"] == "start" and first["version"] == 1 assert len(first["files"]) == first["total"] @@ -35,11 +61,12 @@ def stream(repo, *args, code=0): assert events[-1]["type"] == "complete" succeeded = sum(e["type"] == "file" for e in events) failed = sum(e["type"] == "file_error" for e in events) - assert events[-1] == dict(type="complete", succeeded=succeeded, failed=failed) + assert events[-1] == {"type": "complete", "succeeded": succeeded, "failed": failed} assert succeeded + failed == first["total"] assert all("layout" not in e for e in events) return events + with tempfile.TemporaryDirectory(prefix="diffr-stream-") as temp: repo = Path(temp) git(repo, "init", "-q") @@ -53,25 +80,44 @@ def stream(repo, *args, code=0): (repo / "binary.bin").write_bytes(b"a\0b") (repo / "z.py").write_text("print('new')\n") head = commit(repo, "head") - (repo / ".gitattributes").write_text("*.rs diffr-classify=source\n*.py diffr-classify=test\n*.bin diffr-classify=generated\n") + (repo / ".gitattributes").write_text( + "*.rs diffr-classify=source\n*.py diffr-classify=test\n*.bin diffr-classify=generated\n" + ) (repo / "diffr.toml").write_text('[languages.rust]\nfolds = ""\n') events = stream(repo, base, head, "--order", "test,source,generated", code=2) - assert events[0]["before"] == dict(kind="revision", ref=base) - assert events[0]["after"] == dict(kind="revision", ref=head) - assert [e["file"]["class"] for e in events[1:-1]] == ["test", "test", "test", "source", "generated"] + assert events[0]["before"] == {"kind": "revision", "ref": base} + assert events[0]["after"] == {"kind": "revision", "ref": head} + # Results arrive in completion order; --order governs computation priority only. + assert sorted(e["file"]["class"] for e in events[1:-1]) == [ + "generated", + "source", + "test", + "test", + "test", + ] renamed = next(e["file"] for e in events[1:-1] if e["file"]["status"] == "renamed") assert renamed["old_path"] == "rename.py" and renamed["new_path"] == "renamed.py" rust = next(e for e in events[1:-1] if e["file"]["new_path"] == "a.rs") assert rust["diff"]["rhs_folds"] == [] # An early file failure must not prevent the later successes. events = stream(repo, base, head, "--order", "generated", code=2) - assert events[1]["type"] == "file_error" and events[-1]["succeeded"] == 4 + assert ( + sum(e["type"] == "file_error" for e in events) == 1 + and events[-1]["succeeded"] == 4 + ) events = stream(repo, base, head, "--", "a.rs", "z.py") + assert sorted(e["file"]["new_path"] for e in events[1:-1]) == ["a.rs", "z.py"] + events = stream(repo, base, head, "--jobs", "1", "--", "a.rs", "z.py") assert [e["file"]["new_path"] for e in events[1:-1]] == ["a.rs", "z.py"] assert len(stream(repo, head, head)) == 2 assert len(stream(repo, base, head, "--", "missing.rs")) == 2 stream(repo, base, head, "--exit-code", "--", "a.rs", code=1) - for args in (["bad-ref", head], [base, head, "--quiet"], ["--no-index", "a", "b"], [base, head, "--stat"]): + for args in ( + ["bad-ref", head], + [base, head, "--quiet"], + ["--no-index", "a", "b"], + [base, head, "--stat"], + ): result = cli(repo, "--format", "ndjson", *args) assert result.returncode == 2 and not result.stdout and result.stderr (repo / "diffr.toml").write_text("invalid toml") @@ -93,12 +139,18 @@ def stream(repo, *args, code=0): source.write_text(staged) git(repo, "add", "a.rs") source.write_text(working) - for selection, left, right in (([], staged, working), (["--cached"], initial, staged), ([base], initial, working)): + for selection, left, right in ( + ([], staged, working), + (["--cached"], initial, staged), + ([base], initial, working), + ): for reverse in (False, True): args = [*selection, *(["-R"] if reverse else [])] for output in ("--name-only", "--numstat"): actual = cli(repo, *args, output) - expected = subprocess.check_output(["git", "-C", str(repo), "diff", *args, output], env=ENV) + expected = subprocess.check_output( + ["git", "-C", str(repo), "diff", *args, output], env=ENV + ) assert actual.returncode == 0 and actual.stdout == expected diff = stream(repo, *args)[1]["diff"] assert diff["lhs_src"]["Text"] == (right if reverse else left) @@ -116,8 +168,61 @@ def stream(repo, *args, code=0): (repo / "new.rs").write_text("fn new() {}\n") git(repo, "add", ".") events = stream(repo, "--cached") - assert events[0]["before"] == dict(kind="empty_tree") + assert events[0]["before"] == {"kind": "empty_tree"} assert events[1]["file"]["status"] == "added" +# A configured fold hook fills summaries before each file event; its failures +# are reported per file without losing the diff. +with tempfile.TemporaryDirectory(prefix="diffr-hook-") as temp: + repo = Path(temp) + git(repo, "init", "-q") + git(repo, "commit", "--allow-empty", "-qm", "empty") + base = git(repo, "rev-parse", "HEAD") + rpc_server = ROOT / "tests/hooks/rpc_server.py" + large = "def f():\n a()\n b()\n c()\n\ndef g():\n d()\n" + (repo / "good.py").write_text(large) + (repo / "bad.py").write_text(large) + (repo / "small.py").write_text("def h():\n e()\n") + head = commit(repo, "additions") + + def hook_config(mode, *extra): + command = [sys.executable, str(rpc_server), mode, *extra] + return f"[folds.hook]\ncommand = {json.dumps(command)}\ntags = ['body']\nmin_lines = 3\n" + + (repo / "diffr.toml").write_text(hook_config("echo")) + events = {e["file"]["new_path"]: e for e in stream(repo, base, head)[1:-1]} + good = events["good.py"] + assert "hook_error" not in good + assert [ + f["summary"] for f in good["diff"]["rhs_folds"] if f["tags"] == ["body"] + ] == ["pseudo Body", None] + assert all(f["summary"] is None for f in events["small.py"]["diff"]["rhs_folds"]) + (repo / "diffr.toml").write_text(hook_config("error")) + events = {e["file"]["new_path"]: e for e in stream(repo, base, head)[1:-1]} + assert events["bad.py"]["hook_error"] == "fold hook reported: declined" + assert all(f["summary"] is None for f in events["good.py"]["diff"]["rhs_folds"]) + (repo / "diffr.toml").write_text("[folds.hook]\ncommand = ['./missing-hook']\n") + assert cli(repo, "--format", "ndjson", base, head).returncode == 2 + (repo / "diffr.toml").write_text(hook_config("exit")) + assert cli(repo, "--format", "ndjson", base, head).returncode == 2 + # Relative hook paths resolve against the config file, not the repository. + with tempfile.TemporaryDirectory(prefix="diffr-hook-config-") as elsewhere: + shutil.copy(rpc_server, Path(elsewhere) / "hook.py") + (Path(elsewhere) / "hook.toml").write_text( + f"[folds.hook]\ncommand = [{json.dumps(sys.executable)}, 'hook.py', 'cwd', {json.dumps(str(repo.resolve()) + os.sep)}]\ntags = ['body']\n" + ) + events = stream( + repo, + base, + head, + "--config", + str(Path(elsewhere) / "hook.toml"), + "--", + "good.py", + ) + assert events[1]["diff"]["rhs_folds"][0]["summary"] == str( + Path(elsewhere).resolve() + ) + # Closing the pipe while a multi-file producer is active must not leave it # blocked forever on a full queue. Unix CLI output retains normal SIGPIPE behavior. if os.name == "posix": @@ -129,8 +234,12 @@ def stream(repo, *args, code=0): for index in range(8): (repo / f"{index}.txt").write_text("some new text\n" * 4096) head = commit(repo, "large additions") - with subprocess.Popen([str(EXE), "--repo", str(repo), base, head, "--format", "ndjson"], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=ENV) as process: + with subprocess.Popen( + [str(EXE), "--repo", str(repo), base, head, "--format", "ndjson"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=ENV, + ) as process: assert json.loads(process.stdout.readline())["type"] == "start" process.stdout.close() assert process.wait(timeout=30) != 0 diff --git a/tui/packages/hunk/src/diffr/stream.test.ts b/tui/packages/hunk/src/diffr/stream.test.ts index df0803337..20b26cdd9 100644 --- a/tui/packages/hunk/src/diffr/stream.test.ts +++ b/tui/packages/hunk/src/diffr/stream.test.ts @@ -28,6 +28,7 @@ test("decode byte-fragmented Unicode stream and preserve fold metadata", async ( end: { line: 2, byte_column: 8 }, }, placeholder: "…", + summary: null, match_kind: "Novel", }, ]; diff --git a/tui/packages/hunk/src/diffr/wire.ts b/tui/packages/hunk/src/diffr/wire.ts index d008ef824..f83d32a16 100644 --- a/tui/packages/hunk/src/diffr/wire.ts +++ b/tui/packages/hunk/src/diffr/wire.ts @@ -50,6 +50,7 @@ const fold = z.object({ z.object({ Unchanged: z.object({ opposite: range }) }), ]), placeholder: z.string(), + summary: z.string().nullable(), }); const source = z.union([z.literal("Binary"), z.object({ Text: z.string() })]); export const diffResultSchema = z.object({