From 71c625d1beaf2a6feafc206134a528cb2d89b1a3 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 04:11:09 -0700 Subject: [PATCH 01/19] Separate reading file sources from diffing in DiffSession Introduce LoadedFile so sources are read on the session thread and the diff itself can later run elsewhere. No behavior change. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- src/git.rs | 61 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/src/git.rs b/src/git.rs index 24f0ed50d..2d4c9ec6c 100644 --- a/src/git.rs +++ b/src/git.rs @@ -329,28 +329,57 @@ impl DiffSession { } } -impl Iterator for DiffSession { - type Item = (FileChange, Result); - fn next(&mut self) -> Option { +/// Sources read on the session thread; diffing needs no repository access. +pub(crate) struct LoadedFile { + pub(crate) file: FileChange, + before: String, + after: String, + params: Arc, + context_lines: u32, + diff_options: crate::options::DiffOptions, +} + +impl LoadedFile { + pub(crate) fn diff(&self) -> DiffResult { + DiffResult::from_sources_with_options( + self.file.path(), + &self.before, + &self.after, + &self.params, + &crate::options::DisplayOptions { + num_context_lines: self.context_lines, + ..Default::default() + }, + &self.diff_options, + ) + } +} + +impl DiffSession { + /// Read the next file's sources without diffing them. + pub(crate) fn load(&mut self) -> Option<(FileChange, Result)> { let pending = self.files.next()?; let result = (|| { if matches!(pending.file.status, FileStatus::Conflicted) { return Err("unmerged index entry: resolve the conflict before requesting a structural diff".into()); } - let before = pending.before.read(&self.repo)?; - let after = pending.after.read(&self.repo)?; - Ok(DiffResult::from_sources_with_options( - pending.file.path(), - &before, - &after, - &self.params, - &crate::options::DisplayOptions { - num_context_lines: self.context_lines, - ..Default::default() - }, - &self.diff_options, - )) + Ok(LoadedFile { + before: pending.before.read(&self.repo)?, + after: pending.after.read(&self.repo)?, + file: pending.file.clone(), + params: Arc::clone(&self.params), + context_lines: self.context_lines, + diff_options: self.diff_options.clone(), + }) })(); Some((pending.file, result)) } } + +impl Iterator for DiffSession { + type Item = (FileChange, Result); + fn next(&mut self) -> Option { + let (file, loaded) = self.load()?; + Some((file, loaded.map(|loaded| loaded.diff()))) + } +} From 4e3dfc7ad9e9bc344e24d80a0569ba9c82324e56 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 04:15:28 -0700 Subject: [PATCH 02/19] Diff files on a worker pool and emit results as they finish Streaming output no longer waits for earlier files: a rayon pool of --jobs workers (default 16) pulls sources through the session serially and diffs them concurrently. The file manifest on the start event already lets clients lay out results in any order. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- src/cli.rs | 14 ++++++- src/stream.rs | 88 +++++++++++++++++++++++++++++----------- tests/streaming/check.py | 7 +++- 3 files changed, 82 insertions(+), 27 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 1d2d8e053..d55df378f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -34,6 +34,14 @@ pub(crate) fn run() -> Result { .about("Structural diffs with Git-style comparison inputs") .arg(Arg::new("repo").long("repo").default_value(".")) .arg(Arg::new("config").long("config")) + .arg( + Arg::new("jobs") + .long("jobs") + .short('j') + .value_parser(clap::value_parser!(usize)) + .default_value("16") + .help("Concurrent file diffs for --format ndjson; results are emitted as each finishes"), + ) .arg(Arg::new("order").long("order").value_delimiter(',').action(ArgAction::Append).help("File class priority from diffr-classify attributes")) .arg(flag("cached").visible_alias("staged")) .arg(flag("merge-base")) @@ -162,7 +170,11 @@ pub(crate) fn run() -> Result { session.diff_options = diff_options; let changed = session.remaining() > 0; if streaming { - let failed = crate::stream::write(session, &mut io::stdout().lock())?; + let jobs = *args.get_one::("jobs").unwrap(); + if jobs == 0 { + return Err("--jobs must be at least 1".into()); + } + let failed = crate::stream::write(session, jobs, &mut io::stdout().lock())?; return Ok(if failed { 2 } else { diff --git a/src/stream.rs b/src/stream.rs index eacf341d2..18ef927e5 100644 --- a/src/stream.rs +++ b/src/stream.rs @@ -1,9 +1,12 @@ //! Incremental stdout protocol over the shared file iterator. -use crate::git::{DiffSession, FileChange, Operand, Result}; +use crate::git::{DiffSession, FileChange, LoadedFile, Operand, Result}; +use rayon::iter::{ParallelBridge, ParallelIterator}; use serde::Serialize; use serde_json::Value; use std::io::{BufWriter, Write}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::mpsc::{sync_channel, SendError, SyncSender}; +use std::sync::Arc; use std::thread; #[derive(Serialize)] @@ -30,13 +33,19 @@ enum Event { }, } -/// Returns whether any file failed. The queue holds at most one ready event; -/// computation can overlap output without retaining the whole diff. -pub(crate) fn write(session: DiffSession, output: &mut impl Write) -> Result { +/// Returns whether any file failed. Files are diffed on `jobs` workers and +/// emitted as they finish, so results arrive in completion order. The queue +/// holds at most one ready event; computation can overlap output without +/// retaining the whole diff. +pub(crate) fn write(session: DiffSession, jobs: usize, output: &mut impl Write) -> Result { let (sender, receiver) = sync_channel(1); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(jobs) + .thread_name(|index| format!("diffr-worker-{index}")) + .build()?; let worker = thread::spawn(move || { - // A disconnected consumer cancels production after the current file. - let _ = produce(session, sender); + // A disconnected consumer cancels production after the files in flight. + let _ = produce(session, &pool, sender); }); let mut output = BufWriter::new(output); let result: Result = (|| { @@ -61,6 +70,7 @@ pub(crate) fn write(session: DiffSession, output: &mut impl Write) -> Result, ) -> std::result::Result<(), SendError> { sender.send(Event::Start { @@ -70,28 +80,58 @@ fn produce( total: session.remaining(), files: session.file_manifest(), })?; - let mut succeeded = 0; - let mut failed = 0; - for (file, result) in session { - let event = match result { - Ok(diff) => { - succeeded += 1; - Event::File { - file, - diff: diff.domain_json(), + let succeeded = AtomicUsize::new(0); + let failed = AtomicUsize::new(0); + let cancelled = Arc::new(AtomicBool::new(false)); + let loader = Loader { + session, + cancelled: Arc::clone(&cancelled), + }; + pool.install(|| { + loader.par_bridge().for_each(|(file, loaded)| { + let event = match loaded { + Ok(loaded) => { + let diff = loaded.diff(); + succeeded.fetch_add(1, Ordering::Relaxed); + Event::File { + file, + diff: diff.domain_json(), + } } - } - Err(error) => { - failed += 1; - Event::FileError { - file, - message: error.to_string(), + Err(error) => { + failed.fetch_add(1, Ordering::Relaxed); + Event::FileError { + file, + message: error.to_string(), + } } + }; + if sender.send(event).is_err() { + cancelled.store(true, Ordering::Relaxed); } - }; - sender.send(event)?; + }); + }); + sender.send(Event::Complete { + succeeded: succeeded.into_inner(), + failed: failed.into_inner(), + }) +} + +/// Reads sources serially on whichever worker pulls next; diffing then +/// proceeds on that worker while others pull further files. +struct Loader { + session: DiffSession, + cancelled: Arc, +} + +impl Iterator for Loader { + type Item = (FileChange, crate::git::Result); + fn next(&mut self) -> Option { + if self.cancelled.load(Ordering::Relaxed) { + return None; + } + self.session.load() } - sender.send(Event::Complete { succeeded, failed }) } /// Stream a standalone file comparison through the same file/completion events. diff --git a/tests/streaming/check.py b/tests/streaming/check.py index 80ad5241b..eb53218d9 100644 --- a/tests/streaming/check.py +++ b/tests/streaming/check.py @@ -58,15 +58,18 @@ def stream(repo, *args, code=0): events = stream(repo, base, head, "--order", "test,source,generated", code=2) assert events[0]["before"] == dict(kind="revision", ref=base) assert events[0]["after"] == dict(kind="revision", ref=head) - assert [e["file"]["class"] for e in events[1:-1]] == ["test", "test", "test", "source", "generated"] + # Results arrive in completion order; --order governs computation priority only. + assert sorted(e["file"]["class"] for e in events[1:-1]) == ["generated", "source", "test", "test", "test"] renamed = next(e["file"] for e in events[1:-1] if e["file"]["status"] == "renamed") assert renamed["old_path"] == "rename.py" and renamed["new_path"] == "renamed.py" rust = next(e for e in events[1:-1] if e["file"]["new_path"] == "a.rs") assert rust["diff"]["rhs_folds"] == [] # An early file failure must not prevent the later successes. events = stream(repo, base, head, "--order", "generated", code=2) - assert events[1]["type"] == "file_error" and events[-1]["succeeded"] == 4 + assert sum(e["type"] == "file_error" for e in events) == 1 and events[-1]["succeeded"] == 4 events = stream(repo, base, head, "--", "a.rs", "z.py") + assert sorted(e["file"]["new_path"] for e in events[1:-1]) == ["a.rs", "z.py"] + events = stream(repo, base, head, "--jobs", "1", "--", "a.rs", "z.py") assert [e["file"]["new_path"] for e in events[1:-1]] == ["a.rs", "z.py"] assert len(stream(repo, head, head)) == 2 assert len(stream(repo, base, head, "--", "missing.rs")) == 2 From a077d37e49b62df3c7ba3f32ab558e6fd5d2bf55 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 04:16:16 -0700 Subject: [PATCH 03/19] Carry an optional summary on every fold Folds gain a summary field, always null for now, on the domain model, the wire format, and the terminal frontend schema. No behavior change. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- src/parse/folds.rs | 3 +++ src/review/wire.rs | 1 + tui/packages/hunk/src/diffr/stream.test.ts | 1 + tui/packages/hunk/src/diffr/wire.ts | 1 + 4 files changed, 6 insertions(+) diff --git a/src/parse/folds.rs b/src/parse/folds.rs index e4b128322..8714add33 100644 --- a/src/parse/folds.rs +++ b/src/parse/folds.rs @@ -16,6 +16,8 @@ pub(crate) struct Fold { pub(crate) match_kind: FoldMatch, /// Text shown in place of the source, including supplied pseudocode. pub(crate) placeholder: String, + /// Longer replacement text supplied by a configured fold hook. + pub(crate) summary: Option, } #[derive(Debug, Clone)] @@ -171,5 +173,6 @@ pub(crate) fn project(node: &Syntax<'_>, change: ChangeKind<'_>) -> Option .unwrap_or_default() }) .unwrap_or_else(|| "…".into()), + summary: None, }) } diff --git a/src/review/wire.rs b/src/review/wire.rs index 1304205fc..7abf8cab2 100644 --- a/src/review/wire.rs +++ b/src/review/wire.rs @@ -74,6 +74,7 @@ fn fold(fold: &Fold) -> Value { "range": range(&fold.range), "match_kind": match_kind, "placeholder": fold.placeholder, + "summary": fold.summary, }) } diff --git a/tui/packages/hunk/src/diffr/stream.test.ts b/tui/packages/hunk/src/diffr/stream.test.ts index df0803337..20b26cdd9 100644 --- a/tui/packages/hunk/src/diffr/stream.test.ts +++ b/tui/packages/hunk/src/diffr/stream.test.ts @@ -28,6 +28,7 @@ test("decode byte-fragmented Unicode stream and preserve fold metadata", async ( end: { line: 2, byte_column: 8 }, }, placeholder: "…", + summary: null, match_kind: "Novel", }, ]; diff --git a/tui/packages/hunk/src/diffr/wire.ts b/tui/packages/hunk/src/diffr/wire.ts index d008ef824..f83d32a16 100644 --- a/tui/packages/hunk/src/diffr/wire.ts +++ b/tui/packages/hunk/src/diffr/wire.ts @@ -50,6 +50,7 @@ const fold = z.object({ z.object({ Unchanged: z.object({ opposite: range }) }), ]), placeholder: z.string(), + summary: z.string().nullable(), }); const source = z.union([z.literal("Binary"), z.object({ Text: z.string() })]); export const diffResultSchema = z.object({ From 12c38d5e26170560295afb2d6bda7d559c26e26c Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 04:16:50 -0700 Subject: [PATCH 04/19] Parse fold hook settings from diffr.toml A [folds.hook] table names a trusted command plus tag, size and timeout filters. Parsed and validated only; nothing invokes it yet. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- src/config.rs | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 05b4ce2bf..920c15f82 100644 --- a/src/config.rs +++ b/src/config.rs @@ -13,6 +13,31 @@ use strum::IntoEnumIterator; #[serde(default, deny_unknown_fields)] pub(crate) struct Config { pub(crate) languages: BTreeMap, + pub(crate) folds: FoldsConfig, +} + +#[derive(Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub(crate) struct FoldsConfig { + pub(crate) hook: Option, +} + +/// A trusted subprocess that supplies summaries for large novel folds. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct HookConfig { + pub(crate) command: Vec, + /// None sends every tagged fold; otherwise a fold needs one of these tags. + #[serde(default)] + pub(crate) tags: Option>, + #[serde(default)] + pub(crate) min_lines: usize, + #[serde(default = "default_timeout_ms")] + pub(crate) timeout_ms: u64, +} + +fn default_timeout_ms() -> u64 { + 5000 } #[derive(Default, Deserialize)] @@ -34,6 +59,7 @@ impl std::error::Error for ConfigError {} pub(crate) struct Params { languages: DftHashMap>>, + pub(crate) hook: Option, } pub(crate) struct LanguageParams { @@ -78,6 +104,14 @@ impl Config { } pub(crate) fn compile(self) -> Result { + if let Some(hook) = &self.folds.hook { + if hook.command.is_empty() { + return Err(ConfigError("folds.hook.command must not be empty".into())); + } + if hook.timeout_ms == 0 { + return Err(ConfigError("folds.hook.timeout_ms must be positive".into())); + } + } let defaults = Self::from_toml(include_str!("config/defaults.toml"))?; let mut resolved = defaults.languages; for (name, overrides) in self.languages { @@ -111,7 +145,10 @@ impl Config { })), ); } - Ok(Params { languages }) + Ok(Params { + languages, + hook: self.folds.hook, + }) } } @@ -231,6 +268,28 @@ mod tests { assert_eq!(result.rhs_folds[0].tags, ["embedded"]); } + #[test] + fn parses_fold_hook_settings() { + let params = Config::from_toml( + "[folds.hook]\ncommand = ['uv', 'run', 'summarize.py']\ntags = ['body']\nmin_lines = 30", + ) + .unwrap() + .compile() + .unwrap(); + let hook = params.hook.unwrap(); + assert_eq!(hook.command, ["uv", "run", "summarize.py"]); + assert_eq!(hook.tags.as_deref(), Some(&["body".to_owned()][..])); + assert_eq!((hook.min_lines, hook.timeout_ms), (30, 5000)); + assert!(Config::from_toml("").unwrap().compile().unwrap().hook.is_none()); + for input in [ + "[folds.hook]\ncommand = []", + "[folds.hook]\ncommand = ['x']\ntimeout_ms = 0", + ] { + assert!(Config::from_toml(input).unwrap().compile().is_err(), "{input}"); + } + assert!(Config::from_toml("[folds.hook]\ncommand = ['x']\nunknown = 1").is_err()); + } + #[test] fn rejects_unknown_settings_languages_and_invalid_queries() { assert!(Config::from_toml("typo = true").is_err()); From a9074ddb082a14ca9c7d6dd954ca5763a1c2412b Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 04:19:57 -0700 Subject: [PATCH 05/19] Summarize large novel folds through a configured hook subprocess Streaming spawns the [folds.hook] command once and sends each file's qualifying rhs folds as one NDJSON request; the worker blocks on the reply and fills fold summaries before emitting the file event. Replies are routed by request id so files in flight overlap freely. A failed or timed-out request leaves placeholders in place and adds hook_error to that file. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- src/cli.rs | 15 +- src/hook.rs | 334 +++++++++++++++++++++++++++++++++++++++ src/main.rs | 1 + src/review/mod.rs | 2 +- src/review/wire.rs | 2 +- src/stream.rs | 44 ++++-- tests/streaming/check.py | 34 ++++ 7 files changed, 413 insertions(+), 19 deletions(-) create mode 100644 src/hook.rs diff --git a/src/cli.rs b/src/cli.rs index d55df378f..c67e44727 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,6 +1,7 @@ //! Git-style CLI input; rendering and NDJSON remain adapters over the same engine. use crate::config::Config; use crate::git::{Comparison, DiffSession, FileParams, Operand, Result}; +use crate::hook::Hook; use crate::options::{DiffOptions, DisplayMode, DisplayOptions}; use clap::{Arg, ArgAction, ArgGroup, ArgMatches, Command}; use git2::{DiffStatsFormat, Repository}; @@ -165,6 +166,7 @@ pub(crate) fn run() -> Result { let params = Arc::new( Config::load(workspace, args.get_one::("config").map(Path::new))?.compile()?, ); + let hook = fold_hook(¶ms, workspace)?; let mut session = DiffSession::open(workspace, comparison, params, &files)?; session.context_lines = display.num_context_lines; session.diff_options = diff_options; @@ -174,7 +176,7 @@ pub(crate) fn run() -> Result { if jobs == 0 { return Err("--jobs must be at least 1".into()); } - let failed = crate::stream::write(session, jobs, &mut io::stdout().lock())?; + let failed = crate::stream::write(session, jobs, hook, &mut io::stdout().lock())?; return Ok(if failed { 2 } else { @@ -459,10 +461,12 @@ fn no_index( ) }; if args.get_one::("format").map(String::as_str) == Some("ndjson") { + let hook = fold_hook(&config, Path::new(args.get_one::("repo").unwrap()))?; crate::stream::write_file( &paths[0].to_string_lossy(), &paths[1].to_string_lossy(), compute, + hook.as_deref(), &mut io::stdout().lock(), )?; Ok(i32::from(changed && args.get_flag("exit-code"))) @@ -472,6 +476,15 @@ fn no_index( } } +/// Streaming output summarizes large novel folds through the configured hook. +fn fold_hook(params: &crate::config::Params, workspace: &Path) -> Result>> { + params + .hook + .as_ref() + .map(|config| Hook::spawn(config, workspace).map(Arc::new)) + .transpose() +} + /// Explicit machine/text modes and redirected output must never enter the alternate screen. fn opens_tui(explicit_format: bool, metadata_or_quiet: bool, terminal: bool) -> bool { terminal && !explicit_format && !metadata_or_quiet diff --git a/src/hook.rs b/src/hook.rs new file mode 100644 index 000000000..d54ef48b3 --- /dev/null +++ b/src/hook.rs @@ -0,0 +1,334 @@ +//! Trusted fold-summary subprocess: spawned once per session, NDJSON both ways. +//! +//! Each request carries one file and its large novel folds on the after side. +//! The hook replies in any order, keyed by request id; callers block on their +//! own reply so files still stream out as each worker finishes. +use crate::config::HookConfig; +use crate::parse::folds::FoldMatch; +use crate::review::wire; +use crate::summary::{DiffResult, FileContent, FileFormat}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Write}; +use std::path::Path; +use std::process::{Child, ChildStdin, Command, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{sync_channel, RecvTimeoutError, SyncSender}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::Duration; + +#[derive(Serialize)] +struct Request<'a> { + id: u64, + path: &'a str, + language: Option<&'static str>, + src: &'a str, + folds: Vec>, +} + +#[derive(Serialize)] +struct RequestFold<'a> { + /// Index into the file's rhs folds. + id: usize, + range: Value, + tags: &'a [String], + placeholder: &'a str, +} + +#[derive(Deserialize)] +struct Reply { + id: u64, + #[serde(default)] + texts: HashMap, + #[serde(default)] + error: Option, +} + +enum Outcome { + Reply(Reply), + /// The hook can no longer answer anything. + Closed(String), +} + +type Pending = Arc>, String>>>; + +pub(crate) struct Hook { + config: HookConfig, + child: Mutex, + stdin: Mutex, + pending: Pending, + next_id: AtomicU64, + reader: Mutex>>, +} + +impl Hook { + pub(crate) fn spawn(config: &HookConfig, workspace: &Path) -> crate::git::Result { + let mut child = Command::new(&config.command[0]) + .args(&config.command[1..]) + .current_dir(workspace) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .map_err(|error| format!("could not start fold hook {:?}: {error}", config.command))?; + let stdin = child.stdin.take().expect("piped hook stdin"); + let stdout = child.stdout.take().expect("piped hook stdout"); + let pending: Pending = Arc::new(Mutex::new(Ok(HashMap::new()))); + let reader = std::thread::spawn({ + let pending = Arc::clone(&pending); + move || dispatch(BufReader::new(stdout), &pending) + }); + Ok(Self { + config: config.clone(), + child: Mutex::new(child), + stdin: Mutex::new(stdin), + pending, + next_id: AtomicU64::new(1), + reader: Mutex::new(Some(reader)), + }) + } + + /// Fill in summaries for this file's qualifying folds, blocking on the hook. + /// Files without qualifying folds never reach the hook. + pub(crate) fn summarize(&self, diff: &mut DiffResult) -> Result<(), String> { + let selected: Vec = diff + .rhs_folds + .iter() + .enumerate() + .filter(|(_, fold)| self.qualifies(fold)) + .map(|(index, _)| index) + .collect(); + if selected.is_empty() { + return Ok(()); + } + let FileContent::Text(src) = &diff.rhs_src else { + return Ok(()); + }; + let language = match &diff.file_format { + FileFormat::SupportedLanguage(language) => { + Some(crate::parse::guess_language::language_name(*language)) + } + _ => None, + }; + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let request = Request { + id, + path: &diff.display_path, + language, + src, + folds: selected + .iter() + .map(|&index| { + let fold = &diff.rhs_folds[index]; + RequestFold { + id: index, + range: wire::range(&fold.range), + tags: &fold.tags, + placeholder: &fold.placeholder, + } + }) + .collect(), + }; + let mut line = serde_json::to_vec(&request).map_err(|error| error.to_string())?; + line.push(b'\n'); + let (sender, receiver) = sync_channel(1); + { + let mut pending = self.pending.lock().unwrap(); + match pending.as_mut() { + Ok(pending) => pending.insert(id, sender), + Err(reason) => return Err(reason.clone()), + }; + } + { + let mut stdin = self.stdin.lock().unwrap(); + if let Err(error) = stdin.write_all(&line).and_then(|()| stdin.flush()) { + self.forget(id); + return Err(format!("fold hook stdin closed: {error}")); + } + } + let reply = match receiver.recv_timeout(Duration::from_millis(self.config.timeout_ms)) { + Ok(Outcome::Reply(reply)) => reply, + Ok(Outcome::Closed(reason)) => return Err(reason), + Err(RecvTimeoutError::Timeout) => { + self.forget(id); + return Err(format!("fold hook timed out after {}ms", self.config.timeout_ms)); + } + Err(RecvTimeoutError::Disconnected) => { + unreachable!("reader drops senders only after signalling") + } + }; + if let Some(error) = reply.error { + return Err(format!("fold hook reported: {error}")); + } + for (key, text) in reply.texts { + let index: usize = key + .parse() + .ok() + .filter(|index| selected.contains(index)) + .ok_or_else(|| format!("fold hook answered for unknown fold {key:?}"))?; + diff.rhs_folds[index].summary = Some(text); + } + Ok(()) + } + + fn qualifies(&self, fold: &crate::parse::folds::Fold) -> bool { + if !matches!(fold.match_kind, FoldMatch::Novel) { + return false; + } + let lines = (fold.range.end.line.0 - fold.range.start.line.0 + 1) as usize; + if lines < self.config.min_lines { + return false; + } + match &self.config.tags { + Some(tags) => fold.tags.iter().any(|tag| tags.contains(tag)), + None => true, + } + } + + fn forget(&self, id: u64) { + if let Ok(pending) = self.pending.lock().unwrap().as_mut() { + pending.remove(&id); + } + } +} + +/// Route replies to their waiting request; a malformed line or EOF fails every +/// current and future request, since ids can no longer be trusted. +fn dispatch(stdout: BufReader, pending: &Pending) { + let close = |reason: String| { + let mut pending = pending.lock().unwrap(); + if let Ok(waiting) = std::mem::replace(&mut *pending, Err(reason.clone())) { + for (_, sender) in waiting { + let _ = sender.send(Outcome::Closed(reason.clone())); + } + } + }; + for line in stdout.lines() { + let line = match line { + Ok(line) => line, + Err(error) => return close(format!("fold hook stdout unreadable: {error}")), + }; + if line.trim().is_empty() { + continue; + } + let reply: Reply = match serde_json::from_str(&line) { + Ok(reply) => reply, + Err(error) => return close(format!("fold hook wrote an invalid reply: {error}")), + }; + let sender = match pending.lock().unwrap().as_mut() { + Ok(waiting) => waiting.remove(&reply.id), + Err(_) => return, + }; + // A request that already timed out has no receiver; drop the late reply. + if let Some(sender) = sender { + let _ = sender.send(Outcome::Reply(reply)); + } + } + close("fold hook exited".into()); +} + +impl Drop for Hook { + fn drop(&mut self) { + let mut child = self.child.lock().unwrap(); + let _ = child.kill(); + let _ = child.wait(); + if let Some(reader) = self.reader.lock().unwrap().take() { + let _ = reader.join(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + + fn hook(script: &str, timeout_ms: u64) -> Hook { + let config = HookConfig { + command: vec!["sh".into(), "-c".into(), script.into()], + tags: Some(vec!["body".into()]), + min_lines: 2, + timeout_ms, + }; + Hook::spawn(&config, Path::new(".")).unwrap() + } + + fn diff(rhs: &str) -> DiffResult { + let params = Config::from_toml("").unwrap().compile().unwrap(); + DiffResult::from_sources_with_options( + "file.py", + "", + rhs, + ¶ms, + &crate::options::DisplayOptions::default(), + &crate::options::DiffOptions::default(), + ) + } + + const LARGE: &str = "def f():\n a()\n b()\n c()\n\ndef g():\n d()\n"; + /// Echo each request id back with a summary for fold 0. + const ECHO: &str = r#"while IFS= read -r line; do id=$(printf '%s' "$line" | sed -E 's/^\{"id":([0-9]+).*/\1/'); printf '{"id":%s,"texts":{"0":"summary of f"}}\n' "$id"; done"#; + + #[test] + fn summaries_land_on_selected_folds_only() { + let hook = hook(ECHO, 5000); + let mut result = diff(LARGE); + assert_eq!(result.rhs_folds.len(), 2); + hook.summarize(&mut result).unwrap(); + assert_eq!(result.rhs_folds[0].summary.as_deref(), Some("summary of f")); + assert_eq!(result.rhs_folds[1].summary, None); + assert!(result.lhs_folds.iter().all(|fold| fold.summary.is_none())); + } + + #[test] + fn concurrent_requests_are_routed_by_id() { + let hook = Arc::new(hook(ECHO, 5000)); + let workers: Vec<_> = (0..8) + .map(|_| { + let hook = Arc::clone(&hook); + std::thread::spawn(move || { + let mut result = diff(LARGE); + hook.summarize(&mut result).unwrap(); + result.rhs_folds[0].summary.clone() + }) + }) + .collect(); + for worker in workers { + assert_eq!(worker.join().unwrap().as_deref(), Some("summary of f")); + } + } + + #[test] + fn small_or_untagged_folds_never_reach_the_hook() { + let hook = hook("exit 3", 5000); + let mut result = diff("import os\nimport sys\n"); + hook.summarize(&mut result).unwrap(); + let mut result = diff("def f():\n a()\n"); + hook.summarize(&mut result).unwrap(); + } + + #[test] + fn failures_are_reported_without_losing_the_diff() { + let mut result = diff(LARGE); + let error = hook("sleep 30", 50).summarize(&mut result).unwrap_err(); + assert!(error.contains("timed out"), "{error}"); + let error = hook("exit 0", 5000).summarize(&mut result).unwrap_err(); + assert!(error.contains("exited"), "{error}"); + let error = hook("echo not json", 5000).summarize(&mut result).unwrap_err(); + assert!(error.contains("invalid reply"), "{error}"); + let error = hook( + r#"read -r line; echo '{"id":1,"error":"rate limited"}'; sleep 30"#, + 5000, + ) + .summarize(&mut result) + .unwrap_err(); + assert!(error.contains("rate limited"), "{error}"); + let error = hook(r#"read -r line; echo '{"id":1,"texts":{"7":"x"}}'; sleep 30"#, 5000) + .summarize(&mut result) + .unwrap_err(); + assert!(error.contains("unknown fold"), "{error}"); + assert!(result.rhs_folds.iter().all(|fold| fold.summary.is_none())); + } +} diff --git a/src/main.rs b/src/main.rs index 7f9fa18ab..c1576de45 100644 --- a/src/main.rs +++ b/src/main.rs @@ -54,6 +54,7 @@ mod files; mod git; mod gitattributes; mod hash; +mod hook; mod line_parser; mod lines; mod options; diff --git a/src/review/mod.rs b/src/review/mod.rs index 67fec8262..8ce09f194 100644 --- a/src/review/mod.rs +++ b/src/review/mod.rs @@ -3,4 +3,4 @@ mod render; #[cfg(test)] mod tests; -mod wire; +pub(crate) mod wire; diff --git a/src/review/wire.rs b/src/review/wire.rs index 7abf8cab2..ce3fc9780 100644 --- a/src/review/wire.rs +++ b/src/review/wire.rs @@ -51,7 +51,7 @@ fn position(position: &MatchedPos) -> Value { json!({"pos": span(&position.pos), "kind": kind}) } -fn range(range: &SourceRange) -> Value { +pub(crate) fn range(range: &SourceRange) -> Value { json!({ "start": { "line": range.start.line.0, diff --git a/src/stream.rs b/src/stream.rs index 18ef927e5..4dd6d502e 100644 --- a/src/stream.rs +++ b/src/stream.rs @@ -1,5 +1,7 @@ //! Incremental stdout protocol over the shared file iterator. use crate::git::{DiffSession, FileChange, LoadedFile, Operand, Result}; +use crate::hook::Hook; +use crate::summary::DiffResult; use rayon::iter::{ParallelBridge, ParallelIterator}; use serde::Serialize; use serde_json::Value; @@ -22,6 +24,9 @@ enum Event { File { file: FileChange, diff: Value, + /// The fold hook failed for this file; its folds keep their placeholders. + #[serde(skip_serializing_if = "Option::is_none")] + hook_error: Option, }, FileError { file: FileChange, @@ -37,7 +42,12 @@ enum Event { /// emitted as they finish, so results arrive in completion order. The queue /// holds at most one ready event; computation can overlap output without /// retaining the whole diff. -pub(crate) fn write(session: DiffSession, jobs: usize, output: &mut impl Write) -> Result { +pub(crate) fn write( + session: DiffSession, + jobs: usize, + hook: Option>, + output: &mut impl Write, +) -> Result { let (sender, receiver) = sync_channel(1); let pool = rayon::ThreadPoolBuilder::new() .num_threads(jobs) @@ -45,7 +55,7 @@ pub(crate) fn write(session: DiffSession, jobs: usize, output: &mut impl Write) .build()?; let worker = thread::spawn(move || { // A disconnected consumer cancels production after the files in flight. - let _ = produce(session, &pool, sender); + let _ = produce(session, &pool, hook.as_deref(), sender); }); let mut output = BufWriter::new(output); let result: Result = (|| { @@ -71,6 +81,7 @@ pub(crate) fn write(session: DiffSession, jobs: usize, output: &mut impl Write) fn produce( session: DiffSession, pool: &rayon::ThreadPool, + hook: Option<&Hook>, sender: SyncSender, ) -> std::result::Result<(), SendError> { sender.send(Event::Start { @@ -91,12 +102,8 @@ fn produce( loader.par_bridge().for_each(|(file, loaded)| { let event = match loaded { Ok(loaded) => { - let diff = loaded.diff(); succeeded.fetch_add(1, Ordering::Relaxed); - Event::File { - file, - diff: diff.domain_json(), - } + file_event(file, loaded.diff(), hook) } Err(error) => { failed.fetch_add(1, Ordering::Relaxed); @@ -117,6 +124,16 @@ fn produce( }) } +/// Summaries are filled in before the event so clients never see a fold change. +fn file_event(file: FileChange, mut diff: DiffResult, hook: Option<&Hook>) -> Event { + let hook_error = hook.and_then(|hook| hook.summarize(&mut diff).err()); + Event::File { + file, + diff: diff.domain_json(), + hook_error, + } +} + /// Reads sources serially on whichever worker pulls next; diffing then /// proceeds on that worker while others pull further files. struct Loader { @@ -139,7 +156,8 @@ impl Iterator for Loader { pub(crate) fn write_file( before: &str, after: &str, - compute: impl FnOnce() -> crate::summary::DiffResult, + compute: impl FnOnce() -> DiffResult, + hook: Option<&Hook>, output: &mut impl Write, ) -> Result<()> { let file = FileChange { @@ -165,14 +183,7 @@ pub(crate) fn write_file( )?; output.write_all(b"\n")?; output.flush()?; - let diff = compute(); - serde_json::to_writer( - &mut output, - &Event::File { - file, - diff: diff.domain_json(), - }, - )?; + serde_json::to_writer(&mut output, &file_event(file, compute(), hook))?; output.write_all(b"\n")?; serde_json::to_writer( &mut output, @@ -207,6 +218,7 @@ mod tests { "before.rs", "after.rs", || panic!("must not compute after manifest flush fails"), + None, &mut output, ); assert!(result.is_err()); diff --git a/tests/streaming/check.py b/tests/streaming/check.py index eb53218d9..9f4869853 100644 --- a/tests/streaming/check.py +++ b/tests/streaming/check.py @@ -4,6 +4,7 @@ import os from pathlib import Path import subprocess +import sys import tempfile ROOT = Path(__file__).resolve().parents[2] @@ -121,6 +122,39 @@ def stream(repo, *args, code=0): events = stream(repo, "--cached") assert events[0]["before"] == dict(kind="empty_tree") assert events[1]["file"]["status"] == "added" +# A configured fold hook fills summaries before each file event; its failures +# are reported per file without losing the diff. +with tempfile.TemporaryDirectory(prefix="diffr-hook-") as temp: + repo = Path(temp) + git(repo, "init", "-q") + git(repo, "commit", "--allow-empty", "-qm", "empty") + base = git(repo, "rev-parse", "HEAD") + (repo / "hook.py").write_text( + "import json, sys\n" + "for line in sys.stdin:\n" + " request = json.loads(line)\n" + " assert request['language'] == 'Python' and request['src']\n" + " if request['path'] == 'bad.py':\n" + " reply = {'id': request['id'], 'error': 'declined'}\n" + " else:\n" + " reply = {'id': request['id'], 'texts': {str(f['id']): 'pseudo ' + f['placeholder'] for f in request['folds']}}\n" + " print(json.dumps(reply), flush=True)\n") + large = "def f():\n a()\n b()\n c()\n\ndef g():\n d()\n" + (repo / "good.py").write_text(large) + (repo / "bad.py").write_text(large) + (repo / "small.py").write_text("def h():\n e()\n") + head = commit(repo, "additions") + (repo / "diffr.toml").write_text( + f"[folds.hook]\ncommand = [{json.dumps(sys.executable)}, 'hook.py']\ntags = ['body']\nmin_lines = 3\n") + events = {e["file"]["new_path"]: e for e in stream(repo, base, head)[1:-1]} + good = events["good.py"] + assert "hook_error" not in good + assert [f["summary"] for f in good["diff"]["rhs_folds"] if f["tags"] == ["body"]] == ["pseudo Body", None] + assert events["bad.py"]["hook_error"] == "fold hook reported: declined" + assert all(f["summary"] is None for f in events["small.py"]["diff"]["rhs_folds"]) + (repo / "diffr.toml").write_text("[folds.hook]\ncommand = ['./missing-hook']\n") + assert cli(repo, "--format", "ndjson", base, head).returncode == 2 + # Closing the pipe while a multi-file producer is active must not leave it # blocked forever on a full queue. Unix CLI output retains normal SIGPIPE behavior. if os.name == "posix": From 927393415e002ef044707017179188ee35907043 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 04:21:01 -0700 Subject: [PATCH 06/19] Satisfy clippy and rustfmt in the fold hook AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- src/config.rs | 12 ++++++++++-- src/hook.rs | 27 ++++++++++++++++++--------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/config.rs b/src/config.rs index 920c15f82..6953c21ef 100644 --- a/src/config.rs +++ b/src/config.rs @@ -280,12 +280,20 @@ mod tests { assert_eq!(hook.command, ["uv", "run", "summarize.py"]); assert_eq!(hook.tags.as_deref(), Some(&["body".to_owned()][..])); assert_eq!((hook.min_lines, hook.timeout_ms), (30, 5000)); - assert!(Config::from_toml("").unwrap().compile().unwrap().hook.is_none()); + assert!(Config::from_toml("") + .unwrap() + .compile() + .unwrap() + .hook + .is_none()); for input in [ "[folds.hook]\ncommand = []", "[folds.hook]\ncommand = ['x']\ntimeout_ms = 0", ] { - assert!(Config::from_toml(input).unwrap().compile().is_err(), "{input}"); + assert!( + Config::from_toml(input).unwrap().compile().is_err(), + "{input}" + ); } assert!(Config::from_toml("[folds.hook]\ncommand = ['x']\nunknown = 1").is_err()); } diff --git a/src/hook.rs b/src/hook.rs index d54ef48b3..68f694302 100644 --- a/src/hook.rs +++ b/src/hook.rs @@ -4,12 +4,13 @@ //! The hook replies in any order, keyed by request id; callers block on their //! own reply so files still stream out as each worker finishes. use crate::config::HookConfig; +use crate::hash::DftHashMap; use crate::parse::folds::FoldMatch; use crate::review::wire; use crate::summary::{DiffResult, FileContent, FileFormat}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::HashMap; +use std::collections::BTreeMap; use std::io::{BufRead, BufReader, Write}; use std::path::Path; use std::process::{Child, ChildStdin, Command, Stdio}; @@ -41,7 +42,7 @@ struct RequestFold<'a> { struct Reply { id: u64, #[serde(default)] - texts: HashMap, + texts: BTreeMap, #[serde(default)] error: Option, } @@ -52,7 +53,7 @@ enum Outcome { Closed(String), } -type Pending = Arc>, String>>>; +type Pending = Arc>, String>>>; pub(crate) struct Hook { config: HookConfig, @@ -75,7 +76,7 @@ impl Hook { .map_err(|error| format!("could not start fold hook {:?}: {error}", config.command))?; let stdin = child.stdin.take().expect("piped hook stdin"); let stdout = child.stdout.take().expect("piped hook stdout"); - let pending: Pending = Arc::new(Mutex::new(Ok(HashMap::new()))); + let pending: Pending = Arc::new(Mutex::new(Ok(DftHashMap::default()))); let reader = std::thread::spawn({ let pending = Arc::clone(&pending); move || dispatch(BufReader::new(stdout), &pending) @@ -153,7 +154,10 @@ impl Hook { Ok(Outcome::Closed(reason)) => return Err(reason), Err(RecvTimeoutError::Timeout) => { self.forget(id); - return Err(format!("fold hook timed out after {}ms", self.config.timeout_ms)); + return Err(format!( + "fold hook timed out after {}ms", + self.config.timeout_ms + )); } Err(RecvTimeoutError::Disconnected) => { unreachable!("reader drops senders only after signalling") @@ -316,7 +320,9 @@ mod tests { assert!(error.contains("timed out"), "{error}"); let error = hook("exit 0", 5000).summarize(&mut result).unwrap_err(); assert!(error.contains("exited"), "{error}"); - let error = hook("echo not json", 5000).summarize(&mut result).unwrap_err(); + let error = hook("echo not json", 5000) + .summarize(&mut result) + .unwrap_err(); assert!(error.contains("invalid reply"), "{error}"); let error = hook( r#"read -r line; echo '{"id":1,"error":"rate limited"}'; sleep 30"#, @@ -325,9 +331,12 @@ mod tests { .summarize(&mut result) .unwrap_err(); assert!(error.contains("rate limited"), "{error}"); - let error = hook(r#"read -r line; echo '{"id":1,"texts":{"7":"x"}}'; sleep 30"#, 5000) - .summarize(&mut result) - .unwrap_err(); + let error = hook( + r#"read -r line; echo '{"id":1,"texts":{"7":"x"}}'; sleep 30"#, + 5000, + ) + .summarize(&mut result) + .unwrap_err(); assert!(error.contains("unknown fold"), "{error}"); assert!(result.rhs_folds.iter().all(|fold| fold.summary.is_none())); } From 2639e301f258f89ce37c4200c16486e02498955d Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 04:27:24 -0700 Subject: [PATCH 07/19] Add a reference fold hook that writes Python-style pseudocode examples/hooks/summarize.py answers diffr fold requests concurrently through OpenRouter, defaulting to google/gemini-3.1-flash-lite with reasoning disabled, which answered a three-fold request in under two seconds. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- examples/hooks/summarize.py | 112 ++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100755 examples/hooks/summarize.py diff --git a/examples/hooks/summarize.py b/examples/hooks/summarize.py new file mode 100755 index 000000000..d9fd26bba --- /dev/null +++ b/examples/hooks/summarize.py @@ -0,0 +1,112 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# /// +"""Reference diffr fold hook: rewrite large novel folds as Python-style pseudocode. + +diffr writes one NDJSON request per file on stdin and reads one reply per request +on stdout, matched by id. Requests are answered concurrently, so replies arrive +in whatever order the model finishes; diffr routes them by id. + +Environment: + OPENROUTER_API_KEY required + DIFFR_SUMMARY_MODEL default google/gemini-3.1-flash-lite + DIFFR_SUMMARY_WORKERS concurrent requests, default 16 + DIFFR_SUMMARY_REASONING "off" (default) disables model reasoning; "required" leaves + it on for models whose endpoint refuses to disable it +""" +import json +import os +import sys +import threading +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor + +API_KEY = os.environ["OPENROUTER_API_KEY"] +MODEL = os.environ.get("DIFFR_SUMMARY_MODEL", "google/gemini-3.1-flash-lite") +WORKERS = int(os.environ.get("DIFFR_SUMMARY_WORKERS", "16")) +REASONING = os.environ.get("DIFFR_SUMMARY_REASONING", "off") +if REASONING not in ("off", "required"): + raise SystemExit(f"DIFFR_SUMMARY_REASONING must be off or required, not {REASONING!r}") +URL = "https://openrouter.ai/api/v1/chat/completions" + +SYSTEM = ( + "You rewrite regions of a source file as terse Python-style pseudocode for a diff " + "viewer that shows the pseudocode in place of the collapsed region. The user supplies " + "one numbered source file and a list of folds, each with an id and 1-based line range. " + "For each fold, write pseudocode covering only that fold's lines: keep the control flow " + "and the names that matter, drop types, error plumbing and boilerplate. Aim for about one " + "pseudocode line per five source lines, between one and eight lines per fold. Reply with a " + "JSON object mapping each fold id, as a string such as \"3\", to its pseudocode string." +) + +write_lock = threading.Lock() + + +def prompt(request): + numbered = "\n".join( + f"{number:5d} | {line}" for number, line in enumerate(request["src"].splitlines(), 1) + ) + folds = "\n".join( + f"- fold {fold['id']}: lines {fold['range']['start']['line'] + 1}-" + f"{fold['range']['end']['line'] + 1}" + for fold in request["folds"] + ) + language = request["language"] or "unknown language" + return f"File {request['path']} ({language}):\n\n{numbered}\n\nFolds:\n{folds}" + + +def complete(request): + body = { + "model": MODEL, + "messages": [ + {"role": "system", "content": SYSTEM}, + {"role": "user", "content": prompt(request)}, + ], + "temperature": 0, + "max_tokens": 160 * len(request["folds"]) + 100, + "response_format": {"type": "json_object"}, + "provider": {"sort": "latency"}, + } + if REASONING == "off": + body["reasoning"] = {"enabled": False} + http = urllib.request.Request( + URL, + data=json.dumps(body).encode(), + headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(http, timeout=60) as response: + data = json.load(response) + content = data["choices"][0]["message"]["content"] + # Models vary between "3" and "fold 3" as keys; both identify fold 3. + texts = {key.removeprefix("fold").strip(): text for key, text in json.loads(content).items()} + expected = {str(fold["id"]) for fold in request["folds"]} + unexpected = set(texts) - expected + if unexpected or not all(isinstance(text, str) for text in texts.values()): + raise ValueError(f"model returned malformed fold map: {content[:200]}") + return {key: text.strip() for key, text in texts.items() if text.strip()} + + +def answer(line): + request = json.loads(line) + try: + reply = {"id": request["id"], "texts": complete(request)} + except urllib.error.HTTPError as error: + reply = {"id": request["id"], "error": f"{MODEL}: HTTP {error.code} {error.read()[:200]!r}"} + except (OSError, ValueError, KeyError) as error: + reply = {"id": request["id"], "error": f"{MODEL}: {error}"} + with write_lock: + sys.stdout.write(json.dumps(reply) + "\n") + sys.stdout.flush() + + +def main(): + with ThreadPoolExecutor(WORKERS) as pool: + for line in sys.stdin: + if line.strip(): + pool.submit(answer, line) + + +if __name__ == "__main__": + main() From ab453d07bc25f6b3fd4ea7c99c132b4f24a8a0c6 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 04:28:06 -0700 Subject: [PATCH 08/19] Close hook stdin before stopping the process and stop joining its reader A wrapper such as uv run can be killed while its child keeps the stdout pipe open, which left diffr waiting forever on the reader thread at exit. Closing stdin first lets the hook finish on EOF; the reader now ends on its own. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- src/hook.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/hook.rs b/src/hook.rs index 68f694302..e60d3abe2 100644 --- a/src/hook.rs +++ b/src/hook.rs @@ -17,7 +17,6 @@ use std::process::{Child, ChildStdin, Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc::{sync_channel, RecvTimeoutError, SyncSender}; use std::sync::{Arc, Mutex}; -use std::thread::JoinHandle; use std::time::Duration; #[derive(Serialize)] @@ -58,10 +57,10 @@ type Pending = Arc>, String>>>; pub(crate) struct Hook { config: HookConfig, child: Mutex, - stdin: Mutex, + /// Closed before the child is stopped so hooks can finish on stdin EOF. + stdin: Mutex>, pending: Pending, next_id: AtomicU64, - reader: Mutex>>, } impl Hook { @@ -77,17 +76,18 @@ impl Hook { let stdin = child.stdin.take().expect("piped hook stdin"); let stdout = child.stdout.take().expect("piped hook stdout"); let pending: Pending = Arc::new(Mutex::new(Ok(DftHashMap::default()))); - let reader = std::thread::spawn({ + // Detached: it ends when every holder of the stdout pipe has exited, + // which a wrapper such as `uv run` can outlive being killed. + std::thread::spawn({ let pending = Arc::clone(&pending); move || dispatch(BufReader::new(stdout), &pending) }); Ok(Self { config: config.clone(), child: Mutex::new(child), - stdin: Mutex::new(stdin), + stdin: Mutex::new(Some(stdin)), pending, next_id: AtomicU64::new(1), - reader: Mutex::new(Some(reader)), }) } @@ -144,6 +144,7 @@ impl Hook { } { let mut stdin = self.stdin.lock().unwrap(); + let stdin = stdin.as_mut().expect("stdin is open until drop"); if let Err(error) = stdin.write_all(&line).and_then(|()| stdin.flush()) { self.forget(id); return Err(format!("fold hook stdin closed: {error}")); @@ -235,12 +236,10 @@ fn dispatch(stdout: BufReader, pending: &Pending) { impl Drop for Hook { fn drop(&mut self) { + self.stdin.lock().unwrap().take(); let mut child = self.child.lock().unwrap(); let _ = child.kill(); let _ = child.wait(); - if let Some(reader) = self.reader.lock().unwrap().take() { - let _ = reader.join(); - } } } From 0977af7fdb569eec64c7f141dd21ca773a7912f0 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 04:28:06 -0700 Subject: [PATCH 09/19] Document concurrent streaming and fold hooks AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- .gitignore | 1 + README.md | 4 +-- docs/cli.md | 4 ++- docs/streaming.md | 73 ++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index b46291876..93b4f20b0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ sample_files/compare.result notes.md perf_baseline_*.txt +__pycache__/ diff --git a/README.md b/README.md index 9436d2af5..8781fdfe2 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ diffr main...HEAD -- src/ # merge-base comparison diffr main HEAD --format ndjson ``` -See the [CLI reference](docs/cli.md), [streaming API](docs/streaming.md), -and [fixture viewer](examples/review/viewer/README.md). +See the [CLI reference](docs/cli.md), [streaming API](docs/streaming.md) +including fold hooks, and [fixture viewer](examples/review/viewer/README.md). The executable is `diffr`; the Cargo package remains `difftastic`. The upstream installation commands below install upstream Difftastic, not this fork. diff --git a/docs/cli.md b/docs/cli.md index 2853828e1..cfe066595 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -28,7 +28,9 @@ for changes; ordinary output exits 0. Errors exit 2. `--no-index` supports two files, implies change exit status, and does not yet support metadata options. Structural output uses the existing terminal renderer. `--format json` emits -one domain object per line. `--format ndjson` emits a buffered event stream; `--format snapshot` is the fixture text adapter. +one domain object per line. `--format ndjson` emits a buffered event stream, +diffing `--jobs N` files at once (default 16) and emitting each as it finishes; +`--format snapshot` is the fixture text adapter. `-U N` selects ordinary context padding. Matching limits, `--ignore-comments`, color, width and inline/split display remain configurable; see `--help`. diff --git a/docs/streaming.md b/docs/streaming.md index 13560a51d..22ce9dfe5 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -43,8 +43,8 @@ pipe reads can split records or contain several. | Event | Fields | Consumer action | | --- | --- | --- | -| `start` | `version: 1`, `before`, `after`, `total` | Initialize progress. | -| `file` | `file`, `diff` | Render a result. | +| `start` | `version: 1`, `before`, `after`, `total`, `files` | Lay out every file up front. | +| `file` | `file`, `diff`, optional `hook_error` | Render a result. | | `file_error` | `file`, `message` | Report failure and keep reading. | | `complete` | `succeeded`, `failed` | Mark complete, including partial failures. | @@ -58,6 +58,10 @@ The file descriptor contains nullable `old_path`, `new_path`, `class`, and `stat domain JSON: complete sources, token correspondence, folds and context hunks. There is no display layout in the response. +`files` lists every selected file descriptor in priority order. File results +arrive in completion order, not manifest order, since files are diffed +concurrently (`--jobs`, default 16). Match results to the manifest by identity. + At completion, `succeeded + failed == total`; every selected file has one result or file error. EOF without `complete` means interrupted/incomplete output. Setup failures write to stderr and exit 2 before producing any records. @@ -72,15 +76,62 @@ fail discovery. ## Computation and output Discovery and rename detection finish before `start`; syntax matching is lazy. -A producer thread consumes the file iterator. The calling thread serializes, -writes and flushes each event. A bounded queue holds one ready event, allowing -computation to overlap slow writes without collecting the entire comparison. -When the queue is full, the producer waits. This bounds the number of in-flight -files, not their individual size. - -Closing stdout stops production when its next send fails; an already running file -may finish. Terminate the process to cancel immediately. The CLI also retains its -normal SIGPIPE behavior on Unix. +A pool of `--jobs` workers pulls files from the iterator: each worker reads the +next file's sources under a lock, then diffs them while other workers pull +further files. The calling thread serializes, writes and flushes each event. +A bounded queue holds one ready event, so computation overlaps slow writes +without collecting the entire comparison. In-flight files are bounded by the +pool size, not their individual size. `--jobs 1` restores priority order. + +Closing stdout stops production once the files in flight finish. Terminate the +process to cancel immediately. The CLI also retains its normal SIGPIPE behavior +on Unix. + +## Fold hooks + +A trusted hook subprocess can replace fold placeholders with richer text, such +as pseudocode, before each `file` event is emitted: + +```toml +[folds.hook] +command = ["uv", "run", "--script", "examples/hooks/summarize.py"] +tags = ["body"] # optional; any listed tag qualifies. Omit to send every fold. +min_lines = 12 # optional; default 0 +timeout_ms = 5000 # optional; per request +``` + +The command starts once per invocation, in the workspace directory, with the +caller's environment. Only novel folds on the after side qualify: bodies that +exist in the after source with no counterpart in the before source. Files with +no qualifying fold never reach the hook. Streaming is the only output mode that +runs hooks; the terminal frontend streams, so it does too. + +Requests are one JSON line per file on the hook's stdin, and replies are one JSON +line per request on its stdout, matched by `id` and accepted in any order. The +worker diffing a file blocks on that file's reply; other workers keep going, so +a hook must answer requests concurrently rather than one at a time. + +```jsonc +// diffr -> hook +{"id": 7, "path": "src/auth.py", "language": "Python", "src": "", + "folds": [{"id": 0, "range": {"start": {"line": 40, "byte_column": 0}, "end": {"line": 88, "byte_column": 1}}, + "tags": ["body"], "placeholder": "Body"}]} +// hook -> diffr +{"id": 7, "texts": {"0": "def refresh_token(session):\n ..."}} +{"id": 8, "error": "rate limited"} +``` + +`language` is null for plain text. A fold `id` indexes `rhs_folds` in that file's +`diff`; the matching fold gains a non-null `summary` while `placeholder` is +unchanged. Folds missing from `texts` keep a null `summary`. An `error` reply, a +timeout, an unknown fold id, a malformed line, or hook exit leaves every summary +in that file null and adds `hook_error` to its `file` event. A malformed line or +exit also fails every later request, since ids can no longer be trusted. Hook +stderr passes through to diffr's stderr. Failing to start the command exits 2 +before `start`. + +`examples/hooks/summarize.py` is a reference hook that asks a model through +OpenRouter for Python-style pseudocode, answering up to 16 files at once. ## Fixture viewer From 7d041129bb65ea9ea6bdb2a7ebf8c59825673319 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 12:14:25 -0700 Subject: [PATCH 10/19] Point the reference hook at Gemini 3.8 Flash directly OpenRouter refuses to disable reasoning for Gemini 3.8, so the hook calls the Gemini API with thinking budget zero and a fixed response schema. A three-fold request answers in about two seconds. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- docs/streaming.md | 5 +-- examples/hooks/summarize.py | 66 +++++++++++++++++++------------------ 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/docs/streaming.md b/docs/streaming.md index 22ce9dfe5..94f23d5ff 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -130,8 +130,9 @@ exit also fails every later request, since ids can no longer be trusted. Hook stderr passes through to diffr's stderr. Failing to start the command exits 2 before `start`. -`examples/hooks/summarize.py` is a reference hook that asks a model through -OpenRouter for Python-style pseudocode, answering up to 16 files at once. +`examples/hooks/summarize.py` is a reference hook that asks Gemini 3.8 Flash, +with thinking disabled, for Python-style pseudocode. It needs `GOOGLE_API_KEY` +and answers up to 16 files at once. ## Fixture viewer diff --git a/examples/hooks/summarize.py b/examples/hooks/summarize.py index d9fd26bba..22902ff56 100755 --- a/examples/hooks/summarize.py +++ b/examples/hooks/summarize.py @@ -9,11 +9,9 @@ in whatever order the model finishes; diffr routes them by id. Environment: - OPENROUTER_API_KEY required - DIFFR_SUMMARY_MODEL default google/gemini-3.1-flash-lite + GOOGLE_API_KEY required + DIFFR_SUMMARY_MODEL default gemini-3.8-flash DIFFR_SUMMARY_WORKERS concurrent requests, default 16 - DIFFR_SUMMARY_REASONING "off" (default) disables model reasoning; "required" leaves - it on for models whose endpoint refuses to disable it """ import json import os @@ -23,13 +21,10 @@ import urllib.request from concurrent.futures import ThreadPoolExecutor -API_KEY = os.environ["OPENROUTER_API_KEY"] -MODEL = os.environ.get("DIFFR_SUMMARY_MODEL", "google/gemini-3.1-flash-lite") +API_KEY = os.environ["GOOGLE_API_KEY"] +MODEL = os.environ.get("DIFFR_SUMMARY_MODEL", "gemini-3.8-flash") WORKERS = int(os.environ.get("DIFFR_SUMMARY_WORKERS", "16")) -REASONING = os.environ.get("DIFFR_SUMMARY_REASONING", "off") -if REASONING not in ("off", "required"): - raise SystemExit(f"DIFFR_SUMMARY_REASONING must be off or required, not {REASONING!r}") -URL = "https://openrouter.ai/api/v1/chat/completions" +URL = f"https://generativelanguage.googleapis.com/v1beta/models/{MODEL}:generateContent" SYSTEM = ( "You rewrite regions of a source file as terse Python-style pseudocode for a diff " @@ -37,9 +32,17 @@ "one numbered source file and a list of folds, each with an id and 1-based line range. " "For each fold, write pseudocode covering only that fold's lines: keep the control flow " "and the names that matter, drop types, error plumbing and boilerplate. Aim for about one " - "pseudocode line per five source lines, between one and eight lines per fold. Reply with a " - "JSON object mapping each fold id, as a string such as \"3\", to its pseudocode string." + "pseudocode line per five source lines, between one and eight lines per fold. Reply with " + "one {id, pseudocode} object per fold." ) +SCHEMA = { + "type": "ARRAY", + "items": { + "type": "OBJECT", + "properties": {"id": {"type": "INTEGER"}, "pseudocode": {"type": "STRING"}}, + "required": ["id", "pseudocode"], + }, +} write_lock = threading.Lock() @@ -59,33 +62,32 @@ def prompt(request): def complete(request): body = { - "model": MODEL, - "messages": [ - {"role": "system", "content": SYSTEM}, - {"role": "user", "content": prompt(request)}, - ], - "temperature": 0, - "max_tokens": 160 * len(request["folds"]) + 100, - "response_format": {"type": "json_object"}, - "provider": {"sort": "latency"}, + "systemInstruction": {"parts": [{"text": SYSTEM}]}, + "contents": [{"role": "user", "parts": [{"text": prompt(request)}]}], + "generationConfig": { + "temperature": 0, + "maxOutputTokens": 160 * len(request["folds"]) + 100, + "thinkingConfig": {"thinkingBudget": 0}, + "responseMimeType": "application/json", + "responseSchema": SCHEMA, + }, } - if REASONING == "off": - body["reasoning"] = {"enabled": False} http = urllib.request.Request( URL, data=json.dumps(body).encode(), - headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"}, + headers={"x-goog-api-key": API_KEY, "Content-Type": "application/json"}, ) with urllib.request.urlopen(http, timeout=60) as response: data = json.load(response) - content = data["choices"][0]["message"]["content"] - # Models vary between "3" and "fold 3" as keys; both identify fold 3. - texts = {key.removeprefix("fold").strip(): text for key, text in json.loads(content).items()} - expected = {str(fold["id"]) for fold in request["folds"]} - unexpected = set(texts) - expected - if unexpected or not all(isinstance(text, str) for text in texts.values()): - raise ValueError(f"model returned malformed fold map: {content[:200]}") - return {key: text.strip() for key, text in texts.items() if text.strip()} + content = data["candidates"][0]["content"]["parts"][-1]["text"] + expected = {fold["id"] for fold in request["folds"]} + texts = {} + for item in json.loads(content): + if item["id"] not in expected: + raise ValueError(f"model answered for unknown fold {item['id']}: {content[:200]}") + if item["pseudocode"].strip(): + texts[str(item["id"])] = item["pseudocode"].strip() + return texts def answer(line): From f3c0f64c5fe8eac2d0b1bab697e3c6db7fe27c1f Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 12:22:20 -0700 Subject: [PATCH 11/19] Run fold hooks in the config file's directory Relative command paths now resolve against diffr.toml or the --config file rather than the diffed repository, so one hook config works from any checkout. The repository root is passed as DIFFR_WORKSPACE. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- docs/streaming.md | 7 +++++-- src/config.rs | 16 ++++++++++++++-- src/hook.rs | 4 +++- tests/streaming/check.py | 12 ++++++++++++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/streaming.md b/docs/streaming.md index 94f23d5ff..fd4f2d1bb 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -100,8 +100,11 @@ min_lines = 12 # optional; default 0 timeout_ms = 5000 # optional; per request ``` -The command starts once per invocation, in the workspace directory, with the -caller's environment. Only novel folds on the after side qualify: bodies that +The command starts once per invocation with the caller's environment. It runs +in the directory containing the config file, so relative paths in `command` +resolve against the config wherever it lives, including one given by +`--config` outside the repository. `DIFFR_WORKSPACE` carries the diffed +repository's root. Only novel folds on the after side qualify: bodies that exist in the after source with no counterpart in the before source. Files with no qualifying fold never reach the hook. Streaming is the only output mode that runs hooks; the terminal frontend streams, so it does too. diff --git a/src/config.rs b/src/config.rs index 6953c21ef..043a66658 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,7 +5,7 @@ use crate::parse::{guess_language::Language, tree_sitter_parser}; use query::AnnotationQuery; use serde::Deserialize; use std::collections::BTreeMap; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, OnceLock}; use strum::IntoEnumIterator; @@ -26,6 +26,9 @@ pub(crate) struct FoldsConfig { #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct HookConfig { + /// Relative command paths resolve against the config file, wherever it lives. + #[serde(skip)] + pub(crate) dir: PathBuf, pub(crate) command: Vec, /// None sends every tagged fold; otherwise a fold needs one of these tags. #[serde(default)] @@ -91,7 +94,16 @@ impl Config { .map(Path::to_path_buf) .unwrap_or_else(|| workspace.join("diffr.toml")); match std::fs::read_to_string(&path) { - Ok(source) => Self::from_toml(&source), + Ok(source) => { + let mut config = Self::from_toml(&source)?; + if let Some(hook) = &mut config.folds.hook { + hook.dir = path + .parent() + .expect("config file has a parent") + .to_path_buf(); + } + Ok(config) + } Err(error) if explicit.is_none() && error.kind() == std::io::ErrorKind::NotFound => { Ok(Self::default()) } diff --git a/src/hook.rs b/src/hook.rs index e60d3abe2..f48583536 100644 --- a/src/hook.rs +++ b/src/hook.rs @@ -67,7 +67,8 @@ impl Hook { pub(crate) fn spawn(config: &HookConfig, workspace: &Path) -> crate::git::Result { let mut child = Command::new(&config.command[0]) .args(&config.command[1..]) - .current_dir(workspace) + .current_dir(&config.dir) + .env("DIFFR_WORKSPACE", workspace) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::inherit()) @@ -250,6 +251,7 @@ mod tests { fn hook(script: &str, timeout_ms: u64) -> Hook { let config = HookConfig { + dir: ".".into(), command: vec!["sh".into(), "-c".into(), script.into()], tags: Some(vec!["body".into()]), min_lines: 2, diff --git a/tests/streaming/check.py b/tests/streaming/check.py index 9f4869853..0346d3ccb 100644 --- a/tests/streaming/check.py +++ b/tests/streaming/check.py @@ -154,6 +154,18 @@ def stream(repo, *args, code=0): assert all(f["summary"] is None for f in events["small.py"]["diff"]["rhs_folds"]) (repo / "diffr.toml").write_text("[folds.hook]\ncommand = ['./missing-hook']\n") assert cli(repo, "--format", "ndjson", base, head).returncode == 2 + # Relative hook paths resolve against the config file, not the repository. + with tempfile.TemporaryDirectory(prefix="diffr-hook-config-") as elsewhere: + (Path(elsewhere) / "hook.py").write_text( + "import json, os, sys\n" + "assert os.environ['DIFFR_WORKSPACE'] == sys.argv[1], os.environ['DIFFR_WORKSPACE']\n" + "for line in sys.stdin:\n" + " request = json.loads(line)\n" + " print(json.dumps({'id': request['id'], 'texts': {'0': os.getcwd()}}), flush=True)\n") + (Path(elsewhere) / "hook.toml").write_text( + f"[folds.hook]\ncommand = [{json.dumps(sys.executable)}, 'hook.py', {json.dumps(str(repo.resolve()) + os.sep)}]\ntags = ['body']\n") + events = stream(repo, base, head, "--config", str(Path(elsewhere) / "hook.toml"), "--", "good.py") + assert events[1]["diff"]["rhs_folds"][0]["summary"] == str(Path(elsewhere).resolve()) # Closing the pipe while a multi-file producer is active must not leave it # blocked forever on a full queue. Unix CLI output retains normal SIGPIPE behavior. From 352df3ab5f923bd4e082c4aa4b531c91d0e9b16c Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 12:29:47 -0700 Subject: [PATCH 12/19] Answer hook requests on an asyncio loop with httpx Replace the thread pool and urllib with a single event loop, a shared AsyncClient and a semaphore; replies are written whole between awaits. Three concurrent requests answer in about two seconds. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a08a73-9b4c-7bd1-bb9c-e2efe577c764 Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- docs/streaming.md | 3 +- examples/hooks/summarize.py | 70 ++++++++++++++++++++----------------- 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/docs/streaming.md b/docs/streaming.md index fd4f2d1bb..953ad3cf2 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -135,7 +135,8 @@ before `start`. `examples/hooks/summarize.py` is a reference hook that asks Gemini 3.8 Flash, with thinking disabled, for Python-style pseudocode. It needs `GOOGLE_API_KEY` -and answers up to 16 files at once. +and answers up to 16 files at once on one asyncio loop with a shared httpx +client, which `uv run --script` installs on first use. ## Fixture viewer diff --git a/examples/hooks/summarize.py b/examples/hooks/summarize.py index 22902ff56..2447b5bee 100755 --- a/examples/hooks/summarize.py +++ b/examples/hooks/summarize.py @@ -1,25 +1,26 @@ #!/usr/bin/env -S uv run --script # /// script # requires-python = ">=3.11" +# dependencies = ["httpx>=0.27"] # /// """Reference diffr fold hook: rewrite large novel folds as Python-style pseudocode. diffr writes one NDJSON request per file on stdin and reads one reply per request -on stdout, matched by id. Requests are answered concurrently, so replies arrive -in whatever order the model finishes; diffr routes them by id. +on stdout, matched by id. Requests are answered concurrently on one event loop +with a shared HTTP client, so replies arrive in whatever order the model +finishes; diffr routes them by id. Environment: GOOGLE_API_KEY required DIFFR_SUMMARY_MODEL default gemini-3.8-flash DIFFR_SUMMARY_WORKERS concurrent requests, default 16 """ +import asyncio import json import os import sys -import threading -import urllib.error -import urllib.request -from concurrent.futures import ThreadPoolExecutor + +import httpx API_KEY = os.environ["GOOGLE_API_KEY"] MODEL = os.environ.get("DIFFR_SUMMARY_MODEL", "gemini-3.8-flash") @@ -44,8 +45,6 @@ }, } -write_lock = threading.Lock() - def prompt(request): numbered = "\n".join( @@ -60,7 +59,7 @@ def prompt(request): return f"File {request['path']} ({language}):\n\n{numbered}\n\nFolds:\n{folds}" -def complete(request): +async def complete(client, request): body = { "systemInstruction": {"parts": [{"text": SYSTEM}]}, "contents": [{"role": "user", "parts": [{"text": prompt(request)}]}], @@ -72,14 +71,9 @@ def complete(request): "responseSchema": SCHEMA, }, } - http = urllib.request.Request( - URL, - data=json.dumps(body).encode(), - headers={"x-goog-api-key": API_KEY, "Content-Type": "application/json"}, - ) - with urllib.request.urlopen(http, timeout=60) as response: - data = json.load(response) - content = data["candidates"][0]["content"]["parts"][-1]["text"] + response = await client.post(URL, json=body) + response.raise_for_status() + content = response.json()["candidates"][0]["content"]["parts"][-1]["text"] expected = {fold["id"] for fold in request["folds"]} texts = {} for item in json.loads(content): @@ -90,25 +84,35 @@ def complete(request): return texts -def answer(line): +async def answer(client, limit, line): request = json.loads(line) - try: - reply = {"id": request["id"], "texts": complete(request)} - except urllib.error.HTTPError as error: - reply = {"id": request["id"], "error": f"{MODEL}: HTTP {error.code} {error.read()[:200]!r}"} - except (OSError, ValueError, KeyError) as error: - reply = {"id": request["id"], "error": f"{MODEL}: {error}"} - with write_lock: - sys.stdout.write(json.dumps(reply) + "\n") - sys.stdout.flush() + async with limit: + try: + reply = {"id": request["id"], "texts": await complete(client, request)} + except httpx.HTTPStatusError as error: + reply = { + "id": request["id"], + "error": f"{MODEL}: HTTP {error.response.status_code} {error.response.text[:200]}", + } + except (httpx.HTTPError, ValueError, KeyError) as error: + reply = {"id": request["id"], "error": f"{MODEL}: {error}"} + # Single-threaded: a whole line is written between awaits, never interleaved. + sys.stdout.write(json.dumps(reply) + "\n") + sys.stdout.flush() -def main(): - with ThreadPoolExecutor(WORKERS) as pool: - for line in sys.stdin: - if line.strip(): - pool.submit(answer, line) +async def main(): + reader = asyncio.StreamReader() + await asyncio.get_running_loop().connect_read_pipe( + lambda: asyncio.StreamReaderProtocol(reader), sys.stdin + ) + limit = asyncio.Semaphore(WORKERS) + async with httpx.AsyncClient(headers={"x-goog-api-key": API_KEY}, timeout=60) as client: + async with asyncio.TaskGroup() as tasks: + while line := await reader.readline(): + if line.strip(): + tasks.create_task(answer(client, limit, line)) if __name__ == "__main__": - main() + asyncio.run(main()) From 3043e61bf716aaa544339ead5e5f90032a627ddc Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 14:14:26 -0700 Subject: [PATCH 13/19] Ignore .env so local credentials never get committed AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 93b4f20b0..9cf547af8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ sample_files/compare.result notes.md perf_baseline_*.txt __pycache__/ +.env From 0aa25b38c0755ddb61062ce80d4c629630428acf Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 14:44:14 -0700 Subject: [PATCH 14/19] Add jsonrpsee and tokio for the fold hook client Dependency-only change ahead of moving hook communication to JSON-RPC. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- Cargo.lock | 904 ++++++++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + 2 files changed, 891 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d280c07e..1cba9554e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,17 +82,40 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" -version = "2.5.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bstr" @@ -111,6 +134,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.2.54" @@ -123,6 +152,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.0" @@ -163,6 +198,32 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -215,6 +276,7 @@ dependencies = [ "humansize", "ignore", "imara-diff", + "jsonrpsee", "lazy_static", "libc", "line-numbers", @@ -235,6 +297,7 @@ dependencies = [ "strum", "terminal_size", "tikv-jemallocator", + "tokio", "toml", "tree-sitter", "tree-sitter-ada", @@ -384,6 +447,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -405,6 +474,57 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "git2" version = "0.20.4" @@ -437,6 +557,25 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -469,6 +608,45 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "humansize" version = "2.1.3" @@ -484,6 +662,63 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "log", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -645,6 +880,50 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.106", +] + [[package]] name = "jobserver" version = "0.1.32" @@ -654,6 +933,89 @@ dependencies = [ "libc", ] +[[package]] +name = "jsonrpsee" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3f48dc3e6b8bd21e15436c1ddd0bc22a6a54e8ec46fedd6adf3425f396ec6a" +dependencies = [ + "jsonrpsee-core", + "jsonrpsee-http-client", + "jsonrpsee-proc-macros", + "jsonrpsee-types", + "tracing", +] + +[[package]] +name = "jsonrpsee-core" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "316c96719901f05d1137f19ba598b5fe9c9bc39f4335f67f6be8613921946480" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "jsonrpsee-types", + "pin-project", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "jsonrpsee-http-client" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790bedefcec85321e007ff3af84b4e417540d5c87b3c9779b9e247d1bcc3dab8" +dependencies = [ + "base64", + "http-body", + "hyper", + "hyper-rustls", + "hyper-util", + "jsonrpsee-core", + "jsonrpsee-types", + "rustls", + "rustls-platform-verifier", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tower", + "url", +] + +[[package]] +name = "jsonrpsee-proc-macros" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da3f8ab5ce1bb124b6d082e62dffe997578ceaf0aeb9f3174a214589dc00f07" +dependencies = [ + "heck", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "jsonrpsee-types" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc88ff4688e43cc3fa9883a8a95c6fa27aa2e76c96e610b737b6554d650d7fd5" +dependencies = [ + "http", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -726,6 +1088,17 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "nom" version = "8.0.0" @@ -750,6 +1123,18 @@ dependencies = [ "autocfg", ] +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "owo-colors" version = "4.2.3" @@ -773,6 +1158,32 @@ dependencies = [ "indexmap", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.34" @@ -838,6 +1249,15 @@ dependencies = [ "log", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.14+spec-1.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.101" @@ -911,6 +1331,20 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-hash" version = "2.0.0" @@ -930,6 +1364,80 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.23.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs 0.26.11", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.17" @@ -945,6 +1453,38 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.228" @@ -1004,12 +1544,28 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1049,6 +1605,12 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.106" @@ -1071,6 +1633,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.2" @@ -1107,6 +1675,46 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "tikv-jemalloc-sys" version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" @@ -1137,6 +1745,44 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.8.23" @@ -1145,8 +1791,8 @@ checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", - "toml_datetime", - "toml_edit", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", ] [[package]] @@ -1158,6 +1804,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -1167,8 +1822,86 @@ dependencies = [ "indexmap", "serde", "serde_spanned", - "toml_datetime", - "winnow", + "toml_datetime 0.6.11", + "winnow 0.7.15", +] + +[[package]] +name = "toml_edit" +version = "0.25.14+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2195eec204e2764644a4ea619704f9fbe5e0673038eded55ad9956f24fca0cc" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", ] [[package]] @@ -1772,6 +2505,12 @@ dependencies = [ "petgraph", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "ts-parser-perl" version = "1.2.1" @@ -1800,6 +2539,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -1855,6 +2600,39 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "webpki-root-certs" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" +dependencies = [ + "webpki-root-certs 1.0.9", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi-util" version = "0.1.9" @@ -1864,13 +2642,28 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1879,7 +2672,31 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -1888,28 +2705,46 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1922,24 +2757,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -1955,6 +2814,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "writeable" version = "0.6.4" @@ -2011,6 +2879,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 97176af68..c1605e89a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,8 @@ ignore = ">= 0.4, < 0.4.30" owo-colors = "4.2.3" imara-diff = { version = "0.2.0", default-features = false } rayon = "1.7.0" +jsonrpsee = { version = "0.26", default-features = false, features = ["http-client", "macros"] } +tokio = { version = "1", default-features = false, features = ["rt-multi-thread", "time"] } tree_magic_mini = "3.2.0" bumpalo = "3.18.0" From f035266dd5af7d4319cea31b4e4b1008b60f88f0 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 14:46:56 -0700 Subject: [PATCH 15/19] Call fold hooks over JSON-RPC 2.0 on loopback HTTP The hook is now a server: diffr reserves a port, passes it as DIFFR_HOOK_PORT, waits for it to listen, then calls a typed jsonrpsee client on a small tokio runtime. Rayon workers block on their own call while in-flight requests multiplex on the client. The hand-written framing, id map, reader thread and stdin lock are gone. Hook stdout is discarded so it cannot disturb the event stream; hooks log to stderr. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- src/config.rs | 17 +- src/hook.rs | 319 +++++++++++++++++--------------------- tests/hooks/rpc_server.py | 57 +++++++ tests/streaming/check.py | 34 ++-- 4 files changed, 225 insertions(+), 202 deletions(-) create mode 100644 tests/hooks/rpc_server.py diff --git a/src/config.rs b/src/config.rs index 043a66658..fe29ee81c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -35,14 +35,22 @@ pub(crate) struct HookConfig { pub(crate) tags: Option>, #[serde(default)] pub(crate) min_lines: usize, + /// Per-call limit once the hook is listening. #[serde(default = "default_timeout_ms")] pub(crate) timeout_ms: u64, + /// How long the hook may take to start listening on its port. + #[serde(default = "default_startup_timeout_ms")] + pub(crate) startup_timeout_ms: u64, } fn default_timeout_ms() -> u64 { 5000 } +fn default_startup_timeout_ms() -> u64 { + 30_000 +} + #[derive(Default, Deserialize)] #[serde(default, deny_unknown_fields)] pub(crate) struct LanguageConfig { @@ -120,8 +128,8 @@ impl Config { if hook.command.is_empty() { return Err(ConfigError("folds.hook.command must not be empty".into())); } - if hook.timeout_ms == 0 { - return Err(ConfigError("folds.hook.timeout_ms must be positive".into())); + if hook.timeout_ms == 0 || hook.startup_timeout_ms == 0 { + return Err(ConfigError("folds.hook timeouts must be positive".into())); } } let defaults = Self::from_toml(include_str!("config/defaults.toml"))?; @@ -291,7 +299,10 @@ mod tests { let hook = params.hook.unwrap(); assert_eq!(hook.command, ["uv", "run", "summarize.py"]); assert_eq!(hook.tags.as_deref(), Some(&["body".to_owned()][..])); - assert_eq!((hook.min_lines, hook.timeout_ms), (30, 5000)); + assert_eq!( + (hook.min_lines, hook.timeout_ms, hook.startup_timeout_ms), + (30, 5000, 30_000) + ); assert!(Config::from_toml("") .unwrap() .compile() diff --git a/src/hook.rs b/src/hook.rs index f48583536..742caca34 100644 --- a/src/hook.rs +++ b/src/hook.rs @@ -1,94 +1,91 @@ -//! Trusted fold-summary subprocess: spawned once per session, NDJSON both ways. +//! Trusted fold-summary hook: a JSON-RPC 2.0 server that diffr starts once per +//! session and calls over loopback HTTP. //! -//! Each request carries one file and its large novel folds on the after side. -//! The hook replies in any order, keyed by request id; callers block on their -//! own reply so files still stream out as each worker finishes. +//! The hook receives its port in `DIFFR_HOOK_PORT` and the diffed repository in +//! `DIFFR_WORKSPACE`. Each call carries one file and its large novel folds on +//! the after side. Workers block on their own call while the client multiplexes +//! every in-flight request on a small tokio runtime, so files still stream out +//! as each worker finishes. use crate::config::HookConfig; -use crate::hash::DftHashMap; use crate::parse::folds::FoldMatch; use crate::review::wire; use crate::summary::{DiffResult, FileContent, FileFormat}; -use serde::{Deserialize, Serialize}; +use jsonrpsee::core::ClientError; +use jsonrpsee::http_client::{HttpClient, HttpClientBuilder}; +use jsonrpsee::proc_macros::rpc; +use serde::Serialize; use serde_json::Value; use std::collections::BTreeMap; -use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; use std::path::Path; -use std::process::{Child, ChildStdin, Command, Stdio}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::mpsc::{sync_channel, RecvTimeoutError, SyncSender}; -use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::process::{Child, Command, Stdio}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; #[derive(Serialize)] -struct Request<'a> { - id: u64, - path: &'a str, - language: Option<&'static str>, - src: &'a str, - folds: Vec>, -} - -#[derive(Serialize)] -struct RequestFold<'a> { +pub(crate) struct RequestFold { /// Index into the file's rhs folds. id: usize, range: Value, - tags: &'a [String], - placeholder: &'a str, -} - -#[derive(Deserialize)] -struct Reply { - id: u64, - #[serde(default)] - texts: BTreeMap, - #[serde(default)] - error: Option, + tags: Vec, + placeholder: String, } -enum Outcome { - Reply(Reply), - /// The hook can no longer answer anything. - Closed(String), +/// The interface every hook implements. Params are sent by name; the result +/// maps fold ids, as strings, to replacement text. +#[rpc(client)] +trait FoldHook { + #[method(name = "summarize", param_kind = map)] + async fn summarize( + &self, + path: String, + language: Option, + src: String, + folds: Vec, + ) -> jsonrpsee::core::RpcResult>; } -type Pending = Arc>, String>>>; - pub(crate) struct Hook { config: HookConfig, child: Mutex, - /// Closed before the child is stopped so hooks can finish on stdin EOF. - stdin: Mutex>, - pending: Pending, - next_id: AtomicU64, + runtime: tokio::runtime::Runtime, + client: HttpClient, } impl Hook { pub(crate) fn spawn(config: &HookConfig, workspace: &Path) -> crate::git::Result { + let port = free_port()?; let mut child = Command::new(&config.command[0]) .args(&config.command[1..]) .current_dir(&config.dir) + .env("DIFFR_HOOK_PORT", port.to_string()) .env("DIFFR_WORKSPACE", workspace) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) + .stdin(Stdio::null()) + // Stdout belongs to diffr's own stream; hooks log to stderr. + .stdout(Stdio::null()) .stderr(Stdio::inherit()) .spawn() .map_err(|error| format!("could not start fold hook {:?}: {error}", config.command))?; - let stdin = child.stdin.take().expect("piped hook stdin"); - let stdout = child.stdout.take().expect("piped hook stdout"); - let pending: Pending = Arc::new(Mutex::new(Ok(DftHashMap::default()))); - // Detached: it ends when every holder of the stdout pipe has exited, - // which a wrapper such as `uv run` can outlive being killed. - std::thread::spawn({ - let pending = Arc::clone(&pending); - move || dispatch(BufReader::new(stdout), &pending) - }); + if let Err(error) = await_listening(&mut child, port, config.startup_timeout_ms) { + let _ = child.kill(); + let _ = child.wait(); + return Err(error.into()); + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_name("diffr-hook-client") + .enable_all() + .build()?; + let client = runtime.block_on(async { + HttpClientBuilder::default() + .request_timeout(Duration::from_millis(config.timeout_ms)) + .build(format!("http://127.0.0.1:{port}")) + })?; Ok(Self { config: config.clone(), child: Mutex::new(child), - stdin: Mutex::new(Some(stdin)), - pending, - next_id: AtomicU64::new(1), + runtime, + client, }) } @@ -110,65 +107,39 @@ impl Hook { }; let language = match &diff.file_format { FileFormat::SupportedLanguage(language) => { - Some(crate::parse::guess_language::language_name(*language)) + Some(crate::parse::guess_language::language_name(*language).to_owned()) } _ => None, }; - let id = self.next_id.fetch_add(1, Ordering::Relaxed); - let request = Request { - id, - path: &diff.display_path, - language, - src, - folds: selected - .iter() - .map(|&index| { - let fold = &diff.rhs_folds[index]; - RequestFold { - id: index, - range: wire::range(&fold.range), - tags: &fold.tags, - placeholder: &fold.placeholder, - } - }) - .collect(), - }; - let mut line = serde_json::to_vec(&request).map_err(|error| error.to_string())?; - line.push(b'\n'); - let (sender, receiver) = sync_channel(1); - { - let mut pending = self.pending.lock().unwrap(); - match pending.as_mut() { - Ok(pending) => pending.insert(id, sender), - Err(reason) => return Err(reason.clone()), - }; - } - { - let mut stdin = self.stdin.lock().unwrap(); - let stdin = stdin.as_mut().expect("stdin is open until drop"); - if let Err(error) = stdin.write_all(&line).and_then(|()| stdin.flush()) { - self.forget(id); - return Err(format!("fold hook stdin closed: {error}")); - } - } - let reply = match receiver.recv_timeout(Duration::from_millis(self.config.timeout_ms)) { - Ok(Outcome::Reply(reply)) => reply, - Ok(Outcome::Closed(reason)) => return Err(reason), - Err(RecvTimeoutError::Timeout) => { - self.forget(id); - return Err(format!( - "fold hook timed out after {}ms", - self.config.timeout_ms - )); - } - Err(RecvTimeoutError::Disconnected) => { - unreachable!("reader drops senders only after signalling") - } - }; - if let Some(error) = reply.error { - return Err(format!("fold hook reported: {error}")); - } - for (key, text) in reply.texts { + let folds = selected + .iter() + .map(|&index| { + let fold = &diff.rhs_folds[index]; + RequestFold { + id: index, + range: wire::range(&fold.range), + tags: fold.tags.clone(), + placeholder: fold.placeholder.clone(), + } + }) + .collect(); + let texts = self + .runtime + .block_on(FoldHookClient::summarize( + &self.client, + diff.display_path.clone(), + language, + src.clone(), + folds, + )) + .map_err(|error| match error { + ClientError::Call(error) => format!("fold hook reported: {}", error.message()), + ClientError::RequestTimeout => { + format!("fold hook timed out after {}ms", self.config.timeout_ms) + } + other => format!("fold hook: {other}"), + })?; + for (key, text) in texts { let index: usize = key .parse() .ok() @@ -192,52 +163,36 @@ impl Hook { None => true, } } +} - fn forget(&self, id: u64) { - if let Ok(pending) = self.pending.lock().unwrap().as_mut() { - pending.remove(&id); - } - } +/// Reserve a loopback port for the hook. The listener is released before the +/// hook starts, which is the usual small race on a single machine. +fn free_port() -> std::io::Result { + Ok(TcpListener::bind(("127.0.0.1", 0))?.local_addr()?.port()) } -/// Route replies to their waiting request; a malformed line or EOF fails every -/// current and future request, since ids can no longer be trusted. -fn dispatch(stdout: BufReader, pending: &Pending) { - let close = |reason: String| { - let mut pending = pending.lock().unwrap(); - if let Ok(waiting) = std::mem::replace(&mut *pending, Err(reason.clone())) { - for (_, sender) in waiting { - let _ = sender.send(Outcome::Closed(reason.clone())); - } +/// Poll until the hook accepts connections, or fail early if it exits. +fn await_listening(child: &mut Child, port: u16, startup_timeout_ms: u64) -> Result<(), String> { + let deadline = Instant::now() + Duration::from_millis(startup_timeout_ms); + let address = std::net::SocketAddr::from(([127, 0, 0, 1], port)); + loop { + if TcpStream::connect_timeout(&address, Duration::from_millis(100)).is_ok() { + return Ok(()); } - }; - for line in stdout.lines() { - let line = match line { - Ok(line) => line, - Err(error) => return close(format!("fold hook stdout unreadable: {error}")), - }; - if line.trim().is_empty() { - continue; + if let Some(status) = child.try_wait().map_err(|error| error.to_string())? { + return Err(format!("fold hook exited during startup with {status}")); } - let reply: Reply = match serde_json::from_str(&line) { - Ok(reply) => reply, - Err(error) => return close(format!("fold hook wrote an invalid reply: {error}")), - }; - let sender = match pending.lock().unwrap().as_mut() { - Ok(waiting) => waiting.remove(&reply.id), - Err(_) => return, - }; - // A request that already timed out has no receiver; drop the late reply. - if let Some(sender) = sender { - let _ = sender.send(Outcome::Reply(reply)); + if Instant::now() >= deadline { + return Err(format!( + "fold hook did not listen on port {port} within {startup_timeout_ms}ms" + )); } + std::thread::sleep(Duration::from_millis(20)); } - close("fold hook exited".into()); } impl Drop for Hook { fn drop(&mut self) { - self.stdin.lock().unwrap().take(); let mut child = self.child.lock().unwrap(); let _ = child.kill(); let _ = child.wait(); @@ -248,16 +203,22 @@ impl Drop for Hook { mod tests { use super::*; use crate::config::Config; + use std::sync::Arc; - fn hook(script: &str, timeout_ms: u64) -> Hook { + fn hook(mode: &str, timeout_ms: u64) -> crate::git::Result { let config = HookConfig { - dir: ".".into(), - command: vec!["sh".into(), "-c".into(), script.into()], + dir: env!("CARGO_MANIFEST_DIR").into(), + command: vec![ + "python3".into(), + "tests/hooks/rpc_server.py".into(), + mode.into(), + ], tags: Some(vec!["body".into()]), min_lines: 2, timeout_ms, + startup_timeout_ms: 10_000, }; - Hook::spawn(&config, Path::new(".")).unwrap() + Hook::spawn(&config, Path::new(".")) } fn diff(rhs: &str) -> DiffResult { @@ -273,12 +234,10 @@ mod tests { } const LARGE: &str = "def f():\n a()\n b()\n c()\n\ndef g():\n d()\n"; - /// Echo each request id back with a summary for fold 0. - const ECHO: &str = r#"while IFS= read -r line; do id=$(printf '%s' "$line" | sed -E 's/^\{"id":([0-9]+).*/\1/'); printf '{"id":%s,"texts":{"0":"summary of f"}}\n' "$id"; done"#; #[test] fn summaries_land_on_selected_folds_only() { - let hook = hook(ECHO, 5000); + let hook = hook("first", 5000).unwrap(); let mut result = diff(LARGE); assert_eq!(result.rhs_folds.len(), 2); hook.summarize(&mut result).unwrap(); @@ -288,8 +247,8 @@ mod tests { } #[test] - fn concurrent_requests_are_routed_by_id() { - let hook = Arc::new(hook(ECHO, 5000)); + fn concurrent_calls_share_one_client() { + let hook = Arc::new(hook("echo", 5000).unwrap()); let workers: Vec<_> = (0..8) .map(|_| { let hook = Arc::clone(&hook); @@ -301,13 +260,13 @@ mod tests { }) .collect(); for worker in workers { - assert_eq!(worker.join().unwrap().as_deref(), Some("summary of f")); + assert_eq!(worker.join().unwrap().as_deref(), Some("pseudo Body")); } } #[test] fn small_or_untagged_folds_never_reach_the_hook() { - let hook = hook("exit 3", 5000); + let hook = hook("error", 5000).unwrap(); let mut result = diff("import os\nimport sys\n"); hook.summarize(&mut result).unwrap(); let mut result = diff("def f():\n a()\n"); @@ -315,30 +274,32 @@ mod tests { } #[test] - fn failures_are_reported_without_losing_the_diff() { + fn startup_failures_are_reported() { + let error = hook("exit", 5000) + .err() + .expect("exit must fail") + .to_string(); + assert!(error.contains("exited during startup"), "{error}"); + } + + #[test] + fn call_failures_are_reported_without_losing_the_diff() { let mut result = diff(LARGE); - let error = hook("sleep 30", 50).summarize(&mut result).unwrap_err(); + let error = hook("slow", 200) + .unwrap() + .summarize(&mut result) + .unwrap_err(); assert!(error.contains("timed out"), "{error}"); - let error = hook("exit 0", 5000).summarize(&mut result).unwrap_err(); - assert!(error.contains("exited"), "{error}"); - let error = hook("echo not json", 5000) + let error = hook("error", 5000) + .unwrap() .summarize(&mut result) .unwrap_err(); - assert!(error.contains("invalid reply"), "{error}"); - let error = hook( - r#"read -r line; echo '{"id":1,"error":"rate limited"}'; sleep 30"#, - 5000, - ) - .summarize(&mut result) - .unwrap_err(); - assert!(error.contains("rate limited"), "{error}"); - let error = hook( - r#"read -r line; echo '{"id":1,"texts":{"7":"x"}}'; sleep 30"#, - 5000, - ) - .summarize(&mut result) - .unwrap_err(); - assert!(error.contains("unknown fold"), "{error}"); + assert!(error.contains("declined"), "{error}"); + let error = hook("bad", 5000) + .unwrap() + .summarize(&mut result) + .unwrap_err(); + assert!(error.starts_with("fold hook:"), "{error}"); assert!(result.rhs_folds.iter().all(|fold| fold.summary.is_none())); } } diff --git a/tests/hooks/rpc_server.py b/tests/hooks/rpc_server.py new file mode 100644 index 000000000..68ce0f38d --- /dev/null +++ b/tests/hooks/rpc_server.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Minimal JSON-RPC 2.0 fold hook for tests, standard library only. + +Listens on 127.0.0.1:$DIFFR_HOOK_PORT. The first argument selects a behavior: + echo answer every fold with "pseudo " + first answer only fold 0 with "summary of f" + error return a JSON-RPC error for every request + slow never answer (sleeps inside the handler) + cwd answer fold 0 with the working directory and assert DIFFR_WORKSPACE + bad return a non-JSON body +""" +import json +import os +import sys +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +MODE = sys.argv[1] + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + assert request["jsonrpc"] == "2.0" and request["method"] == "summarize", request + params = request["params"] + assert params["src"] and isinstance(params["folds"], list), params + if MODE == "slow": + time.sleep(30) + if MODE == "bad": + body = b"not json" + elif MODE == "error": + body = json.dumps({"jsonrpc": "2.0", "id": request["id"], + "error": {"code": -32000, "message": "declined"}}).encode() + else: + if MODE == "echo": + texts = {str(f["id"]): "pseudo " + f["placeholder"] for f in params["folds"]} + elif MODE == "first": + texts = {"0": "summary of f"} + elif MODE == "cwd": + assert os.environ["DIFFR_WORKSPACE"] == sys.argv[2], os.environ["DIFFR_WORKSPACE"] + texts = {"0": os.getcwd()} + else: + raise SystemExit(f"unknown mode {MODE}") + body = json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": texts}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +if MODE == "exit": + raise SystemExit(3) +ThreadingHTTPServer(("127.0.0.1", int(os.environ["DIFFR_HOOK_PORT"])), Handler).serve_forever() diff --git a/tests/streaming/check.py b/tests/streaming/check.py index 0346d3ccb..5217e5e05 100644 --- a/tests/streaming/check.py +++ b/tests/streaming/check.py @@ -3,6 +3,7 @@ import json import os from pathlib import Path +import shutil import subprocess import sys import tempfile @@ -129,41 +130,34 @@ def stream(repo, *args, code=0): git(repo, "init", "-q") git(repo, "commit", "--allow-empty", "-qm", "empty") base = git(repo, "rev-parse", "HEAD") - (repo / "hook.py").write_text( - "import json, sys\n" - "for line in sys.stdin:\n" - " request = json.loads(line)\n" - " assert request['language'] == 'Python' and request['src']\n" - " if request['path'] == 'bad.py':\n" - " reply = {'id': request['id'], 'error': 'declined'}\n" - " else:\n" - " reply = {'id': request['id'], 'texts': {str(f['id']): 'pseudo ' + f['placeholder'] for f in request['folds']}}\n" - " print(json.dumps(reply), flush=True)\n") + rpc_server = ROOT / "tests/hooks/rpc_server.py" large = "def f():\n a()\n b()\n c()\n\ndef g():\n d()\n" (repo / "good.py").write_text(large) (repo / "bad.py").write_text(large) (repo / "small.py").write_text("def h():\n e()\n") head = commit(repo, "additions") - (repo / "diffr.toml").write_text( - f"[folds.hook]\ncommand = [{json.dumps(sys.executable)}, 'hook.py']\ntags = ['body']\nmin_lines = 3\n") + def hook_config(mode, *extra): + command = [sys.executable, str(rpc_server), mode, *extra] + return f"[folds.hook]\ncommand = {json.dumps(command)}\ntags = ['body']\nmin_lines = 3\n" + (repo / "diffr.toml").write_text(hook_config("echo")) events = {e["file"]["new_path"]: e for e in stream(repo, base, head)[1:-1]} good = events["good.py"] assert "hook_error" not in good assert [f["summary"] for f in good["diff"]["rhs_folds"] if f["tags"] == ["body"]] == ["pseudo Body", None] - assert events["bad.py"]["hook_error"] == "fold hook reported: declined" assert all(f["summary"] is None for f in events["small.py"]["diff"]["rhs_folds"]) + (repo / "diffr.toml").write_text(hook_config("error")) + events = {e["file"]["new_path"]: e for e in stream(repo, base, head)[1:-1]} + assert events["bad.py"]["hook_error"] == "fold hook reported: declined" + assert all(f["summary"] is None for f in events["good.py"]["diff"]["rhs_folds"]) (repo / "diffr.toml").write_text("[folds.hook]\ncommand = ['./missing-hook']\n") assert cli(repo, "--format", "ndjson", base, head).returncode == 2 + (repo / "diffr.toml").write_text(hook_config("exit")) + assert cli(repo, "--format", "ndjson", base, head).returncode == 2 # Relative hook paths resolve against the config file, not the repository. with tempfile.TemporaryDirectory(prefix="diffr-hook-config-") as elsewhere: - (Path(elsewhere) / "hook.py").write_text( - "import json, os, sys\n" - "assert os.environ['DIFFR_WORKSPACE'] == sys.argv[1], os.environ['DIFFR_WORKSPACE']\n" - "for line in sys.stdin:\n" - " request = json.loads(line)\n" - " print(json.dumps({'id': request['id'], 'texts': {'0': os.getcwd()}}), flush=True)\n") + shutil.copy(rpc_server, Path(elsewhere) / "hook.py") (Path(elsewhere) / "hook.toml").write_text( - f"[folds.hook]\ncommand = [{json.dumps(sys.executable)}, 'hook.py', {json.dumps(str(repo.resolve()) + os.sep)}]\ntags = ['body']\n") + f"[folds.hook]\ncommand = [{json.dumps(sys.executable)}, 'hook.py', 'cwd', {json.dumps(str(repo.resolve()) + os.sep)}]\ntags = ['body']\n") events = stream(repo, base, head, "--config", str(Path(elsewhere) / "hook.toml"), "--", "good.py") assert events[1]["diff"]["rhs_folds"][0]["summary"] == str(Path(elsewhere).resolve()) From 8663a255c68791d3ee22e523eb516ad3f8a5bbf9 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 14:47:23 -0700 Subject: [PATCH 16/19] Serve the reference hook with aiohttp and jsonrpcserver One POST route hands each body to jsonrpcserver's async dispatch; the summarize method awaits Gemini on the shared httpx client under a semaphore. No hand-written protocol code remains on either side. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- examples/hooks/summarize.py | 89 +++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 49 deletions(-) diff --git a/examples/hooks/summarize.py b/examples/hooks/summarize.py index 2447b5bee..8a4adafee 100755 --- a/examples/hooks/summarize.py +++ b/examples/hooks/summarize.py @@ -1,28 +1,31 @@ #!/usr/bin/env -S uv run --script # /// script # requires-python = ">=3.11" -# dependencies = ["httpx>=0.27"] +# dependencies = ["aiohttp>=3.9", "httpx>=0.27", "jsonrpcserver>=5"] # /// """Reference diffr fold hook: rewrite large novel folds as Python-style pseudocode. -diffr writes one NDJSON request per file on stdin and reads one reply per request -on stdout, matched by id. Requests are answered concurrently on one event loop -with a shared HTTP client, so replies arrive in whatever order the model -finishes; diffr routes them by id. +diffr starts this server once per invocation with the port in DIFFR_HOOK_PORT and +calls the JSON-RPC 2.0 method `summarize` once per file, concurrently. Each call +awaits one Gemini request on a shared httpx client. Log to stderr; stdout is +discarded by diffr. Environment: + DIFFR_HOOK_PORT set by diffr GOOGLE_API_KEY required DIFFR_SUMMARY_MODEL default gemini-3.8-flash - DIFFR_SUMMARY_WORKERS concurrent requests, default 16 + DIFFR_SUMMARY_WORKERS concurrent model requests, default 16 """ import asyncio import json import os -import sys import httpx +from aiohttp import web +from jsonrpcserver import Error, Result, Success, async_dispatch, method API_KEY = os.environ["GOOGLE_API_KEY"] +PORT = int(os.environ["DIFFR_HOOK_PORT"]) MODEL = os.environ.get("DIFFR_SUMMARY_MODEL", "gemini-3.8-flash") WORKERS = int(os.environ.get("DIFFR_SUMMARY_WORKERS", "16")) URL = f"https://generativelanguage.googleapis.com/v1beta/models/{MODEL}:generateContent" @@ -45,36 +48,37 @@ }, } +client = httpx.AsyncClient(headers={"x-goog-api-key": API_KEY}, timeout=60) +limit = asyncio.Semaphore(WORKERS) -def prompt(request): - numbered = "\n".join( - f"{number:5d} | {line}" for number, line in enumerate(request["src"].splitlines(), 1) - ) - folds = "\n".join( + +def prompt(path, language, src, folds): + numbered = "\n".join(f"{n:5d} | {line}" for n, line in enumerate(src.splitlines(), 1)) + ranges = "\n".join( f"- fold {fold['id']}: lines {fold['range']['start']['line'] + 1}-" f"{fold['range']['end']['line'] + 1}" - for fold in request["folds"] + for fold in folds ) - language = request["language"] or "unknown language" - return f"File {request['path']} ({language}):\n\n{numbered}\n\nFolds:\n{folds}" + return f"File {path} ({language or 'unknown language'}):\n\n{numbered}\n\nFolds:\n{ranges}" -async def complete(client, request): +async def complete(path, language, src, folds): body = { "systemInstruction": {"parts": [{"text": SYSTEM}]}, - "contents": [{"role": "user", "parts": [{"text": prompt(request)}]}], + "contents": [{"role": "user", "parts": [{"text": prompt(path, language, src, folds)}]}], "generationConfig": { "temperature": 0, - "maxOutputTokens": 160 * len(request["folds"]) + 100, + "maxOutputTokens": 160 * len(folds) + 100, "thinkingConfig": {"thinkingBudget": 0}, "responseMimeType": "application/json", "responseSchema": SCHEMA, }, } - response = await client.post(URL, json=body) + async with limit: + response = await client.post(URL, json=body) response.raise_for_status() content = response.json()["candidates"][0]["content"]["parts"][-1]["text"] - expected = {fold["id"] for fold in request["folds"]} + expected = {fold["id"] for fold in folds} texts = {} for item in json.loads(content): if item["id"] not in expected: @@ -84,35 +88,22 @@ async def complete(client, request): return texts -async def answer(client, limit, line): - request = json.loads(line) - async with limit: - try: - reply = {"id": request["id"], "texts": await complete(client, request)} - except httpx.HTTPStatusError as error: - reply = { - "id": request["id"], - "error": f"{MODEL}: HTTP {error.response.status_code} {error.response.text[:200]}", - } - except (httpx.HTTPError, ValueError, KeyError) as error: - reply = {"id": request["id"], "error": f"{MODEL}: {error}"} - # Single-threaded: a whole line is written between awaits, never interleaved. - sys.stdout.write(json.dumps(reply) + "\n") - sys.stdout.flush() - - -async def main(): - reader = asyncio.StreamReader() - await asyncio.get_running_loop().connect_read_pipe( - lambda: asyncio.StreamReaderProtocol(reader), sys.stdin - ) - limit = asyncio.Semaphore(WORKERS) - async with httpx.AsyncClient(headers={"x-goog-api-key": API_KEY}, timeout=60) as client: - async with asyncio.TaskGroup() as tasks: - while line := await reader.readline(): - if line.strip(): - tasks.create_task(answer(client, limit, line)) +@method +async def summarize(path, language, src, folds) -> Result: + try: + return Success(await complete(path, language, src, folds)) + except httpx.HTTPStatusError as error: + return Error(-32000, f"{MODEL}: HTTP {error.response.status_code} {error.response.text[:200]}") + except (httpx.HTTPError, ValueError, KeyError) as error: + return Error(-32000, f"{MODEL}: {error}") + + +async def handle(request: web.Request) -> web.Response: + return web.Response(text=await async_dispatch(await request.text()), content_type="application/json") + +app = web.Application() +app.router.add_post("/", handle) if __name__ == "__main__": - asyncio.run(main()) + web.run_app(app, host="127.0.0.1", port=PORT, print=None, access_log=None) From eb2c3485c3035999b081e23fb4c109d56540ab0a Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 14:47:23 -0700 Subject: [PATCH 17/19] Document the JSON-RPC fold hook protocol AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- docs/streaming.md | 74 +++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/docs/streaming.md b/docs/streaming.md index 953ad3cf2..2a0062344 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -89,54 +89,60 @@ on Unix. ## Fold hooks -A trusted hook subprocess can replace fold placeholders with richer text, such -as pseudocode, before each `file` event is emitted: +A trusted hook can replace fold placeholders with richer text, such as +pseudocode, before each `file` event is emitted. A hook is a JSON-RPC 2.0 +server over HTTP that diffr starts once per invocation and calls on loopback: ```toml [folds.hook] command = ["uv", "run", "--script", "examples/hooks/summarize.py"] -tags = ["body"] # optional; any listed tag qualifies. Omit to send every fold. -min_lines = 12 # optional; default 0 -timeout_ms = 5000 # optional; per request +tags = ["body"] # optional; any listed tag qualifies. Omit to send every fold. +min_lines = 12 # optional; default 0 +timeout_ms = 5000 # optional; per call +startup_timeout_ms = 30000 # optional; time allowed to start listening ``` -The command starts once per invocation with the caller's environment. It runs -in the directory containing the config file, so relative paths in `command` -resolve against the config wherever it lives, including one given by -`--config` outside the repository. `DIFFR_WORKSPACE` carries the diffed -repository's root. Only novel folds on the after side qualify: bodies that -exist in the after source with no counterpart in the before source. Files with -no qualifying fold never reach the hook. Streaming is the only output mode that -runs hooks; the terminal frontend streams, so it does too. +The command starts with the caller's environment plus `DIFFR_HOOK_PORT`, the +loopback port it must listen on, and `DIFFR_WORKSPACE`, the diffed repository's +root. It runs in the directory containing the config file, so relative paths in +`command` resolve against the config wherever it lives, including one given by +`--config` outside the repository. Its stdout is discarded because diffr's own +stdout carries the event stream; log to stderr. diffr polls the port until the +hook accepts connections, exits 2 before `start` if the hook exits or misses +`startup_timeout_ms`, and kills the hook when the comparison ends. -Requests are one JSON line per file on the hook's stdin, and replies are one JSON -line per request on its stdout, matched by `id` and accepted in any order. The -worker diffing a file blocks on that file's reply; other workers keep going, so -a hook must answer requests concurrently rather than one at a time. +Only novel folds on the after side qualify: bodies that exist in the after +source with no counterpart in the before source. Files with no qualifying fold +never reach the hook. Streaming is the only output mode that runs hooks; the +terminal frontend streams, so it does too. + +One call per file, method `summarize`, params by name. The worker diffing that +file blocks on the reply; other workers keep calling, so a hook must serve +requests concurrently rather than one at a time. ```jsonc -// diffr -> hook -{"id": 7, "path": "src/auth.py", "language": "Python", "src": "", - "folds": [{"id": 0, "range": {"start": {"line": 40, "byte_column": 0}, "end": {"line": 88, "byte_column": 1}}, - "tags": ["body"], "placeholder": "Body"}]} +// diffr -> hook POST / with a JSON-RPC 2.0 request +{"jsonrpc": "2.0", "id": 7, "method": "summarize", "params": { + "path": "src/auth.py", "language": "Python", "src": "", + "folds": [{"id": 0, "range": {"start": {"line": 40, "byte_column": 0}, "end": {"line": 88, "byte_column": 1}}, + "tags": ["body"], "placeholder": "Body"}]}} // hook -> diffr -{"id": 7, "texts": {"0": "def refresh_token(session):\n ..."}} -{"id": 8, "error": "rate limited"} +{"jsonrpc": "2.0", "id": 7, "result": {"0": "def refresh_token(session):\n ..."}} +{"jsonrpc": "2.0", "id": 8, "error": {"code": -32000, "message": "rate limited"}} ``` `language` is null for plain text. A fold `id` indexes `rhs_folds` in that file's `diff`; the matching fold gains a non-null `summary` while `placeholder` is -unchanged. Folds missing from `texts` keep a null `summary`. An `error` reply, a -timeout, an unknown fold id, a malformed line, or hook exit leaves every summary -in that file null and adds `hook_error` to its `file` event. A malformed line or -exit also fails every later request, since ids can no longer be trusted. Hook -stderr passes through to diffr's stderr. Failing to start the command exits 2 -before `start`. - -`examples/hooks/summarize.py` is a reference hook that asks Gemini 3.8 Flash, -with thinking disabled, for Python-style pseudocode. It needs `GOOGLE_API_KEY` -and answers up to 16 files at once on one asyncio loop with a shared httpx -client, which `uv run --script` installs on first use. +unchanged. Folds missing from the result keep a null `summary`. An error +object, a timeout, an unknown fold id, or an invalid response leaves every +summary in that file null and adds `hook_error` to its `file` event. + +`examples/hooks/summarize.py` is a reference hook: an aiohttp server that hands +each request to jsonrpcserver and asks Gemini 3.8 Flash, with thinking disabled, +for Python-style pseudocode. It needs `GOOGLE_API_KEY` and answers up to 16 +files at once on one asyncio loop with a shared httpx client. `uv run --script` +installs its dependencies on first use. `tests/hooks/rpc_server.py` is a +dependency-free hook used by the tests. ## Fixture viewer From 9cd9486b95f21008bffd480468c32a6be00ee02f Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 15:08:45 -0700 Subject: [PATCH 18/19] Format and lint the Python hook and test scripts with ruff Sorted imports, literal dicts, an explicit check flag on subprocess.run, and executable bits on the shebang scripts. No behavior change. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: a8ed5d94-5d5f-4d7a-ae57-7c90ef8c7d6b Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- examples/hooks/summarize.py | 22 +++++-- tests/hooks/rpc_server.py | 26 ++++++-- tests/streaming/check.py | 114 ++++++++++++++++++++++++++++-------- 3 files changed, 127 insertions(+), 35 deletions(-) mode change 100644 => 100755 tests/hooks/rpc_server.py mode change 100644 => 100755 tests/streaming/check.py diff --git a/examples/hooks/summarize.py b/examples/hooks/summarize.py index 8a4adafee..6821423e7 100755 --- a/examples/hooks/summarize.py +++ b/examples/hooks/summarize.py @@ -16,6 +16,7 @@ DIFFR_SUMMARY_MODEL default gemini-3.8-flash DIFFR_SUMMARY_WORKERS concurrent model requests, default 16 """ + import asyncio import json import os @@ -53,7 +54,9 @@ def prompt(path, language, src, folds): - numbered = "\n".join(f"{n:5d} | {line}" for n, line in enumerate(src.splitlines(), 1)) + numbered = "\n".join( + f"{n:5d} | {line}" for n, line in enumerate(src.splitlines(), 1) + ) ranges = "\n".join( f"- fold {fold['id']}: lines {fold['range']['start']['line'] + 1}-" f"{fold['range']['end']['line'] + 1}" @@ -65,7 +68,9 @@ def prompt(path, language, src, folds): async def complete(path, language, src, folds): body = { "systemInstruction": {"parts": [{"text": SYSTEM}]}, - "contents": [{"role": "user", "parts": [{"text": prompt(path, language, src, folds)}]}], + "contents": [ + {"role": "user", "parts": [{"text": prompt(path, language, src, folds)}]} + ], "generationConfig": { "temperature": 0, "maxOutputTokens": 160 * len(folds) + 100, @@ -82,7 +87,9 @@ async def complete(path, language, src, folds): texts = {} for item in json.loads(content): if item["id"] not in expected: - raise ValueError(f"model answered for unknown fold {item['id']}: {content[:200]}") + raise ValueError( + f"model answered for unknown fold {item['id']}: {content[:200]}" + ) if item["pseudocode"].strip(): texts[str(item["id"])] = item["pseudocode"].strip() return texts @@ -93,13 +100,18 @@ async def summarize(path, language, src, folds) -> Result: try: return Success(await complete(path, language, src, folds)) except httpx.HTTPStatusError as error: - return Error(-32000, f"{MODEL}: HTTP {error.response.status_code} {error.response.text[:200]}") + return Error( + -32000, + f"{MODEL}: HTTP {error.response.status_code} {error.response.text[:200]}", + ) except (httpx.HTTPError, ValueError, KeyError) as error: return Error(-32000, f"{MODEL}: {error}") async def handle(request: web.Request) -> web.Response: - return web.Response(text=await async_dispatch(await request.text()), content_type="application/json") + return web.Response( + text=await async_dispatch(await request.text()), content_type="application/json" + ) app = web.Application() diff --git a/tests/hooks/rpc_server.py b/tests/hooks/rpc_server.py old mode 100644 new mode 100755 index 68ce0f38d..668b33032 --- a/tests/hooks/rpc_server.py +++ b/tests/hooks/rpc_server.py @@ -9,6 +9,7 @@ cwd answer fold 0 with the working directory and assert DIFFR_WORKSPACE bad return a non-JSON body """ + import json import os import sys @@ -29,19 +30,30 @@ def do_POST(self): if MODE == "bad": body = b"not json" elif MODE == "error": - body = json.dumps({"jsonrpc": "2.0", "id": request["id"], - "error": {"code": -32000, "message": "declined"}}).encode() + body = json.dumps( + { + "jsonrpc": "2.0", + "id": request["id"], + "error": {"code": -32000, "message": "declined"}, + } + ).encode() else: if MODE == "echo": - texts = {str(f["id"]): "pseudo " + f["placeholder"] for f in params["folds"]} + texts = { + str(f["id"]): "pseudo " + f["placeholder"] for f in params["folds"] + } elif MODE == "first": texts = {"0": "summary of f"} elif MODE == "cwd": - assert os.environ["DIFFR_WORKSPACE"] == sys.argv[2], os.environ["DIFFR_WORKSPACE"] + assert os.environ["DIFFR_WORKSPACE"] == sys.argv[2], os.environ[ + "DIFFR_WORKSPACE" + ] texts = {"0": os.getcwd()} else: raise SystemExit(f"unknown mode {MODE}") - body = json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": texts}).encode() + body = json.dumps( + {"jsonrpc": "2.0", "id": request["id"], "result": texts} + ).encode() self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) @@ -54,4 +66,6 @@ def log_message(self, *args): if MODE == "exit": raise SystemExit(3) -ThreadingHTTPServer(("127.0.0.1", int(os.environ["DIFFR_HOOK_PORT"])), Handler).serve_forever() +ThreadingHTTPServer( + ("127.0.0.1", int(os.environ["DIFFR_HOOK_PORT"])), Handler +).serve_forever() diff --git a/tests/streaming/check.py b/tests/streaming/check.py old mode 100644 new mode 100755 index 5217e5e05..db57aea86 --- a/tests/streaming/check.py +++ b/tests/streaming/check.py @@ -1,33 +1,57 @@ #!/usr/bin/env python3 """Exercise the CLI stream against real Git repositories and partial staging.""" + import json import os -from pathlib import Path import shutil import subprocess import sys import tempfile +from pathlib import Path ROOT = Path(__file__).resolve().parents[2] EXE = ROOT / "target/debug/diffr" -ENV = dict(os.environ, GIT_CONFIG_GLOBAL="/dev/null", GIT_CONFIG_NOSYSTEM="1", - GIT_AUTHOR_NAME="Test", GIT_AUTHOR_EMAIL="test@example.invalid", - GIT_COMMITTER_NAME="Test", GIT_COMMITTER_EMAIL="test@example.invalid") +ENV = dict( + os.environ, + GIT_CONFIG_GLOBAL="/dev/null", + GIT_CONFIG_NOSYSTEM="1", + GIT_AUTHOR_NAME="Test", + GIT_AUTHOR_EMAIL="test@example.invalid", + GIT_COMMITTER_NAME="Test", + GIT_COMMITTER_EMAIL="test@example.invalid", +) + def git(repo, *args): - return subprocess.check_output(["git", "-C", str(repo), *args], env=ENV).decode().strip() + return ( + subprocess.check_output(["git", "-C", str(repo), *args], env=ENV) + .decode() + .strip() + ) + def commit(repo, message): git(repo, "add", ".") git(repo, "commit", "-qm", message) return git(repo, "rev-parse", "HEAD") + def cli(repo, *args): - return subprocess.run([str(EXE), "--repo", str(repo), *args], capture_output=True, env=ENV) + return subprocess.run( + [str(EXE), "--repo", str(repo), *args], + capture_output=True, + env=ENV, + check=False, + ) + def stream(repo, *args, code=0): - with subprocess.Popen([str(EXE), "--repo", str(repo), "--format", "ndjson", *args], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=ENV) as process: + with subprocess.Popen( + [str(EXE), "--repo", str(repo), "--format", "ndjson", *args], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=ENV, + ) as process: first = json.loads(process.stdout.readline()) assert first["type"] == "start" and first["version"] == 1 assert len(first["files"]) == first["total"] @@ -37,11 +61,12 @@ def stream(repo, *args, code=0): assert events[-1]["type"] == "complete" succeeded = sum(e["type"] == "file" for e in events) failed = sum(e["type"] == "file_error" for e in events) - assert events[-1] == dict(type="complete", succeeded=succeeded, failed=failed) + assert events[-1] == {"type": "complete", "succeeded": succeeded, "failed": failed} assert succeeded + failed == first["total"] assert all("layout" not in e for e in events) return events + with tempfile.TemporaryDirectory(prefix="diffr-stream-") as temp: repo = Path(temp) git(repo, "init", "-q") @@ -55,20 +80,31 @@ def stream(repo, *args, code=0): (repo / "binary.bin").write_bytes(b"a\0b") (repo / "z.py").write_text("print('new')\n") head = commit(repo, "head") - (repo / ".gitattributes").write_text("*.rs diffr-classify=source\n*.py diffr-classify=test\n*.bin diffr-classify=generated\n") + (repo / ".gitattributes").write_text( + "*.rs diffr-classify=source\n*.py diffr-classify=test\n*.bin diffr-classify=generated\n" + ) (repo / "diffr.toml").write_text('[languages.rust]\nfolds = ""\n') events = stream(repo, base, head, "--order", "test,source,generated", code=2) - assert events[0]["before"] == dict(kind="revision", ref=base) - assert events[0]["after"] == dict(kind="revision", ref=head) + assert events[0]["before"] == {"kind": "revision", "ref": base} + assert events[0]["after"] == {"kind": "revision", "ref": head} # Results arrive in completion order; --order governs computation priority only. - assert sorted(e["file"]["class"] for e in events[1:-1]) == ["generated", "source", "test", "test", "test"] + assert sorted(e["file"]["class"] for e in events[1:-1]) == [ + "generated", + "source", + "test", + "test", + "test", + ] renamed = next(e["file"] for e in events[1:-1] if e["file"]["status"] == "renamed") assert renamed["old_path"] == "rename.py" and renamed["new_path"] == "renamed.py" rust = next(e for e in events[1:-1] if e["file"]["new_path"] == "a.rs") assert rust["diff"]["rhs_folds"] == [] # An early file failure must not prevent the later successes. events = stream(repo, base, head, "--order", "generated", code=2) - assert sum(e["type"] == "file_error" for e in events) == 1 and events[-1]["succeeded"] == 4 + assert ( + sum(e["type"] == "file_error" for e in events) == 1 + and events[-1]["succeeded"] == 4 + ) events = stream(repo, base, head, "--", "a.rs", "z.py") assert sorted(e["file"]["new_path"] for e in events[1:-1]) == ["a.rs", "z.py"] events = stream(repo, base, head, "--jobs", "1", "--", "a.rs", "z.py") @@ -76,7 +112,12 @@ def stream(repo, *args, code=0): assert len(stream(repo, head, head)) == 2 assert len(stream(repo, base, head, "--", "missing.rs")) == 2 stream(repo, base, head, "--exit-code", "--", "a.rs", code=1) - for args in (["bad-ref", head], [base, head, "--quiet"], ["--no-index", "a", "b"], [base, head, "--stat"]): + for args in ( + ["bad-ref", head], + [base, head, "--quiet"], + ["--no-index", "a", "b"], + [base, head, "--stat"], + ): result = cli(repo, "--format", "ndjson", *args) assert result.returncode == 2 and not result.stdout and result.stderr (repo / "diffr.toml").write_text("invalid toml") @@ -98,12 +139,18 @@ def stream(repo, *args, code=0): source.write_text(staged) git(repo, "add", "a.rs") source.write_text(working) - for selection, left, right in (([], staged, working), (["--cached"], initial, staged), ([base], initial, working)): + for selection, left, right in ( + ([], staged, working), + (["--cached"], initial, staged), + ([base], initial, working), + ): for reverse in (False, True): args = [*selection, *(["-R"] if reverse else [])] for output in ("--name-only", "--numstat"): actual = cli(repo, *args, output) - expected = subprocess.check_output(["git", "-C", str(repo), "diff", *args, output], env=ENV) + expected = subprocess.check_output( + ["git", "-C", str(repo), "diff", *args, output], env=ENV + ) assert actual.returncode == 0 and actual.stdout == expected diff = stream(repo, *args)[1]["diff"] assert diff["lhs_src"]["Text"] == (right if reverse else left) @@ -121,7 +168,7 @@ def stream(repo, *args, code=0): (repo / "new.rs").write_text("fn new() {}\n") git(repo, "add", ".") events = stream(repo, "--cached") - assert events[0]["before"] == dict(kind="empty_tree") + assert events[0]["before"] == {"kind": "empty_tree"} assert events[1]["file"]["status"] == "added" # A configured fold hook fills summaries before each file event; its failures # are reported per file without losing the diff. @@ -136,14 +183,18 @@ def stream(repo, *args, code=0): (repo / "bad.py").write_text(large) (repo / "small.py").write_text("def h():\n e()\n") head = commit(repo, "additions") + def hook_config(mode, *extra): command = [sys.executable, str(rpc_server), mode, *extra] return f"[folds.hook]\ncommand = {json.dumps(command)}\ntags = ['body']\nmin_lines = 3\n" + (repo / "diffr.toml").write_text(hook_config("echo")) events = {e["file"]["new_path"]: e for e in stream(repo, base, head)[1:-1]} good = events["good.py"] assert "hook_error" not in good - assert [f["summary"] for f in good["diff"]["rhs_folds"] if f["tags"] == ["body"]] == ["pseudo Body", None] + assert [ + f["summary"] for f in good["diff"]["rhs_folds"] if f["tags"] == ["body"] + ] == ["pseudo Body", None] assert all(f["summary"] is None for f in events["small.py"]["diff"]["rhs_folds"]) (repo / "diffr.toml").write_text(hook_config("error")) events = {e["file"]["new_path"]: e for e in stream(repo, base, head)[1:-1]} @@ -157,9 +208,20 @@ def hook_config(mode, *extra): with tempfile.TemporaryDirectory(prefix="diffr-hook-config-") as elsewhere: shutil.copy(rpc_server, Path(elsewhere) / "hook.py") (Path(elsewhere) / "hook.toml").write_text( - f"[folds.hook]\ncommand = [{json.dumps(sys.executable)}, 'hook.py', 'cwd', {json.dumps(str(repo.resolve()) + os.sep)}]\ntags = ['body']\n") - events = stream(repo, base, head, "--config", str(Path(elsewhere) / "hook.toml"), "--", "good.py") - assert events[1]["diff"]["rhs_folds"][0]["summary"] == str(Path(elsewhere).resolve()) + f"[folds.hook]\ncommand = [{json.dumps(sys.executable)}, 'hook.py', 'cwd', {json.dumps(str(repo.resolve()) + os.sep)}]\ntags = ['body']\n" + ) + events = stream( + repo, + base, + head, + "--config", + str(Path(elsewhere) / "hook.toml"), + "--", + "good.py", + ) + assert events[1]["diff"]["rhs_folds"][0]["summary"] == str( + Path(elsewhere).resolve() + ) # Closing the pipe while a multi-file producer is active must not leave it # blocked forever on a full queue. Unix CLI output retains normal SIGPIPE behavior. @@ -172,8 +234,12 @@ def hook_config(mode, *extra): for index in range(8): (repo / f"{index}.txt").write_text("some new text\n" * 4096) head = commit(repo, "large additions") - with subprocess.Popen([str(EXE), "--repo", str(repo), base, head, "--format", "ndjson"], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=ENV) as process: + with subprocess.Popen( + [str(EXE), "--repo", str(repo), base, head, "--format", "ndjson"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=ENV, + ) as process: assert json.loads(process.stdout.readline())["type"] == "start" process.stdout.close() assert process.wait(timeout=30) != 0 From a22545aeb27ef358241f8b72107413c1f00870e3 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Thu, 10 Sep 2026 15:14:16 -0700 Subject: [PATCH 19/19] Fix a README typo and exclude patch fixtures from the typo check Both findings predate this branch: one misspelling in the review examples README, and a Git blob hash in a fixture patch that spells a word. AI assistance: implemented with Claude Code. Claude-Session: https://claude.ai/code/session_015ZLcD4fU7YTe4CTcB1Q7oN Agent-Session: 01a089c9-50b5-7593-81bc-8024113a43d4 Agent-Session: c0881bbc-fcf4-466b-b42b-d04811dd045e Agent-Session: 26b74d72-c5f7-46ce-9404-849b5bb0a0ae Agent-Session: 01a08ccf-048f-7790-9e74-4889a953c39f Agent-Session: 118d399f-d820-431e-8b32-8c27fbbc44eb --- .typos.toml | 2 ++ examples/review/README.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.typos.toml b/.typos.toml index 5bf9439f4..011989477 100644 --- a/.typos.toml +++ b/.typos.toml @@ -5,6 +5,8 @@ extend-exclude = [ "homepage/home_img", "demo_files/", "difft.1", + # Git index lines carry blob hashes that can spell words. + "examples/review/real/*/change.patch", ] [default.extend-words] diff --git a/examples/review/README.md b/examples/review/README.md index e3a39bbb7..b017e1a34 100644 --- a/examples/review/README.md +++ b/examples/review/README.md @@ -148,7 +148,7 @@ remain outside this v0. - Fold candidates are generated but ignored by this printer. Visible import lines stay expanded; no placeholders are emitted. - Left/right line numbers, grouped deletions/additions and explicit gaps. `~` marks matched syntax whose indentation differs; both exact source lines are retained in text snapshots. This - is a review snapshot, not an applyable Git patch. Token-level colors and split + is a review snapshot, not an applicable Git patch. Token-level colors and split view are not implemented. - Unsupported languages or Difftastic limit fallbacks retain a textual diff without syntax annotations. The context rules currently target the corpus's