From c83eab7da0d0c7cfa7c2ac05b024c29dfa52917d Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Sat, 19 Sep 2026 14:48:27 -0700 Subject: [PATCH 1/2] Define the Rust computed-diff storage contract Keep the Store trait beside the search index that consumes it. Cache query-independent computed trees under versioned content/settings keys; selected highlights and plugin instances are excluded. AI assistance: reorganized with Codex. --- docs/diff-store.md | 91 +++++++++++++++++++++++++++++++++++++++++++++ src/search/store.rs | 62 ++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 docs/diff-store.md create mode 100644 src/search/store.rs diff --git a/docs/diff-store.md b/docs/diff-store.md new file mode 100644 index 000000000..36dd43210 --- /dev/null +++ b/docs/diff-store.md @@ -0,0 +1,91 @@ +# Computed diff storage + +Storage is a host-owned diffr setting. JS, Review API payloads, and MCP inputs +never name a backend or storage path. Configure it with the existing CLI: + +```sh +diffr config set storage.backend sqlite +diffr config set storage.path .cache/diffr +``` + +Or in diffr's configuration file (`$XDG_CONFIG_HOME/diffr/config.toml`, normally +`~/.config/diffr/config.toml`): + +```toml +[storage] +backend = "sqlite" # "memory" (default), "file", or "sqlite" +path = ".cache/diffr" +``` + +`storage.path` is a directory, relative to the source repository unless absolute. +File storage writes entries there; SQLite uses `diffs.sqlite` inside it. Memory +ignores the path. Persistent directories are created automatically. File entries +are published with atomic rename; SQLite uses WAL and a busy timeout. + +The native Rust entry points load diffr config and select the store. Configuration +is checked on each call, and a changed setting selects a runtime with that config. +Storage settings are excluded from computed-diff keys: moving the backing store +does not change analysis identity. Analysis uses the same loaded diff/plugin +configuration. All native reads, writes, setup, and computation run on N-API's +blocking worker pool. JS simply calls `hydrate(scope, hits)` and +`postprocess(scope, results)` as before. + +Rust callers can provide their own synchronous store: + +```rust +use std::sync::Arc; +use difftastic::{storage::FileStore, search::{Index, Session}}; + +let store = Arc::new(FileStore::open(".cache/diffr")?); +let index = Arc::new(Index::new(store)); +let mut session = Session::new(scope, index.clone())?; +let candidates = session.hydrate(hits)?; +let results = session.postprocess(candidates)?; +// Other comparisons use the same index.clone(). +# Ok::<(), anyhow::Error>(()) +``` + +`search::store::Store: Send + Sync` has `get`, `put`, and `remove` methods, using `DiffKey` +and `StoredDiff`. `StoredDiff` supports serde for custom backends. Built-in +implementations are `storage::MemoryStore`, `storage::FileStore`, and `storage::SqliteStore`. + +Each entry contains the computed base/head source trees, source text, syntax, +alignments, changed spans, file identities, and classification metadata. It has +no search highlights or query-specific presentation state. Hydration and +postprocessing work on copies. Jev responses are not stored here. + +Keys hash canonical JSON containing the engine/package and cache-format version, +analysis configuration, resolved query contents (including imports), paths, +blob IDs, modes, status, and classification tags. Different queries over the same +files reuse the computed trees. Different analysis settings produce different +keys. Worktree pins are validated even on a warm cache. Developers must bump +`CACHE_VERSION` in `src/search/store.rs` when algorithms, parsers, or tree semantics +change incompatibly without a package version bump. + +`Index` owns the shared store and performs cache lookup and diff +computation. Each `Session` owns its pinned scope, compiled queries, plugin +pipeline, and comparison manifest, and holds a reference to the same index. +Sessions are created per native call; dropping one does not discard computed +diffs. There is no cache of indexes keyed by scope or plugin configuration. + +The host holds one active index. Changing commits or analysis settings creates +new session state while keeping that index and store. Only changing the storage +backend or resolved directory replaces the active index; existing sessions can +finish with their original index. Memory ignores the configured path and is +shared across repositories. For persistent storage, a repository-relative path +resolves to that repository's directory. + +Hydration and postprocessing use typed Rust inputs; only the N-API boundary +converts JSON. Sessions do not share mutable plugin state or serialize all search +operations through a global lock. Concurrent cold misses may compute the same +diff independently; writes atomically replace the entry for that key. + +Store errors and corrupt records are reported to the caller. No automatic +retention/eviction policy is included: memory entries live until removed or the +index/store is dropped; persistent entries live until removed. Old-version +entries can be deleted. Replacing the active memory backend discards its entries once existing sessions finish. + +Rust tests load real configuration files and reopen persistent stores with writes +forbidden to verify cache reuse, identical results, configuration isolation, and +absence of query highlights in saved entries. Store contract tests cover replacement/removal, reopen, corruption, +canonical keys, and concurrent file writers. diff --git a/src/search/store.rs b/src/search/store.rs new file mode 100644 index 000000000..ba17ed941 --- /dev/null +++ b/src/search/store.rs @@ -0,0 +1,62 @@ +//! Storage contract for query-independent indexed diffs. +use crate::{ + pairing::Pairing, + protocol::{FileChange, Source}, +}; +use anyhow::{ensure, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Bump when diff algorithms, parsers, or serialized tree semantics change. +const CACHE_VERSION: &str = concat!(env!("CARGO_PKG_VERSION"), ":search-diff-v1"); + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct DiffKey(String); +impl DiffKey { + /// Hash a canonical JSON description of analysis settings and source identities. + pub fn new(identity: &impl Serialize) -> Result { + let mut canonical = serde_json::to_value((CACHE_VERSION, identity))?; + canonical.sort_all_objects(); + Ok(Self(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&canonical)?) + ))) + } + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom for DiffKey { + type Error = anyhow::Error; + fn try_from(value: String) -> Result { + ensure!( + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)), + "invalid diff key" + ); + Ok(Self(value)) + } +} +impl From for String { + fn from(key: DiffKey) -> Self { + key.0 + } +} + +/// Serializable computed data. Plugin instances and search highlights are not stored. +/// Fields remain internal; custom stores can serialize/deserialize this value with serde. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StoredDiff { + pub(crate) entry: FileChange, + pub(crate) sources: Pairing, +} + +pub trait Store: Send + Sync { + fn get(&self, key: &DiffKey) -> Result>; + fn put(&self, key: &DiffKey, diff: &StoredDiff) -> Result<()>; + fn remove(&self, key: &DiffKey) -> Result<()>; +} From 5141f722d50a16ee65cf4f445153cf102e3e5f95 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Sat, 19 Sep 2026 14:48:28 -0700 Subject: [PATCH 2/2] Implement memory, file and SQLite stores behind host configuration Select backend and location exclusively in Rust configuration. Include shared backend contract, persistence, corruption and config tests. Index lifetime and dependency injection are connected in the search-engine layer above. AI assistance: reorganized with Codex. --- src/config.rs | 3 + src/config/default.toml | 4 + src/config/store.rs | 21 +++++ src/storage.rs | 170 ++++++++++++++++++++++++++++++++++++++++ src/storage/file.rs | 57 ++++++++++++++ src/storage/memory.rs | 32 ++++++++ src/storage/sqlite.rs | 62 +++++++++++++++ 7 files changed, 349 insertions(+) create mode 100644 src/storage.rs create mode 100644 src/storage/file.rs create mode 100644 src/storage/memory.rs create mode 100644 src/storage/sqlite.rs diff --git a/src/config.rs b/src/config.rs index c1ad8bf4d..0c2746f74 100644 --- a/src/config.rs +++ b/src/config.rs @@ -55,6 +55,9 @@ pub(crate) struct Config { /// Limits on the structural comparison itself. #[serde(default)] pub(crate) diff: DiffConfig, + /// Host-owned cache for computed diffs. + #[serde(default)] + pub(crate) storage: crate::storage::StoreConfig, } impl Default for Config { diff --git a/src/config/default.toml b/src/config/default.toml index 6056aa8a7..c5ad9181d 100644 --- a/src/config/default.toml +++ b/src/config/default.toml @@ -59,3 +59,7 @@ name = "default-dark" byte_limit = 1000000 graph_limit = 3000000 parse_error_limit = 0 + +[storage] +backend = "memory" +path = ".cache/diffr" diff --git a/src/config/store.rs b/src/config/store.rs index 10ec5cc95..aa2c9296b 100644 --- a/src/config/store.rs +++ b/src/config/store.rs @@ -511,3 +511,24 @@ mod materialization_tests { assert!(!text.contains("bundled.context")); } } + +#[cfg(test)] +mod storage_settings_tests { + use super::*; + #[test] + fn backend_and_directory_are_editable_config_settings() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("config.toml"); + set(&path, "storage.backend", "file").unwrap(); + set(&path, "storage.path", ".cache/custom").unwrap(); + let shown = show(&Config::load(Some(&path)).unwrap(), false); + assert_eq!(shown["storage"]["backend"], "file"); + assert_eq!(shown["storage"]["path"], ".cache/custom"); + set(&path, "storage.backend", "sqlite").unwrap(); + assert_eq!( + show(&Config::load(Some(&path)).unwrap(), false)["storage"]["backend"], + "sqlite" + ); + assert!(set(&path, "storage.backend", "unknown").is_err()); + } +} diff --git a/src/storage.rs b/src/storage.rs new file mode 100644 index 000000000..9e1bc0898 --- /dev/null +++ b/src/storage.rs @@ -0,0 +1,170 @@ +//! Built-in computed-diff stores and host configuration. +mod file; +mod memory; +mod sqlite; +use crate::search::store::Store; +use anyhow::Result; +pub use file::FileStore; +pub use memory::MemoryStore; +use serde::{Deserialize, Serialize}; +pub use sqlite::SqliteStore; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +/// Host-owned storage settings, loaded through diffr config. +#[derive(Clone, Debug, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(default, deny_unknown_fields)] +pub(crate) struct StoreConfig { + /// Where computed diffs are kept. Memory lasts for the process; file and SQLite persist. + #[schemars(title = "Diff storage backend", extend("x-group" = "Storage"))] + pub(crate) backend: StoreBackend, + /// Storage directory, relative to the source repository unless absolute. SQLite uses diffs.sqlite inside it. + #[schemars(title = "Diff storage directory", extend("x-group" = "Storage"))] + pub(crate) path: PathBuf, +} +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "lowercase")] +pub(crate) enum StoreBackend { + #[default] + Memory, + File, + Sqlite, +} +impl Default for StoreConfig { + fn default() -> Self { + Self { + backend: StoreBackend::Memory, + path: ".cache/diffr".into(), + } + } +} +impl StoreConfig { + pub(crate) fn open(&self, repository: &Path) -> Result> { + let directory = repository.join(&self.path); + Ok(match self.backend { + StoreBackend::Memory => Arc::new(MemoryStore::default()), + StoreBackend::File => Arc::new(FileStore::open(directory)?), + StoreBackend::Sqlite => Arc::new(SqliteStore::open(directory.join("diffs.sqlite"))?), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::{FileChange, FileRef, FileStatus}; + use crate::{ + pairing::Pairing, + protocol::Source, + search::store::{DiffKey, StoredDiff}, + }; + use rusqlite::Connection; + use std::fs; + + fn sample(text: &str) -> StoredDiff { + StoredDiff { + entry: FileChange { + file: Pairing::RightOnly { + rhs: FileRef { + path: "x.rs".into(), + oid: "abc".into(), + mode: "100644".into(), + }, + }, + status: FileStatus::Added, + tags: vec![], + }, + sources: Pairing::RightOnly { + rhs: Source { + text: text.into(), + syntax: vec![], + regions: vec![], + }, + }, + } + } + fn contract(store: &dyn Store) { + let key = DiffKey::new(&("a.rs", "blob", "config")).unwrap(); + let other = DiffKey::new(&("a.rs", "blob2", "config")).unwrap(); + assert!(store.get(&key).unwrap().is_none()); + let value = sample("hello\n"); + store.put(&key, &value).unwrap(); + assert_eq!( + serde_json::to_value(store.get(&key).unwrap().unwrap()).unwrap(), + serde_json::to_value(value).unwrap() + ); + assert!(store.get(&other).unwrap().is_none()); + store.put(&key, &sample("replacement")).unwrap(); + assert_eq!( + serde_json::to_value(store.get(&key).unwrap().unwrap()).unwrap(), + serde_json::to_value(sample("replacement")).unwrap() + ); + store.remove(&key).unwrap(); + store.remove(&key).unwrap(); + assert!(store.get(&key).unwrap().is_none()); + } + #[test] + fn all_backends_obey_the_same_contract() { + let directory = tempfile::tempdir().unwrap(); + contract(&MemoryStore::default()); + contract(&FileStore::open(directory.path().join("files")).unwrap()); + contract(&SqliteStore::open(directory.path().join("diffs.sqlite")).unwrap()); + } + #[test] + fn durable_stores_reopen_and_report_corruption() { + let directory = tempfile::tempdir().unwrap(); + let key = DiffKey::new(&"identity").unwrap(); + let files = directory.path().join("files"); + FileStore::open(&files) + .unwrap() + .put(&key, &sample("saved")) + .unwrap(); + let reopened = FileStore::open(&files).unwrap(); + assert!(reopened.get(&key).unwrap().is_some()); + fs::write(files.join(format!("{}.json", key.as_str())), b"broken JSON").unwrap(); + assert!(reopened.get(&key).is_err()); + let db = directory.path().join("diffs.sqlite"); + SqliteStore::open(&db) + .unwrap() + .put(&key, &sample("saved")) + .unwrap(); + assert!(SqliteStore::open(&db).unwrap().get(&key).unwrap().is_some()); + Connection::open(&db) + .unwrap() + .execute("UPDATE diffs SET data = ?1", [b"broken JSON".as_slice()]) + .unwrap(); + assert!(SqliteStore::open(&db).unwrap().get(&key).is_err()); + } + #[test] + fn keys_are_canonical_and_reject_path_injection() { + let a: serde_json::Value = serde_json::from_str(r#"{"blob":"a","config":"x"}"#).unwrap(); + let b: serde_json::Value = serde_json::from_str(r#"{"config":"x","blob":"a"}"#).unwrap(); + assert_eq!(DiffKey::new(&a).unwrap(), DiffKey::new(&b).unwrap()); + assert_ne!( + DiffKey::new(&a).unwrap(), + DiffKey::new(&serde_json::json!({"blob":"a","config":"y"})).unwrap() + ); + assert!(serde_json::from_str::(r#""../../outside""#).is_err()); + } + #[test] + fn concurrent_file_writers_publish_whole_records() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileStore::open(directory.path()).unwrap()); + let key = DiffKey::new(&"shared").unwrap(); + store.put(&key, &sample("initial")).unwrap(); + std::thread::scope(|scope| { + for index in 0..4 { + let store = &store; + let key = &key; + scope.spawn(move || { + for _ in 0..10 { + store.put(key, &sample(&index.to_string())).unwrap(); + assert!(store.get(key).unwrap().is_some()); + } + }); + } + }); + } +} diff --git a/src/storage/file.rs b/src/storage/file.rs new file mode 100644 index 000000000..f2c335e55 --- /dev/null +++ b/src/storage/file.rs @@ -0,0 +1,57 @@ +use crate::search::store::{DiffKey, Store, StoredDiff}; +use anyhow::{ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use std::{fs, io::Write, path::PathBuf}; + +pub struct FileStore { + directory: PathBuf, +} +impl FileStore { + pub fn open(directory: impl Into) -> Result { + let directory = directory.into(); + fs::create_dir_all(&directory).context("create diff store directory")?; + Ok(Self { directory }) + } + fn path(&self, key: &DiffKey) -> PathBuf { + self.directory.join(format!("{}.json", key.as_str())) + } +} +#[derive(Serialize, Deserialize)] +struct Record { + key: DiffKey, + diff: StoredDiff, +} +impl Store for FileStore { + fn get(&self, key: &DiffKey) -> Result> { + let bytes = match fs::read(self.path(key)) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error).context("read diff store entry"), + }; + let record: Record = serde_json::from_slice(&bytes).context("decode diff store entry")?; + ensure!(&record.key == key, "diff store key mismatch"); + Ok(Some(record.diff)) + } + fn put(&self, key: &DiffKey, diff: &StoredDiff) -> Result<()> { + let mut file = tempfile::NamedTempFile::new_in(&self.directory)?; + serde_json::to_writer( + &mut file, + &Record { + key: key.clone(), + diff: diff.clone(), + }, + )?; + file.flush()?; + file.as_file().sync_all()?; + file.persist(self.path(key)) + .context("publish diff store entry")?; + Ok(()) + } + fn remove(&self, key: &DiffKey) -> Result<()> { + match fs::remove_file(self.path(key)) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + } +} diff --git a/src/storage/memory.rs b/src/storage/memory.rs new file mode 100644 index 000000000..09e98509a --- /dev/null +++ b/src/storage/memory.rs @@ -0,0 +1,32 @@ +use crate::search::store::{DiffKey, Store, StoredDiff}; +use anyhow::Result; +use std::{collections::HashMap, sync::Mutex}; + +#[derive(Default)] +pub struct MemoryStore { + entries: Mutex>, +} +impl Store for MemoryStore { + fn get(&self, key: &DiffKey) -> Result> { + Ok(self + .entries + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))? + .get(key) + .cloned()) + } + fn put(&self, key: &DiffKey, diff: &StoredDiff) -> Result<()> { + self.entries + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))? + .insert(key.clone(), diff.clone()); + Ok(()) + } + fn remove(&self, key: &DiffKey) -> Result<()> { + self.entries + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))? + .remove(key); + Ok(()) + } +} diff --git a/src/storage/sqlite.rs b/src/storage/sqlite.rs new file mode 100644 index 000000000..ba95aa9e1 --- /dev/null +++ b/src/storage/sqlite.rs @@ -0,0 +1,62 @@ +use crate::search::store::{DiffKey, Store, StoredDiff}; +use anyhow::{Context, Result}; +use rusqlite::{params, Connection, OptionalExtension}; +use std::{fs, path::Path, sync::Mutex}; + +pub struct SqliteStore { + connection: Mutex, +} +impl SqliteStore { + pub fn open(path: impl AsRef) -> Result { + let path = path.as_ref(); + if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + fs::create_dir_all(parent)?; + } + let connection = Connection::open(path)?; + connection.busy_timeout(std::time::Duration::from_secs(5))?; + connection.execute_batch( + "PRAGMA journal_mode=WAL; + CREATE TABLE IF NOT EXISTS diffs (key TEXT PRIMARY KEY, data BLOB NOT NULL);", + )?; + Ok(Self { + connection: Mutex::new(connection), + }) + } +} +impl Store for SqliteStore { + fn get(&self, key: &DiffKey) -> Result> { + let connection = self + .connection + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))?; + let bytes: Option> = connection + .query_row( + "SELECT data FROM diffs WHERE key = ?1", + [key.as_str()], + |row| row.get(0), + ) + .optional()?; + bytes + .map(|bytes| serde_json::from_slice(&bytes).context("decode SQLite diff entry")) + .transpose() + } + fn put(&self, key: &DiffKey, diff: &StoredDiff) -> Result<()> { + let connection = self + .connection + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))?; + connection.execute( + "INSERT INTO diffs(key,data) VALUES (?1,?2) + ON CONFLICT(key) DO UPDATE SET data=excluded.data", + params![key.as_str(), serde_json::to_vec(diff)?], + )?; + Ok(()) + } + fn remove(&self, key: &DiffKey) -> Result<()> { + self.connection + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))? + .execute("DELETE FROM diffs WHERE key = ?1", [key.as_str()])?; + Ok(()) + } +}