diff --git a/docs/diff-store.md b/docs/diff-store.md new file mode 100644 index 000000000..36dd43210 --- /dev/null +++ b/docs/diff-store.md @@ -0,0 +1,91 @@ +# Computed diff storage + +Storage is a host-owned diffr setting. JS, Review API payloads, and MCP inputs +never name a backend or storage path. Configure it with the existing CLI: + +```sh +diffr config set storage.backend sqlite +diffr config set storage.path .cache/diffr +``` + +Or in diffr's configuration file (`$XDG_CONFIG_HOME/diffr/config.toml`, normally +`~/.config/diffr/config.toml`): + +```toml +[storage] +backend = "sqlite" # "memory" (default), "file", or "sqlite" +path = ".cache/diffr" +``` + +`storage.path` is a directory, relative to the source repository unless absolute. +File storage writes entries there; SQLite uses `diffs.sqlite` inside it. Memory +ignores the path. Persistent directories are created automatically. File entries +are published with atomic rename; SQLite uses WAL and a busy timeout. + +The native Rust entry points load diffr config and select the store. Configuration +is checked on each call, and a changed setting selects a runtime with that config. +Storage settings are excluded from computed-diff keys: moving the backing store +does not change analysis identity. Analysis uses the same loaded diff/plugin +configuration. All native reads, writes, setup, and computation run on N-API's +blocking worker pool. JS simply calls `hydrate(scope, hits)` and +`postprocess(scope, results)` as before. + +Rust callers can provide their own synchronous store: + +```rust +use std::sync::Arc; +use difftastic::{storage::FileStore, search::{Index, Session}}; + +let store = Arc::new(FileStore::open(".cache/diffr")?); +let index = Arc::new(Index::new(store)); +let mut session = Session::new(scope, index.clone())?; +let candidates = session.hydrate(hits)?; +let results = session.postprocess(candidates)?; +// Other comparisons use the same index.clone(). +# Ok::<(), anyhow::Error>(()) +``` + +`search::store::Store: Send + Sync` has `get`, `put`, and `remove` methods, using `DiffKey` +and `StoredDiff`. `StoredDiff` supports serde for custom backends. Built-in +implementations are `storage::MemoryStore`, `storage::FileStore`, and `storage::SqliteStore`. + +Each entry contains the computed base/head source trees, source text, syntax, +alignments, changed spans, file identities, and classification metadata. It has +no search highlights or query-specific presentation state. Hydration and +postprocessing work on copies. Jev responses are not stored here. + +Keys hash canonical JSON containing the engine/package and cache-format version, +analysis configuration, resolved query contents (including imports), paths, +blob IDs, modes, status, and classification tags. Different queries over the same +files reuse the computed trees. Different analysis settings produce different +keys. Worktree pins are validated even on a warm cache. Developers must bump +`CACHE_VERSION` in `src/search/store.rs` when algorithms, parsers, or tree semantics +change incompatibly without a package version bump. + +`Index` owns the shared store and performs cache lookup and diff +computation. Each `Session` owns its pinned scope, compiled queries, plugin +pipeline, and comparison manifest, and holds a reference to the same index. +Sessions are created per native call; dropping one does not discard computed +diffs. There is no cache of indexes keyed by scope or plugin configuration. + +The host holds one active index. Changing commits or analysis settings creates +new session state while keeping that index and store. Only changing the storage +backend or resolved directory replaces the active index; existing sessions can +finish with their original index. Memory ignores the configured path and is +shared across repositories. For persistent storage, a repository-relative path +resolves to that repository's directory. + +Hydration and postprocessing use typed Rust inputs; only the N-API boundary +converts JSON. Sessions do not share mutable plugin state or serialize all search +operations through a global lock. Concurrent cold misses may compute the same +diff independently; writes atomically replace the entry for that key. + +Store errors and corrupt records are reported to the caller. No automatic +retention/eviction policy is included: memory entries live until removed or the +index/store is dropped; persistent entries live until removed. Old-version +entries can be deleted. Replacing the active memory backend discards its entries once existing sessions finish. + +Rust tests load real configuration files and reopen persistent stores with writes +forbidden to verify cache reuse, identical results, configuration isolation, and +absence of query highlights in saved entries. Store contract tests cover replacement/removal, reopen, corruption, +canonical keys, and concurrent file writers. diff --git a/src/config.rs b/src/config.rs index c1ad8bf4d..0c2746f74 100644 --- a/src/config.rs +++ b/src/config.rs @@ -55,6 +55,9 @@ pub(crate) struct Config { /// Limits on the structural comparison itself. #[serde(default)] pub(crate) diff: DiffConfig, + /// Host-owned cache for computed diffs. + #[serde(default)] + pub(crate) storage: crate::storage::StoreConfig, } impl Default for Config { diff --git a/src/config/default.toml b/src/config/default.toml index 6056aa8a7..c5ad9181d 100644 --- a/src/config/default.toml +++ b/src/config/default.toml @@ -59,3 +59,7 @@ name = "default-dark" byte_limit = 1000000 graph_limit = 3000000 parse_error_limit = 0 + +[storage] +backend = "memory" +path = ".cache/diffr" diff --git a/src/config/store.rs b/src/config/store.rs index 10ec5cc95..aa2c9296b 100644 --- a/src/config/store.rs +++ b/src/config/store.rs @@ -511,3 +511,24 @@ mod materialization_tests { assert!(!text.contains("bundled.context")); } } + +#[cfg(test)] +mod storage_settings_tests { + use super::*; + #[test] + fn backend_and_directory_are_editable_config_settings() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("config.toml"); + set(&path, "storage.backend", "file").unwrap(); + set(&path, "storage.path", ".cache/custom").unwrap(); + let shown = show(&Config::load(Some(&path)).unwrap(), false); + assert_eq!(shown["storage"]["backend"], "file"); + assert_eq!(shown["storage"]["path"], ".cache/custom"); + set(&path, "storage.backend", "sqlite").unwrap(); + assert_eq!( + show(&Config::load(Some(&path)).unwrap(), false)["storage"]["backend"], + "sqlite" + ); + assert!(set(&path, "storage.backend", "unknown").is_err()); + } +} diff --git a/src/search/store.rs b/src/search/store.rs new file mode 100644 index 000000000..ba17ed941 --- /dev/null +++ b/src/search/store.rs @@ -0,0 +1,62 @@ +//! Storage contract for query-independent indexed diffs. +use crate::{ + pairing::Pairing, + protocol::{FileChange, Source}, +}; +use anyhow::{ensure, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Bump when diff algorithms, parsers, or serialized tree semantics change. +const CACHE_VERSION: &str = concat!(env!("CARGO_PKG_VERSION"), ":search-diff-v1"); + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct DiffKey(String); +impl DiffKey { + /// Hash a canonical JSON description of analysis settings and source identities. + pub fn new(identity: &impl Serialize) -> Result { + let mut canonical = serde_json::to_value((CACHE_VERSION, identity))?; + canonical.sort_all_objects(); + Ok(Self(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&canonical)?) + ))) + } + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom for DiffKey { + type Error = anyhow::Error; + fn try_from(value: String) -> Result { + ensure!( + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)), + "invalid diff key" + ); + Ok(Self(value)) + } +} +impl From for String { + fn from(key: DiffKey) -> Self { + key.0 + } +} + +/// Serializable computed data. Plugin instances and search highlights are not stored. +/// Fields remain internal; custom stores can serialize/deserialize this value with serde. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StoredDiff { + pub(crate) entry: FileChange, + pub(crate) sources: Pairing, +} + +pub trait Store: Send + Sync { + fn get(&self, key: &DiffKey) -> Result>; + fn put(&self, key: &DiffKey, diff: &StoredDiff) -> Result<()>; + fn remove(&self, key: &DiffKey) -> Result<()>; +} diff --git a/src/storage.rs b/src/storage.rs new file mode 100644 index 000000000..9e1bc0898 --- /dev/null +++ b/src/storage.rs @@ -0,0 +1,170 @@ +//! Built-in computed-diff stores and host configuration. +mod file; +mod memory; +mod sqlite; +use crate::search::store::Store; +use anyhow::Result; +pub use file::FileStore; +pub use memory::MemoryStore; +use serde::{Deserialize, Serialize}; +pub use sqlite::SqliteStore; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +/// Host-owned storage settings, loaded through diffr config. +#[derive(Clone, Debug, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(default, deny_unknown_fields)] +pub(crate) struct StoreConfig { + /// Where computed diffs are kept. Memory lasts for the process; file and SQLite persist. + #[schemars(title = "Diff storage backend", extend("x-group" = "Storage"))] + pub(crate) backend: StoreBackend, + /// Storage directory, relative to the source repository unless absolute. SQLite uses diffs.sqlite inside it. + #[schemars(title = "Diff storage directory", extend("x-group" = "Storage"))] + pub(crate) path: PathBuf, +} +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "lowercase")] +pub(crate) enum StoreBackend { + #[default] + Memory, + File, + Sqlite, +} +impl Default for StoreConfig { + fn default() -> Self { + Self { + backend: StoreBackend::Memory, + path: ".cache/diffr".into(), + } + } +} +impl StoreConfig { + pub(crate) fn open(&self, repository: &Path) -> Result> { + let directory = repository.join(&self.path); + Ok(match self.backend { + StoreBackend::Memory => Arc::new(MemoryStore::default()), + StoreBackend::File => Arc::new(FileStore::open(directory)?), + StoreBackend::Sqlite => Arc::new(SqliteStore::open(directory.join("diffs.sqlite"))?), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::{FileChange, FileRef, FileStatus}; + use crate::{ + pairing::Pairing, + protocol::Source, + search::store::{DiffKey, StoredDiff}, + }; + use rusqlite::Connection; + use std::fs; + + fn sample(text: &str) -> StoredDiff { + StoredDiff { + entry: FileChange { + file: Pairing::RightOnly { + rhs: FileRef { + path: "x.rs".into(), + oid: "abc".into(), + mode: "100644".into(), + }, + }, + status: FileStatus::Added, + tags: vec![], + }, + sources: Pairing::RightOnly { + rhs: Source { + text: text.into(), + syntax: vec![], + regions: vec![], + }, + }, + } + } + fn contract(store: &dyn Store) { + let key = DiffKey::new(&("a.rs", "blob", "config")).unwrap(); + let other = DiffKey::new(&("a.rs", "blob2", "config")).unwrap(); + assert!(store.get(&key).unwrap().is_none()); + let value = sample("hello\n"); + store.put(&key, &value).unwrap(); + assert_eq!( + serde_json::to_value(store.get(&key).unwrap().unwrap()).unwrap(), + serde_json::to_value(value).unwrap() + ); + assert!(store.get(&other).unwrap().is_none()); + store.put(&key, &sample("replacement")).unwrap(); + assert_eq!( + serde_json::to_value(store.get(&key).unwrap().unwrap()).unwrap(), + serde_json::to_value(sample("replacement")).unwrap() + ); + store.remove(&key).unwrap(); + store.remove(&key).unwrap(); + assert!(store.get(&key).unwrap().is_none()); + } + #[test] + fn all_backends_obey_the_same_contract() { + let directory = tempfile::tempdir().unwrap(); + contract(&MemoryStore::default()); + contract(&FileStore::open(directory.path().join("files")).unwrap()); + contract(&SqliteStore::open(directory.path().join("diffs.sqlite")).unwrap()); + } + #[test] + fn durable_stores_reopen_and_report_corruption() { + let directory = tempfile::tempdir().unwrap(); + let key = DiffKey::new(&"identity").unwrap(); + let files = directory.path().join("files"); + FileStore::open(&files) + .unwrap() + .put(&key, &sample("saved")) + .unwrap(); + let reopened = FileStore::open(&files).unwrap(); + assert!(reopened.get(&key).unwrap().is_some()); + fs::write(files.join(format!("{}.json", key.as_str())), b"broken JSON").unwrap(); + assert!(reopened.get(&key).is_err()); + let db = directory.path().join("diffs.sqlite"); + SqliteStore::open(&db) + .unwrap() + .put(&key, &sample("saved")) + .unwrap(); + assert!(SqliteStore::open(&db).unwrap().get(&key).unwrap().is_some()); + Connection::open(&db) + .unwrap() + .execute("UPDATE diffs SET data = ?1", [b"broken JSON".as_slice()]) + .unwrap(); + assert!(SqliteStore::open(&db).unwrap().get(&key).is_err()); + } + #[test] + fn keys_are_canonical_and_reject_path_injection() { + let a: serde_json::Value = serde_json::from_str(r#"{"blob":"a","config":"x"}"#).unwrap(); + let b: serde_json::Value = serde_json::from_str(r#"{"config":"x","blob":"a"}"#).unwrap(); + assert_eq!(DiffKey::new(&a).unwrap(), DiffKey::new(&b).unwrap()); + assert_ne!( + DiffKey::new(&a).unwrap(), + DiffKey::new(&serde_json::json!({"blob":"a","config":"y"})).unwrap() + ); + assert!(serde_json::from_str::(r#""../../outside""#).is_err()); + } + #[test] + fn concurrent_file_writers_publish_whole_records() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new(FileStore::open(directory.path()).unwrap()); + let key = DiffKey::new(&"shared").unwrap(); + store.put(&key, &sample("initial")).unwrap(); + std::thread::scope(|scope| { + for index in 0..4 { + let store = &store; + let key = &key; + scope.spawn(move || { + for _ in 0..10 { + store.put(key, &sample(&index.to_string())).unwrap(); + assert!(store.get(key).unwrap().is_some()); + } + }); + } + }); + } +} diff --git a/src/storage/file.rs b/src/storage/file.rs new file mode 100644 index 000000000..f2c335e55 --- /dev/null +++ b/src/storage/file.rs @@ -0,0 +1,57 @@ +use crate::search::store::{DiffKey, Store, StoredDiff}; +use anyhow::{ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use std::{fs, io::Write, path::PathBuf}; + +pub struct FileStore { + directory: PathBuf, +} +impl FileStore { + pub fn open(directory: impl Into) -> Result { + let directory = directory.into(); + fs::create_dir_all(&directory).context("create diff store directory")?; + Ok(Self { directory }) + } + fn path(&self, key: &DiffKey) -> PathBuf { + self.directory.join(format!("{}.json", key.as_str())) + } +} +#[derive(Serialize, Deserialize)] +struct Record { + key: DiffKey, + diff: StoredDiff, +} +impl Store for FileStore { + fn get(&self, key: &DiffKey) -> Result> { + let bytes = match fs::read(self.path(key)) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error).context("read diff store entry"), + }; + let record: Record = serde_json::from_slice(&bytes).context("decode diff store entry")?; + ensure!(&record.key == key, "diff store key mismatch"); + Ok(Some(record.diff)) + } + fn put(&self, key: &DiffKey, diff: &StoredDiff) -> Result<()> { + let mut file = tempfile::NamedTempFile::new_in(&self.directory)?; + serde_json::to_writer( + &mut file, + &Record { + key: key.clone(), + diff: diff.clone(), + }, + )?; + file.flush()?; + file.as_file().sync_all()?; + file.persist(self.path(key)) + .context("publish diff store entry")?; + Ok(()) + } + fn remove(&self, key: &DiffKey) -> Result<()> { + match fs::remove_file(self.path(key)) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + } +} diff --git a/src/storage/memory.rs b/src/storage/memory.rs new file mode 100644 index 000000000..09e98509a --- /dev/null +++ b/src/storage/memory.rs @@ -0,0 +1,32 @@ +use crate::search::store::{DiffKey, Store, StoredDiff}; +use anyhow::Result; +use std::{collections::HashMap, sync::Mutex}; + +#[derive(Default)] +pub struct MemoryStore { + entries: Mutex>, +} +impl Store for MemoryStore { + fn get(&self, key: &DiffKey) -> Result> { + Ok(self + .entries + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))? + .get(key) + .cloned()) + } + fn put(&self, key: &DiffKey, diff: &StoredDiff) -> Result<()> { + self.entries + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))? + .insert(key.clone(), diff.clone()); + Ok(()) + } + fn remove(&self, key: &DiffKey) -> Result<()> { + self.entries + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))? + .remove(key); + Ok(()) + } +} diff --git a/src/storage/sqlite.rs b/src/storage/sqlite.rs new file mode 100644 index 000000000..ba95aa9e1 --- /dev/null +++ b/src/storage/sqlite.rs @@ -0,0 +1,62 @@ +use crate::search::store::{DiffKey, Store, StoredDiff}; +use anyhow::{Context, Result}; +use rusqlite::{params, Connection, OptionalExtension}; +use std::{fs, path::Path, sync::Mutex}; + +pub struct SqliteStore { + connection: Mutex, +} +impl SqliteStore { + pub fn open(path: impl AsRef) -> Result { + let path = path.as_ref(); + if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + fs::create_dir_all(parent)?; + } + let connection = Connection::open(path)?; + connection.busy_timeout(std::time::Duration::from_secs(5))?; + connection.execute_batch( + "PRAGMA journal_mode=WAL; + CREATE TABLE IF NOT EXISTS diffs (key TEXT PRIMARY KEY, data BLOB NOT NULL);", + )?; + Ok(Self { + connection: Mutex::new(connection), + }) + } +} +impl Store for SqliteStore { + fn get(&self, key: &DiffKey) -> Result> { + let connection = self + .connection + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))?; + let bytes: Option> = connection + .query_row( + "SELECT data FROM diffs WHERE key = ?1", + [key.as_str()], + |row| row.get(0), + ) + .optional()?; + bytes + .map(|bytes| serde_json::from_slice(&bytes).context("decode SQLite diff entry")) + .transpose() + } + fn put(&self, key: &DiffKey, diff: &StoredDiff) -> Result<()> { + let connection = self + .connection + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))?; + connection.execute( + "INSERT INTO diffs(key,data) VALUES (?1,?2) + ON CONFLICT(key) DO UPDATE SET data=excluded.data", + params![key.as_str(), serde_json::to_vec(diff)?], + )?; + Ok(()) + } + fn remove(&self, key: &DiffKey) -> Result<()> { + self.connection + .lock() + .map_err(|_| anyhow::anyhow!("diff store lock poisoned"))? + .execute("DELETE FROM diffs WHERE key = ?1", [key.as_str()])?; + Ok(()) + } +}