diff --git a/crates/autophagy-cli/src/detection.rs b/crates/autophagy-cli/src/detection.rs new file mode 100644 index 0000000..0a065ca --- /dev/null +++ b/crates/autophagy-cli/src/detection.rs @@ -0,0 +1,136 @@ +//! Persisted detection-findings cache and detection progress reporting. +//! +//! Every findings-consuming command (`patterns`, `digest`, `mutations propose`, +//! `mutations synthesize`, `status --with-findings`) runs the same deterministic +//! detection pass over every stored event. On a large corpus that pass costs +//! tens of seconds, so [`detect_cached`] memoizes the report in the store keyed +//! by a content fingerprint of its exact inputs. An unchanged corpus at +//! unchanged thresholds returns instantly; any change to the events, thresholds, +//! project filter, or detector spec version misses and recomputes. When a fresh +//! pass does run, a concise before/after progress line is written to stderr so +//! the command never appears hung — and stdout stays a clean report. + +use std::{io::Write, time::Instant}; + +use autophagy_patterns::{ + DETECTION_SPEC_VERSION, DetectionReport, DetectorConfig, detect_with_report, +}; +use autophagy_store::{DetectionFingerprint, EventStore}; +use sha2::{Digest, Sha256}; + +use crate::CliError; + +/// Return the detection report for `project` at `config`, served from the store +/// cache when the corpus and thresholds are unchanged, otherwise computed fresh +/// and cached. Setting `recompute` forces a fresh pass and refreshes the entry. +/// +/// The returned report is byte-for-byte identical whether it came from the cache +/// or a fresh pass, so every caller — findings, diagnostics, and the exact +/// evidence identifiers each finding carries — sees the same deterministic +/// result. +/// +/// # Errors +/// +/// Returns [`CliError`] when the store cannot be queried or a stored event no +/// longer satisfies the AEP contract. +pub(crate) fn detect_cached( + store: &EventStore, + project: Option<&str>, + config: DetectorConfig, + recompute: bool, +) -> Result { + let fingerprint = store.detection_fingerprint(project)?; + let key = cache_key(project, config, &fingerprint); + if !recompute { + if let Some(json) = store.cached_findings(&key)? { + if let Ok(report) = serde_json::from_str::(&json) { + return Ok(report); + } + // A payload from an older cache shape that no longer deserializes is + // ignored and recomputed below rather than surfaced as an error. + } + } + let report = run_with_progress(store, project, config, &fingerprint)?; + let generation = store.detection_generation()?; + let json = serde_json::to_string(&report)?; + store.store_findings(&key, &generation, &json)?; + Ok(report) +} + +/// Run a fresh detection pass, bracketing it with progress lines on stderr. +fn run_with_progress( + store: &EventStore, + project: Option<&str>, + config: DetectorConfig, + fingerprint: &DetectionFingerprint, +) -> Result { + // Progress goes to stderr; stdout stays a clean report so JSON output mode is + // never polluted. + let mut stderr = std::io::stderr(); + let _ = writeln!( + stderr, + "digesting {} events across {} sessions…", + group(fingerprint.event_count), + group(fingerprint.session_count), + ); + let started = Instant::now(); + let events = store.list_events_for_detection(project)?; + let report = detect_with_report(&events, config); + let _ = writeln!( + stderr, + "digested in {:.1}s — {} finding(s)", + started.elapsed().as_secs_f64(), + report.findings.len(), + ); + Ok(report) +} + +/// Derive the 32-byte cache validity key from every input a detection pass +/// depends on: the detector spec version, the effective thresholds, the project +/// filter, and the cheap content fingerprint (event count, max row id, and the +/// monotonic import watermark). A NUL separator keeps field boundaries +/// unambiguous so distinct inputs cannot collide by concatenation. +fn cache_key( + project: Option<&str>, + config: DetectorConfig, + fingerprint: &DetectionFingerprint, +) -> [u8; 32] { + let min_occurrences = config.min_occurrences.to_string(); + let min_sessions = config.min_sessions.to_string(); + let min_support_ratio_bps = config.min_support_ratio_bps.to_string(); + let event_count = fingerprint.event_count.to_string(); + let max_row_id = fingerprint.max_row_id.to_string(); + let parts: [&str; 8] = [ + DETECTION_SPEC_VERSION, + &min_occurrences, + &min_sessions, + &min_support_ratio_bps, + project.unwrap_or(""), + &event_count, + &max_row_id, + &fingerprint.import_watermark, + ]; + let mut hasher = Sha256::new(); + for part in parts { + hasher.update(part.as_bytes()); + hasher.update([0_u8]); + } + hasher.finalize().into() +} + +/// Group an integer's digits with thousands separators (`69,400`). +fn group(value: i64) -> String { + let digits = value.unsigned_abs().to_string(); + let mut grouped = String::with_capacity(digits.len() + digits.len() / 3); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index) % 3 == 0 { + grouped.push(','); + } + grouped.push(digit); + } + if value < 0 { + format!("-{grouped}") + } else { + grouped + } +} diff --git a/crates/autophagy-cli/src/main.rs b/crates/autophagy-cli/src/main.rs index a3763b7..d7e3030 100644 --- a/crates/autophagy-cli/src/main.rs +++ b/crates/autophagy-cli/src/main.rs @@ -2,6 +2,7 @@ mod config; mod daemon; +mod detection; mod setup; mod status; mod watch; @@ -38,8 +39,7 @@ use autophagy_install::{ }; use autophagy_mutations::{GenerationOutcome, equivalence_key, generate_candidates}; use autophagy_patterns::{ - DetectionDiagnostics, DetectionReport, DetectorConfig, EvidencePacket, Observation, detect, - detect_with_report, + DetectionDiagnostics, DetectionReport, DetectorConfig, EvidencePacket, Observation, }; use autophagy_replay::{ CounterfactualOutcome, ExpectedAction, ReplayDraftError, ReplayEvaluationError, ReplayReport, @@ -216,6 +216,10 @@ enum Commands { #[command(flatten)] thresholds: ThresholdArgs, + + /// Ignore any cached findings and run a fresh detection pass. + #[arg(long)] + recompute: bool, }, /// List deterministic Evidence Packet v0.1 findings. @@ -226,6 +230,10 @@ enum Commands { #[command(flatten)] thresholds: ThresholdArgs, + + /// Ignore any cached findings and run a fresh detection pass. + #[arg(long)] + recompute: bool, }, /// Manage review-only, zero-permission mutation candidates. @@ -1275,27 +1283,31 @@ fn execute( Commands::Digest { project, thresholds, + recompute, } => { let database = resolve_database_path(cli.database)?; let store = open_store(&database)?; - let events = store.list_events_for_detection(project.as_deref())?; - let report = detect_with_report( - &events, + let report = detection::detect_cached( + &store, + project.as_deref(), config::resolve_thresholds(leaf, thresholds, config), - ); + recompute, + )?; Ok(CommandReport::Digest(digest_report(report)?)) } Commands::Patterns { project, thresholds, + recompute, } => { let database = resolve_database_path(cli.database)?; let store = open_store(&database)?; - let events = store.list_events_for_detection(project.as_deref())?; - let report = detect_with_report( - &events, + let report = detection::detect_cached( + &store, + project.as_deref(), config::resolve_thresholds(leaf, thresholds, config), - ); + recompute, + )?; let DetectionDiagnostics { events_scanned, sessions_scanned, @@ -1492,11 +1504,13 @@ fn execute_mutation_action( thresholds, dry_run, } => { - let events = store.list_events_for_detection(project.as_deref())?; - let findings = detect( - &events, + let findings = detection::detect_cached( + &store, + project.as_deref(), config::resolve_thresholds(matches, thresholds, config), - ); + false, + )? + .findings; let generated = generate_candidates(&findings); let mut registrations = Vec::new(); if !dry_run { @@ -1613,11 +1627,13 @@ fn execute_mutation_action( } } } - let events = store.list_events_for_detection(project.as_deref())?; - let findings = detect( - &events, + let findings = detection::detect_cached( + &store, + project.as_deref(), config::resolve_thresholds(matches, thresholds, config), - ); + false, + )? + .findings; let synthesized = synthesize_candidates(&findings, &model_manifest, synthesis_provider.as_ref()); let (total_prompt_tokens, total_completion_tokens) = aggregate_usage(&synthesized); diff --git a/crates/autophagy-cli/src/setup.rs b/crates/autophagy-cli/src/setup.rs index dbc15c4..8898fd5 100644 --- a/crates/autophagy-cli/src/setup.rs +++ b/crates/autophagy-cli/src/setup.rs @@ -31,7 +31,7 @@ use autophagy_adapter_pi::{ PiImportOptions, default_sessions_root as pi_sessions_root, discover as pi_discover, import_pi, }; use autophagy_core::{ReindexOptions, ReindexSummary, reindex}; -use autophagy_patterns::{DetectorConfig, detect_with_report}; +use autophagy_patterns::DetectorConfig; use autophagy_store::EventStore; use clap::ValueEnum; use serde::Serialize; @@ -351,8 +351,12 @@ pub fn run( // still shows the scan stats and near-threshold observations rather than a // silent nothing. ui.say(""); - let events = store.list_events_for_detection(None)?; - let digest = digest_report(detect_with_report(&events, DetectorConfig::default()))?; + let digest = digest_report(crate::detection::detect_cached( + &store, + None, + DetectorConfig::default(), + false, + )?)?; let digest_events_scanned = digest.events_scanned; let digest_findings = digest.findings.len(); let digest_observations = digest.observations.len(); diff --git a/crates/autophagy-cli/src/status.rs b/crates/autophagy-cli/src/status.rs index ec8c013..02452d2 100644 --- a/crates/autophagy-cli/src/status.rs +++ b/crates/autophagy-cli/src/status.rs @@ -9,7 +9,6 @@ use std::{collections::BTreeMap, io::Write, path::PathBuf}; -use autophagy_patterns::detect; use serde::Serialize; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; @@ -142,14 +141,19 @@ pub fn run( }) .collect(); - // Thresholds are cheap (config or defaults). Findings are opt-in: computing - // them means loading and deserializing every event and running a full - // detection pass — digest-cost on a large store — so `status` stays fast by - // default and only pays that cost when `--with-findings` is passed. + // Thresholds are cheap (config or defaults). Findings are opt-in: the first + // pass at a given corpus and thresholds loads and deserializes every event + // and runs a full detection pass — digest-cost on a large store — so + // `status` stays fast by default and only pays that cost when + // `--with-findings` is passed. The result is cached in the store, so + // repeats at an unchanged corpus are instant. let detector = config.detector_config(); let findings = if with_findings { - let events = store.list_events_for_detection(None)?; - Some(detect(&events, detector).len()) + Some( + crate::detection::detect_cached(&store, None, detector, false)? + .findings + .len(), + ) } else { None }; diff --git a/crates/autophagy-cli/tests/cli.rs b/crates/autophagy-cli/tests/cli.rs index 5a82fec..30631bf 100644 --- a/crates/autophagy-cli/tests/cli.rs +++ b/crates/autophagy-cli/tests/cli.rs @@ -1638,6 +1638,102 @@ fn run_watch_summary(database: &Path, claude_config_dir: &Path) -> Value { serde_json::from_str(last).expect("summary JSON") } +#[test] +fn detection_findings_are_cached_and_report_progress() { + let directory = tempfile::tempdir().expect("temporary directory"); + let indexable = directory.path().join("indexable.jsonl"); + let database = directory.path().join("autophagy.db"); + fs::write(&indexable, INDEXABLE_JSONL).expect("write fixture"); + + run_json( + &database, + [ + "import", + indexable.to_str().expect("UTF-8 path"), + "--instance-key", + "fixture:cache", + "--index-tool-input", + ], + ); + + // Run `patterns`, returning the parsed report and captured stderr so we can + // observe whether a fresh detection pass actually ran (it prints a progress + // line to stderr) versus being served from the cache (silent). + let run = |args: &[&str]| -> (Value, String) { + let output = command(&database) + .args(["--output", "json"]) + .args(args) + .output() + .expect("run command"); + assert!( + output.status.success(), + "command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + ( + serde_json::from_slice(&output.stdout).expect("JSON output"), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) + }; + + // First pass computes fresh and reports progress. + let (first, first_stderr) = run(&["patterns"]); + assert!( + first_stderr.contains("digesting"), + "first pass should report detection progress: {first_stderr}" + ); + + // A second identical pass is a cache hit: byte-for-byte identical findings + // (exact evidence IDs included) and no fresh detection pass. + let (second, second_stderr) = run(&["patterns"]); + assert_eq!( + first["result"], second["result"], + "cache hit must return identical findings" + ); + assert!( + !second_stderr.contains("digesting"), + "an unchanged corpus at unchanged thresholds must not recompute: {second_stderr}" + ); + + // Changing thresholds invalidates the key, forcing a fresh pass. + let (_, retuned_stderr) = run(&["patterns", "--min-occurrences", "1"]); + assert!( + retuned_stderr.contains("digesting"), + "a threshold change must invalidate the cache: {retuned_stderr}" + ); + + // --recompute forces a fresh pass even on an otherwise-cached key. + let (_, forced_stderr) = run(&["patterns", "--recompute"]); + assert!( + forced_stderr.contains("digesting"), + "--recompute must always run a fresh pass: {forced_stderr}" + ); + + // A new import changes the corpus and invalidates the cache. + let more = directory.path().join("more.jsonl"); + fs::write(&more, VALID_JSONL).expect("write second fixture"); + run_json( + &database, + [ + "import", + more.to_str().expect("UTF-8 path"), + "--instance-key", + "fixture:cache-2", + ], + ); + let (_, after_import_stderr) = run(&["patterns"]); + assert!( + after_import_stderr.contains("digesting"), + "a new import must invalidate the cache: {after_import_stderr}" + ); + // The corpus is stable again, so the next identical pass is a hit. + let (_, repeat_stderr) = run(&["patterns"]); + assert!( + !repeat_stderr.contains("digesting"), + "the cache should be warm again after the invalidating import: {repeat_stderr}" + ); +} + fn run_json(database: &Path, args: [&str; N]) -> Value { let output = command(database) .args(["--output", "json"]) diff --git a/crates/autophagy-patterns/src/lib.rs b/crates/autophagy-patterns/src/lib.rs index 044acf0..123b262 100644 --- a/crates/autophagy-patterns/src/lib.rs +++ b/crates/autophagy-patterns/src/lib.rs @@ -23,6 +23,14 @@ use autophagy_events::Event; /// Maximum near-threshold observations retained per detection pass. const OBSERVATION_LIMIT: usize = 5; +/// Version tag for the deterministic detection and signature-normalization +/// contract, folded into the persisted findings-cache key. +/// +/// Bump this whenever detector output or the signature normalization in +/// `autophagy-events` changes, so every previously persisted cache entry is +/// automatically treated as stale rather than served from an outdated pass. +pub const DETECTION_SPEC_VERSION: &str = "detection/v1"; + /// Thresholds shared by deterministic recurrence detectors. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct DetectorConfig { diff --git a/crates/autophagy-store/migrations/0002_findings_cache.sql b/crates/autophagy-store/migrations/0002_findings_cache.sql new file mode 100644 index 0000000..c2e7870 --- /dev/null +++ b/crates/autophagy-store/migrations/0002_findings_cache.sql @@ -0,0 +1,23 @@ +-- Derived detection-findings cache (v2). +-- +-- Every findings-consuming command (patterns, digest, mutations propose, +-- mutations synthesize, status --with-findings) otherwise re-runs a full +-- deterministic detection pass over every stored event on each invocation — +-- tens of seconds of silent recomputation on a large corpus. This table +-- memoizes the serialized detection report keyed by a content fingerprint of its +-- exact inputs, so an unchanged corpus at unchanged thresholds answers instantly +-- while any change to the events, thresholds, project filter, or detector spec +-- produces a different key and misses. +-- +-- The cache is derived, deterministic, and fully reconstructable at any time by +-- deleting every row: it holds no new source text, only the exact evidence +-- identifiers the detectors already produce. See ADR 0013. It is ordered after +-- the v1 baseline and, like every migration from the first release onward, is +-- immutable — add new migrations, never edit this one. + +CREATE TABLE findings_cache ( + cache_key BLOB PRIMARY KEY CHECK (length(cache_key) = 32), + generation BLOB NOT NULL CHECK (length(generation) = 32), + report_json TEXT NOT NULL CHECK (json_valid(report_json)), + created_at TEXT NOT NULL +) STRICT; diff --git a/crates/autophagy-store/src/lib.rs b/crates/autophagy-store/src/lib.rs index 6204899..142f135 100644 --- a/crates/autophagy-store/src/lib.rs +++ b/crates/autophagy-store/src/lib.rs @@ -12,13 +12,14 @@ mod util; pub use error::StoreError; pub use model::{ - AdapterActivity, DeleteAllSummary, DeleteSummary, InsertOutcome, InstallationRegistration, - InstallationTransitionOutcome, MutationDetails, MutationInstallationRecord, MutationRecord, - MutationRegisterOutcome, MutationRegistration, MutationReplayRecord, MutationShadowRecord, - MutationTransition, MutationTransitionOutcome, PruneSummary, RankingExplanation, RankingSignal, - RankingSignalKind, RebuildSummary, ReplayRegisterOutcome, ReplayRegistration, RetrievalFilter, - RetrievalFilterField, RetrievalHit, RetrievalMatchKind, RetrievalOutcome, RetrievalQuery, - SearchHit, SearchProjection, SessionSummary, ShadowRegisterOutcome, ShadowRegistration, - SourceCursor, SourceIdentity, StoreStats, + AdapterActivity, DeleteAllSummary, DeleteSummary, DetectionFingerprint, InsertOutcome, + InstallationRegistration, InstallationTransitionOutcome, MutationDetails, + MutationInstallationRecord, MutationRecord, MutationRegisterOutcome, MutationRegistration, + MutationReplayRecord, MutationShadowRecord, MutationTransition, MutationTransitionOutcome, + PruneSummary, RankingExplanation, RankingSignal, RankingSignalKind, RebuildSummary, + ReplayRegisterOutcome, ReplayRegistration, RetrievalFilter, RetrievalFilterField, RetrievalHit, + RetrievalMatchKind, RetrievalOutcome, RetrievalQuery, SearchHit, SearchProjection, + SessionSummary, ShadowRegisterOutcome, ShadowRegistration, SourceCursor, SourceIdentity, + StoreStats, }; pub use store::EventStore; diff --git a/crates/autophagy-store/src/migration.rs b/crates/autophagy-store/src/migration.rs index 4536a29..fa91379 100644 --- a/crates/autophagy-store/src/migration.rs +++ b/crates/autophagy-store/src/migration.rs @@ -26,11 +26,18 @@ struct Migration { /// The baseline is schema-identical to that chain's final state, proven by /// `tests/schema_equivalence.rs`. From the first release onward this chain is /// ordered and immutable — add new migrations, never edit an applied one. -const MIGRATIONS: &[Migration] = &[Migration { - version: 1, - description: "initial schema", - sql: include_str!("../migrations/0001_initial_schema.sql"), -}]; +const MIGRATIONS: &[Migration] = &[ + Migration { + version: 1, + description: "initial schema", + sql: include_str!("../migrations/0001_initial_schema.sql"), + }, + Migration { + version: 2, + description: "derived detection-findings cache", + sql: include_str!("../migrations/0002_findings_cache.sql"), + }, +]; /// SHA-256 checksums of the eight pre-release development migrations, in order /// (v1..=v8). Exactly one database was ever built by that chain — the author's @@ -217,22 +224,23 @@ mod tests { use crate::{StoreError, util}; #[test] - fn fresh_database_applies_the_v1_baseline() { + fn fresh_database_applies_the_full_chain() { let mut connection = Connection::open_in_memory().expect("database"); apply(&mut connection).expect("initial migration"); + let latest = MIGRATIONS.last().expect("migration").version; assert_eq!( connection .pragma_query_value(None, "user_version", |row| row.get::<_, i64>(0)) .expect("schema version"), - 1 + latest ); assert_eq!( connection .query_row("SELECT count(*) FROM schema_migrations", [], |row| row .get::<_, i64>(0)) .expect("ledger rows"), - 1 + i64::try_from(MIGRATIONS.len()).expect("chain length fits i64") ); // A representative table from the squashed tail exists at v1. assert!( @@ -248,7 +256,7 @@ mod tests { } #[test] - fn reapplying_the_baseline_is_a_no_op() { + fn reapplying_the_chain_is_a_no_op() { let mut connection = Connection::open_in_memory().expect("database"); apply(&mut connection).expect("initial migration"); apply(&mut connection).expect("second apply"); @@ -258,7 +266,7 @@ mod tests { .query_row("SELECT count(*) FROM schema_migrations", [], |row| row .get::<_, i64>(0)) .expect("ledger rows"), - 1 + i64::try_from(MIGRATIONS.len()).expect("chain length fits i64") ); } diff --git a/crates/autophagy-store/src/model.rs b/crates/autophagy-store/src/model.rs index 34f9883..a089ad6 100644 --- a/crates/autophagy-store/src/model.rs +++ b/crates/autophagy-store/src/model.rs @@ -328,6 +328,27 @@ pub struct StoreStats { pub conflicts: i64, } +/// Cheap content fingerprint of the events a detection pass would scan. +/// +/// Computed from indexed columns only — never by deserializing event JSON — so +/// it is orders of magnitude cheaper than a detection pass. Any import, delete, +/// or prune changes at least one field, which is what lets the derived +/// findings cache key make an unchanged corpus a hit and a changed corpus a +/// miss without explicit invalidation. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +pub struct DetectionFingerprint { + /// Events in scope (after the optional project filter). + pub event_count: i64, + /// Highest `row_id` in scope, or zero when empty. + pub max_row_id: i64, + /// Distinct sessions in scope, for human-facing progress only. + pub session_count: i64, + /// Latest `imported_at` in scope — a monotonic import watermark that + /// advances on every insert, distinguishing a delete-then-reimport from the + /// original corpus even when the count and max row id happen to coincide. + pub import_watermark: String, +} + /// Per-adapter import activity, for status reporting. #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] pub struct AdapterActivity { diff --git a/crates/autophagy-store/src/store.rs b/crates/autophagy-store/src/store.rs index 28d72ed..bd2f99d 100644 --- a/crates/autophagy-store/src/store.rs +++ b/crates/autophagy-store/src/store.rs @@ -8,14 +8,15 @@ use rusqlite::{ use time::OffsetDateTime; use crate::{ - AdapterActivity, DeleteAllSummary, DeleteSummary, InsertOutcome, InstallationRegistration, - InstallationTransitionOutcome, MutationDetails, MutationInstallationRecord, MutationRecord, - MutationRegisterOutcome, MutationRegistration, MutationReplayRecord, MutationShadowRecord, - MutationTransition, MutationTransitionOutcome, PruneSummary, RankingExplanation, RankingSignal, - RankingSignalKind, RebuildSummary, ReplayRegisterOutcome, ReplayRegistration, RetrievalFilter, - RetrievalFilterField, RetrievalHit, RetrievalMatchKind, RetrievalQuery, SearchHit, - SearchProjection, SessionSummary, ShadowRegisterOutcome, ShadowRegistration, SourceCursor, - SourceIdentity, StoreError, StoreStats, migration, util, + AdapterActivity, DeleteAllSummary, DeleteSummary, DetectionFingerprint, InsertOutcome, + InstallationRegistration, InstallationTransitionOutcome, MutationDetails, + MutationInstallationRecord, MutationRecord, MutationRegisterOutcome, MutationRegistration, + MutationReplayRecord, MutationShadowRecord, MutationTransition, MutationTransitionOutcome, + PruneSummary, RankingExplanation, RankingSignal, RankingSignalKind, RebuildSummary, + ReplayRegisterOutcome, ReplayRegistration, RetrievalFilter, RetrievalFilterField, RetrievalHit, + RetrievalMatchKind, RetrievalQuery, SearchHit, SearchProjection, SessionSummary, + ShadowRegisterOutcome, ShadowRegistration, SourceCursor, SourceIdentity, StoreError, + StoreStats, migration, util, }; /// Score contribution for an exact normalized-signature match, in basis points. @@ -424,6 +425,119 @@ impl EventStore { .collect() } + /// Cheap content fingerprint of the events a detection pass would scan under + /// the same optional project filter as [`Self::list_events_for_detection`]. + /// + /// Computed from indexed columns only, never by deserializing event JSON, so + /// it is orders of magnitude cheaper than detection itself. The caller folds + /// it, the effective thresholds, the project filter, and the detector spec + /// version into the [`findings cache`](Self::cached_findings) key; any event + /// change moves at least one field and so misses without explicit + /// invalidation. + /// + /// # Errors + /// + /// Returns [`StoreError`] when `SQLite` cannot execute the query. + pub fn detection_fingerprint( + &self, + project: Option<&str>, + ) -> Result { + Ok(self.connection.query_row( + "SELECT + count(*), + coalesce(max(row_id), 0), + count(DISTINCT session_id), + coalesce(max(imported_at), '') + FROM events + WHERE (?1 IS NULL OR project_path = ?1)", + [project], + |row| { + Ok(DetectionFingerprint { + event_count: row.get(0)?, + max_row_id: row.get(1)?, + session_count: row.get(2)?, + import_watermark: row.get(3)?, + }) + }, + )?) + } + + /// Global corpus generation stamp: the whole-store fingerprint hashed to 32 + /// bytes. Written alongside each cache entry so a write can garbage-collect + /// every entry from a prior corpus state in one statement, keeping the cache + /// bounded regardless of how many threshold or project variants were stored. + /// + /// # Errors + /// + /// Returns [`StoreError`] when `SQLite` cannot execute the query. + pub fn detection_generation(&self) -> Result<[u8; 32], StoreError> { + let fingerprint = self.detection_fingerprint(None)?; + let material = format!( + "generation/v1\0{}\0{}\0{}", + fingerprint.event_count, fingerprint.max_row_id, fingerprint.import_watermark + ); + Ok(util::sha256(material.as_bytes())) + } + + /// Look up a previously stored detection report by its exact validity key. + /// + /// Returns the serialized report the caller stored under `cache_key`, or + /// `None` on a miss. The key is the caller's sole validity contract: the + /// store treats the payload as opaque JSON and never interprets it, so the + /// cache stays independent of the detector types it memoizes. + /// + /// # Errors + /// + /// Returns [`StoreError`] when `SQLite` cannot execute the query. + pub fn cached_findings(&self, cache_key: &[u8; 32]) -> Result, StoreError> { + Ok(self + .connection + .query_row( + "SELECT report_json FROM findings_cache WHERE cache_key = ?1", + [cache_key.as_slice()], + |row| row.get::<_, String>(0), + ) + .optional()?) + } + + /// Store a serialized detection report under its validity `cache_key`, + /// tagged with the current corpus `generation`. + /// + /// The write first drops every entry from an older generation, then inserts + /// (or replaces) this one, all in one immediate transaction. Concurrent + /// current-generation entries at other thresholds or project filters + /// survive; only stale generations are collected. + /// + /// # Errors + /// + /// Returns [`StoreError`] when a transactional `SQLite` operation fails. + pub fn store_findings( + &self, + cache_key: &[u8; 32], + generation: &[u8; 32], + report_json: &str, + ) -> Result<(), StoreError> { + let created_at = util::now_timestamp()?; + let transaction = self.connection.unchecked_transaction()?; + transaction.execute( + "DELETE FROM findings_cache WHERE generation <> ?1", + [generation.as_slice()], + )?; + transaction.execute( + "INSERT OR REPLACE INTO + findings_cache(cache_key, generation, report_json, created_at) + VALUES (?1, ?2, ?3, ?4)", + params![ + cache_key.as_slice(), + generation.as_slice(), + report_json, + created_at + ], + )?; + transaction.commit()?; + Ok(()) + } + /// Number of rows currently in the exact normalized-signature index. /// /// Zero against a non-empty `events` table means the retrieval index was diff --git a/crates/autophagy-store/tests/legacy_adoption.rs b/crates/autophagy-store/tests/legacy_adoption.rs index 8493c23..729d9a6 100644 --- a/crates/autophagy-store/tests/legacy_adoption.rs +++ b/crates/autophagy-store/tests/legacy_adoption.rs @@ -99,16 +99,18 @@ fn known_legacy_v8_database_is_adopted_and_data_preserved() { "precondition: legacy v8 ledger" ); - // First open through the store adopts the baseline. + // First open through the store adopts the baseline, collapsing the eight + // legacy rows to the v1 baseline, then applies the post-baseline chain (v2) + // on top — landing on the current released schema. { let store = EventStore::open(&path).expect("open adopts legacy database"); - assert_eq!(store.schema_version().expect("schema version"), 1); + assert_eq!(store.schema_version().expect("schema version"), 2); } assert_eq!( ledger_state(&path), - (1, 1, 1), - "ledger collapsed to a single v1 baseline row and user_version reset" + (2, 2, 2), + "legacy chain replaced by the released baseline chain and user_version reset" ); assert_eq!( data_counts(&path), @@ -116,12 +118,12 @@ fn known_legacy_v8_database_is_adopted_and_data_preserved() { "all events, sessions, candidates, evidence, and audit rows preserved" ); - // Second open is a pure no-op: still v1, still one ledger row, data intact. + // Second open is a pure no-op: same schema, same ledger, data intact. { let store = EventStore::open(&path).expect("second open is a no-op"); - assert_eq!(store.schema_version().expect("schema version"), 1); + assert_eq!(store.schema_version().expect("schema version"), 2); } - assert_eq!(ledger_state(&path), (1, 1, 1)); + assert_eq!(ledger_state(&path), (2, 2, 2)); assert_eq!(data_counts(&path), before); } diff --git a/crates/autophagy-store/tests/store.rs b/crates/autophagy-store/tests/store.rs index 39bebbb..ba8b461 100644 --- a/crates/autophagy-store/tests/store.rs +++ b/crates/autophagy-store/tests/store.rs @@ -30,7 +30,7 @@ fn migrations_persist_and_reopen_cleanly() { { let mut store = EventStore::open(&database).expect("store should open"); - assert_eq!(store.schema_version().expect("schema version"), 1); + assert_eq!(store.schema_version().expect("schema version"), 2); assert!(matches!( store .insert_event(&source, &event, &SearchProjection::default()) @@ -40,7 +40,7 @@ fn migrations_persist_and_reopen_cleanly() { } let reopened = EventStore::open(&database).expect("store should reopen"); - assert_eq!(reopened.schema_version().expect("schema version"), 1); + assert_eq!(reopened.schema_version().expect("schema version"), 2); assert_eq!( reopened .get_event(event.event_id.as_str()) @@ -956,6 +956,97 @@ fn claude_code_installation_registers_audits_and_reverses() { assert_eq!(retire.reason, "Claude Code repo skill uninstalled"); } +#[test] +fn detection_fingerprint_tracks_the_scanned_corpus() { + let mut store = EventStore::open_in_memory().expect("store"); + let source = source("fingerprint"); + + let empty = store + .detection_fingerprint(None) + .expect("empty fingerprint"); + assert_eq!(empty.event_count, 0); + assert_eq!(empty.max_row_id, 0); + assert_eq!(empty.session_count, 0); + assert_eq!(empty.import_watermark, ""); + + let start = session_event( + "evt_fp_start", + "ses_fp", + EventKind::SessionStarted, + "2026-07-16T10:00:00Z", + 0, + ); + store + .insert_event(&source, &start, &SearchProjection::default()) + .expect("insert start"); + let failure = tool_failure("evt_fp_fail", "ses_fp", "2026-07-16T10:01:00Z", 1); + store + .insert_event(&source, &failure, &SearchProjection::default()) + .expect("insert failure"); + + let after = store.detection_fingerprint(None).expect("fingerprint"); + assert_eq!(after.event_count, 2, "both events counted"); + assert_eq!(after.max_row_id, 2, "highest row id advances"); + assert_eq!(after.session_count, 1, "one distinct session"); + assert!( + !after.import_watermark.is_empty(), + "import watermark set once events exist" + ); + + // The project filter scopes the fingerprint to matching events only. + let scoped = store + .detection_fingerprint(Some("/nonexistent")) + .expect("scoped fingerprint"); + assert_eq!(scoped.event_count, 0); +} + +#[test] +fn findings_cache_round_trips_and_collects_stale_generations() { + let store = EventStore::open_in_memory().expect("store"); + let key_a = [1_u8; 32]; + let key_b = [2_u8; 32]; + let generation_one = [9_u8; 32]; + + assert_eq!( + store.cached_findings(&key_a).expect("initial miss"), + None, + "an unwritten key misses" + ); + + store + .store_findings(&key_a, &generation_one, "{\"findings\":[]}") + .expect("store a"); + store + .store_findings(&key_b, &generation_one, "{\"findings\":[1]}") + .expect("store b"); + assert_eq!( + store.cached_findings(&key_a).expect("hit a").as_deref(), + Some("{\"findings\":[]}") + ); + assert_eq!( + store.cached_findings(&key_b).expect("hit b").as_deref(), + Some("{\"findings\":[1]}"), + "concurrent current-generation entries coexist" + ); + + // Writing under a new generation collects every prior-generation entry. + let key_c = [3_u8; 32]; + let generation_two = [8_u8; 32]; + store + .store_findings(&key_c, &generation_two, "{\"findings\":[2]}") + .expect("store c"); + assert_eq!( + store.cached_findings(&key_a).expect("evicted a"), + None, + "stale-generation entries are collected" + ); + assert_eq!(store.cached_findings(&key_b).expect("evicted b"), None); + assert_eq!( + store.cached_findings(&key_c).expect("hit c").as_deref(), + Some("{\"findings\":[2]}") + ); +} + fn source(instance_key: &str) -> SourceIdentity { SourceIdentity::new("codex", instance_key).with_display_name("Codex") } diff --git a/docs/architecture/database-schema.md b/docs/architecture/database-schema.md index 763f8b8..afc2d31 100644 --- a/docs/architecture/database-schema.md +++ b/docs/architecture/database-schema.md @@ -14,8 +14,9 @@ release, while no external database existed (see ADR 0012). From the first release onward the chain is ordered and immutable — new migrations are added, never edited. The squash is schema-identical to the old chain's final state, proven by `tests/schema_equivalence.rs`, and the single legacy database is -adopted to the v1 ledger in place on first open. The logical schema and its -trust boundaries are summarized here. +adopted to the v1 ledger in place on first open. Migration `0002` then adds the +derived `findings_cache` table (see ADR 0013). The logical schema and its trust +boundaries are summarized here. ```sql CREATE TABLE schema_migrations ( @@ -259,6 +260,14 @@ CREATE TABLE event_signatures ( CREATE INDEX event_signatures_lookup ON event_signatures(signature, event_row_id); + +-- Migration 0002: derived detection-findings cache (see ADR 0013). +CREATE TABLE findings_cache ( + cache_key BLOB PRIMARY KEY CHECK (length(cache_key) = 32), + generation BLOB NOT NULL CHECK (length(generation) = 32), + report_json TEXT NOT NULL CHECK (json_valid(report_json)), + created_at TEXT NOT NULL +) STRICT; ``` Insert, update, and delete triggers keep the external-content FTS5 table aligned @@ -277,6 +286,19 @@ only when the source's tool input is approved for indexing. Rows cascade on even deletion, so quarantine, prune, and delete-all keep the index consistent without a separate deletion path. +`findings_cache` (migration 0002) memoizes the deterministic detection report so +the findings-consuming commands do not re-run a full detection pass over every +event on each invocation. It is derived, deterministic, local-only data holding +no new source text — only the serialized report, whose findings carry the exact +evidence identifiers the detectors already produce. `cache_key` is a SHA-256 over +every input the pass depends on (detector spec version, effective thresholds, +project filter, and a cheap content fingerprint of the events in scope: event +count, max `row_id`, and the monotonic max `imported_at` watermark), so any +import, delete, or prune changes the key and misses without explicit +invalidation. `generation` tags each row with the global corpus state so a write +collects entries from superseded states. The cache is fully reconstructable at +any time by deleting every row; see ADR 0013. + ## Idempotency 1. Adapters preserve a native stable identifier when one exists; otherwise they diff --git a/docs/decisions/0013-detection-findings-cache.md b/docs/decisions/0013-detection-findings-cache.md new file mode 100644 index 0000000..01e16aa --- /dev/null +++ b/docs/decisions/0013-detection-findings-cache.md @@ -0,0 +1,71 @@ +# ADR 0013: Cache deterministic detection findings in the store + +- Status: accepted +- Date: 2026-07-17 + +## Context + +Every findings-consuming command re-runs the full deterministic detection pass +over every stored event on each invocation: `patterns`, `digest`, +`mutations propose`, `mutations synthesize`, and `status --with-findings`. The +pass loads and deserializes each event's canonical JSON and runs all three +recurrence detectors. On a 69,400-event database that is roughly 45 seconds of +work, repeated in full every time, with no output until it finishes — the +command appears hung. Detection is a pure, deterministic function of its inputs, +so recomputing an unchanged result is wasted work. + +The stored-schema constraint applies: this adds a table, so it takes an ordered, +immutable migration and this record. Two engineering constraints bound the +design — the default path stays local and offline, and every derived finding +must retain its exact evidence identifiers. + +## Decision + +Add a `findings_cache` table (migration `0002`) that memoizes the serialized +detection report, and route all five call sites through it. + +- **Content-addressed validity, no explicit invalidation.** The cache key is a + SHA-256 over every input the pass depends on: the detector/signature spec + version (`DETECTION_SPEC_VERSION`), the effective `DetectorConfig` thresholds + (`min_occurrences`, `min_sessions`, `min_support_ratio_bps`), the project + filter, and a cheap content fingerprint of the events in scope — the event + count, the maximum `row_id`, and the maximum `imported_at` (a monotonic import + watermark that advances on every insert, distinguishing a delete-then-reimport + from the original corpus even when count and max row id coincide). The + fingerprint is computed from indexed columns only, never by deserializing event + JSON, so the validity check is orders of magnitude cheaper than the pass it + guards. Any import, delete, or prune moves at least one field, so a stale key + simply misses; nothing has to remember to invalidate. A reindex changes only + the derived search projection, not the events detection reads, so it correctly + leaves the cache valid. Bumping `DETECTION_SPEC_VERSION` invalidates every + entry, so a future signature-normalization change cannot serve an outdated + report. +- **Bounded growth by generation.** Each row is tagged with a global corpus + generation stamp. A write first drops every entry from an older generation, + then inserts its own, so entries from superseded corpus states never + accumulate while current-generation entries at different thresholds or projects + coexist. +- **Store stays type-agnostic.** Detection types live in `autophagy-patterns`, + which depends on the store, so the store must not depend on them. It treats the + cached payload as opaque JSON and exposes only fingerprint, generation, and + get/put primitives; the CLI — which sees both crates — computes the key and + serializes the report. This preserves the one-way dependency direction. +- **Minimal surface.** `patterns` and `digest` gain a `--recompute` flag to force + a fresh pass and refresh the entry; the other three call sites always read + through the cache. When a fresh pass does run, a concise before/after progress + line is written to stderr (never stdout), so a long pass is visible and JSON + output stays clean. + +This is the first migration added after the `0001` baseline. The ADR 0012 +pre-release adoption shim still runs first on open: a recognized legacy database +is adopted to the v1 baseline and then this v2 migration applies on top. Removing +that shim remains future work, out of scope here. + +## Privacy + +The cache is derived, deterministic, local-only data. It stores no new source +text — only the serialized detection report, whose findings carry the exact +evidence identifiers the detectors already produce from events already on disk. +It never leaves the machine, adds no network path, and is fully reconstructable +at any time by deleting every row (or passing `--recompute`). No secret or raw +cloud payload is persisted.