diff --git a/crates/autophagy-cli/src/main.rs b/crates/autophagy-cli/src/main.rs index 70e9a65..7c593ee 100644 --- a/crates/autophagy-cli/src/main.rs +++ b/crates/autophagy-cli/src/main.rs @@ -1491,6 +1491,11 @@ const fn efficacy_reason_phrase(reason: InsufficientReason) -> &'static str { InsufficientReason::PostWindowTooShort => "a longer post-install window", InsufficientReason::SparseOccurrences => "more observed occurrences", InsufficientReason::PartialIndexCoverage => "fuller command-index coverage", + // Rendered as its own actionable sentence by + // `selector_grammar_mismatch_phrase`, never folded into a "needs …" list. + InsufficientReason::SelectorGrammarMismatch => { + "a selector minted under the current signature grammar" + } } } @@ -1515,17 +1520,65 @@ fn efficacy_summary_line(report: &EfficacyReport) -> String { percent(report.coverage.coverage_bps) )); if report.verdict == Verdict::InsufficientData && !report.insufficient_reasons.is_empty() { - let needs = report + // A grammar mismatch is the dominant, actionable cause: the selector + // matches zero rows, so the other reasons (sparse, short window) are just + // consequences of that blindness. Report only the mismatch sentence. + if report .insufficient_reasons - .iter() - .map(|reason| efficacy_reason_phrase(*reason)) - .collect::>() - .join(", "); - parts.push(format!("needs {needs}")); + .contains(&InsufficientReason::SelectorGrammarMismatch) + { + parts.push(selector_grammar_mismatch_phrase( + &report.signature_selectors, + )); + } else { + let needs = report + .insufficient_reasons + .iter() + .map(|reason| efficacy_reason_phrase(*reason)) + .collect::>() + .join(", "); + parts.push(format!("needs {needs}")); + } } parts.join(" · ") } +/// The actionable sentence for a [`InsufficientReason::SelectorGrammarMismatch`]: +/// which grammar the selector uses, which grammar the index now carries, and +/// what to do about it. The index grammar is the current signature spec version +/// (`reindex --index-tool-input` re-mints every row under it). +fn selector_grammar_mismatch_phrase(selectors: &[String]) -> String { + let index_grammar = autophagy_events::signature::SIGNATURE_SPEC_VERSION; + let mut selector_grammars = selectors + .iter() + .filter_map(|selector| selector_grammar_token(selector)) + .filter(|grammar| *grammar != index_grammar) + .collect::>(); + selector_grammars.sort_unstable(); + selector_grammars.dedup(); + let selector_grammar = if selector_grammars.is_empty() { + "an older version".to_owned() + } else { + selector_grammars.join("/") + }; + format!( + "selector uses signature grammar {selector_grammar} but the index is {index_grammar} \ + — re-propose from current findings to measure efficacy" + ) +} + +/// The `v` grammar token of a trigger selector (`failure/v1|…` → `v1`), or +/// `None` when the selector carries no such token. +fn selector_grammar_token(selector: &str) -> Option<&str> { + let after_kind = selector.split_once('/')?.1; + let token = after_kind.split('|').next()?; + if token.starts_with('v') && token[1..].chars().all(|c| c.is_ascii_digit()) && token.len() > 1 { + Some(token) + } else { + None + } +} + /// Present the verdict enum in prose. const fn efficacy_verdict_label(verdict: Verdict) -> &'static str { match verdict { @@ -2830,6 +2883,7 @@ fn execute_mutation_action( .unwrap_or("0.0.0") .to_owned(); let bounds = WindowBounds::derive(installed_at, now)?; + let index_grammar_versions = store.signature_grammar_versions()?; let gathered = store.efficacy_occurrences(&selectors, bounds.pre_start, bounds.now)?; let mut occurrences = Vec::with_capacity(gathered.occurrences.len()); for occurrence in gathered.occurrences { @@ -2855,6 +2909,7 @@ fn execute_mutation_action( classifiable_failures: gathered.classifiable_failures, total_failures: gathered.total_failures, }, + index_grammar_versions, }; let evaluation = evaluate_efficacy(&observations, installed_at, now)?; let report = serde_json::to_value(&evaluation)?; diff --git a/crates/autophagy-efficacy/src/evaluate.rs b/crates/autophagy-efficacy/src/evaluate.rs index e689138..07dd03e 100644 --- a/crates/autophagy-efficacy/src/evaluate.rs +++ b/crates/autophagy-efficacy/src/evaluate.rs @@ -86,8 +86,17 @@ pub fn evaluate( let post_stats = window_stats(&post, bounds.installed_at, bounds.now, post_duration); let coverage = coverage(observations); - let insufficient_reasons = - insufficient_reasons(post_duration, &pre_stats, &post_stats, &coverage); + let grammar_mismatch = selector_grammar_mismatch( + &observations.signature_selectors, + &observations.index_grammar_versions, + ); + let insufficient_reasons = insufficient_reasons( + post_duration, + &pre_stats, + &post_stats, + &coverage, + grammar_mismatch, + ); let rate_delta_bps = rate_delta_bps(&pre_stats, &post_stats); let verdict = verdict( &insufficient_reasons, @@ -205,8 +214,15 @@ fn insufficient_reasons( pre: &WindowStats, post: &WindowStats, coverage: &Coverage, + grammar_mismatch: bool, ) -> Vec { let mut reasons = Vec::new(); + // Listed first: a grammar mismatch is the dominant, actionable cause — the + // other reasons below are usually just consequences of the selector matching + // zero events. + if grammar_mismatch { + reasons.push(InsufficientReason::SelectorGrammarMismatch); + } if post_duration_seconds < MIN_POST_WINDOW_SECONDS { reasons.push(InsufficientReason::PostWindowTooShort); } @@ -219,6 +235,38 @@ fn insufficient_reasons( reasons } +/// Parse the signature grammar version out of a trigger selector. +/// +/// Both selector grammars carry the version as the first `|`-delimited token +/// after the kind: `failure/v1|…` and `operation/v2|…` yield `1` and `2`. +/// Returns `None` for a selector that does not carry a `v` grammar token. +fn selector_grammar_version(selector: &str) -> Option { + let after_kind = selector.split_once('/')?.1; + let token = after_kind.split('|').next()?; + token.strip_prefix('v')?.parse::().ok() +} + +/// Whether any selector's signature grammar is older than the index's. +/// +/// The measurement can only see events indexed under a selector's own grammar. +/// When the newest grammar present in the index is newer than a selector's, and +/// the index holds no rows of that older grammar (as happens after +/// `reindex --index-tool-input` re-mints every row under the current grammar), +/// the selector matches nothing — the "0 → 0, no prior baseline" this would +/// otherwise produce is misleading, so it is reported as insufficient data. +/// +/// An index that still carries the selector's own grammar (a partial or skipped +/// reindex) does not trip this: the selector can still match those rows. +fn selector_grammar_mismatch(selectors: &[String], index_grammar_versions: &BTreeSet) -> bool { + let Some(&newest) = index_grammar_versions.iter().next_back() else { + return false; + }; + selectors.iter().any(|selector| { + selector_grammar_version(selector) + .is_some_and(|version| version < newest && !index_grammar_versions.contains(&version)) + }) +} + /// Signed relative change in weekly rate, in basis points, when a nonzero /// baseline exists. Because the windows are equal-length, this reduces to the /// relative change in raw counts. diff --git a/crates/autophagy-efficacy/src/model.rs b/crates/autophagy-efficacy/src/model.rs index 6bafeef..4d80523 100644 --- a/crates/autophagy-efficacy/src/model.rs +++ b/crates/autophagy-efficacy/src/model.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use serde::{Deserialize, Serialize}; use time::OffsetDateTime; @@ -64,6 +66,14 @@ pub struct EfficacyObservations { pub occurrences: Vec, /// Index-coverage diagnostics over the span. pub coverage: CoverageInput, + /// Signature grammar versions present in the exact-signature index for this + /// evaluation (for example `{2}` when the index was re-minted under grammar + /// `v2`). Empty when the index holds no signature rows. A selector whose + /// grammar version is older than the newest present here, and for which the + /// index holds no rows of that older grammar, cannot match any events — the + /// evaluator reports [`InsufficientReason::SelectorGrammarMismatch`] rather + /// than a misleading "0 → 0, no prior baseline". + pub index_grammar_versions: BTreeSet, } impl EfficacyObservations { @@ -159,6 +169,12 @@ pub enum InsufficientReason { SparseOccurrences, /// Too many in-span failures lack an index row to trust the counts. PartialIndexCoverage, + /// A trigger selector uses a signature grammar older than the index's, and + /// the index holds no rows of that older grammar, so the selector cannot + /// match the events it was minted against. The measurement is blind to those + /// occurrences (not evidence that they stopped): re-propose the mutation from + /// current findings to measure efficacy under the current grammar. + SelectorGrammarMismatch, } /// Complete deterministic post-install efficacy report. diff --git a/crates/autophagy-efficacy/tests/efficacy.rs b/crates/autophagy-efficacy/tests/efficacy.rs index e107660..444a7f5 100644 --- a/crates/autophagy-efficacy/tests/efficacy.rs +++ b/crates/autophagy-efficacy/tests/efficacy.rs @@ -1,5 +1,7 @@ //! Deterministic evaluation math, verdict thresholds, and schema conformance. +use std::collections::BTreeSet; + use autophagy_efficacy::{ CoverageInput, EfficacyObservations, EfficacyReport, FailureOccurrence, InsufficientReason, MatchingRule, Verdict, WindowBounds, evaluate, @@ -12,6 +14,7 @@ const VALID: &[&str] = &[ include_str!("../../../docs/specs/efficacy/0.1/valid/improved.json"), include_str!("../../../docs/specs/efficacy/0.1/valid/regressed_no_baseline.json"), include_str!("../../../docs/specs/efficacy/0.1/valid/insufficient_data.json"), + include_str!("../../../docs/specs/efficacy/0.1/valid/selector_grammar_mismatch.json"), ]; const INVALID: &[&str] = &[ include_str!("../../../docs/specs/efficacy/0.1/invalid/bad_spec_version.json"), @@ -38,14 +41,30 @@ fn occurrence(event_id: &str, session_id: &str, timestamp: &str) -> FailureOccur fn observations( occurrences: Vec, coverage: CoverageInput, +) -> EfficacyObservations { + // Default: a current-grammar (v2) selector against a v2 index — no mismatch. + selectors_against_index( + vec!["failure/v2|shell|go build|exit:1".to_owned()], + occurrences, + coverage, + BTreeSet::from([2]), + ) +} + +fn selectors_against_index( + signature_selectors: Vec, + occurrences: Vec, + coverage: CoverageInput, + index_grammar_versions: BTreeSet, ) -> EfficacyObservations { EfficacyObservations { mutation_id: "mut_efficacy-fixture".to_owned(), mutation_version: "0.1.0".to_owned(), - signature_selectors: vec!["failure/v2|shell|go build|exit:1".to_owned()], + signature_selectors, matching_rule: MatchingRule::FailureSignatureRecurrence, occurrences, coverage, + index_grammar_versions, } } @@ -205,6 +224,94 @@ fn partial_index_coverage_forces_insufficient_data() { assert!(!report.coverage.complete); } +#[test] +fn a_v1_selector_against_a_v2_index_is_a_grammar_mismatch() { + // The real defect: the mutation's trigger selector is grammar v1, but the + // index was re-minted to v2, so the v1 operation key matches zero rows. The + // eight pre-install failures exist — they are simply indexed under v2 — so + // "0 → 0, no prior baseline" would be misleading. + let report = evaluate( + &selectors_against_index( + vec!["failure/v1|shell|cd zuzoto && go build ./... 2>&1|exit:1".to_owned()], + Vec::new(), + CoverageInput { + classifiable_failures: 161, + total_failures: 200, + }, + BTreeSet::from([2]), + ), + at("2026-07-18T00:00:00Z"), + at("2026-11-16T00:00:00Z"), + ) + .expect("evaluate"); + assert_eq!(report.verdict, Verdict::InsufficientData); + assert!( + report + .insufficient_reasons + .contains(&InsufficientReason::SelectorGrammarMismatch) + ); +} + +#[test] +fn matching_grammars_do_not_trip_the_mismatch_reason() { + // A current-grammar selector against a current-grammar index measures + // normally: no grammar-mismatch reason regardless of the verdict. + let occurrences = vec![ + occurrence("evt_m1", "ses_a", "2026-01-10T00:00:00Z"), + occurrence("evt_m2", "ses_b", "2026-02-01T00:00:00Z"), + occurrence("evt_m3", "ses_b", "2026-03-01T00:00:00Z"), + ]; + let report = evaluate( + &selectors_against_index( + vec!["failure/v2|shell|go build|exit:1".to_owned()], + occurrences, + CoverageInput { + classifiable_failures: 50, + total_failures: 50, + }, + BTreeSet::from([2]), + ), + at("2026-04-01T00:00:00Z"), + at("2026-07-01T00:00:00Z"), + ) + .expect("evaluate"); + assert_eq!(report.verdict, Verdict::Improved); + assert!( + !report + .insufficient_reasons + .contains(&InsufficientReason::SelectorGrammarMismatch) + ); +} + +#[test] +fn an_old_selector_still_present_in_the_index_is_not_a_mismatch() { + // A partial or skipped reindex leaves the selector's own grammar in the + // index: it can still match those rows, so this is measured, not flagged. + let occurrences = vec![ + occurrence("evt_o1", "ses_a", "2026-01-10T00:00:00Z"), + occurrence("evt_o2", "ses_b", "2026-02-01T00:00:00Z"), + ]; + let report = evaluate( + &selectors_against_index( + vec!["failure/v1|shell|go build|exit:1".to_owned()], + occurrences, + CoverageInput { + classifiable_failures: 40, + total_failures: 40, + }, + BTreeSet::from([1, 2]), + ), + at("2026-04-01T00:00:00Z"), + at("2026-07-01T00:00:00Z"), + ) + .expect("evaluate"); + assert!( + !report + .insufficient_reasons + .contains(&InsufficientReason::SelectorGrammarMismatch) + ); +} + #[test] fn identical_inputs_produce_an_identical_content_id() { let make = || { diff --git a/crates/autophagy-store/src/store.rs b/crates/autophagy-store/src/store.rs index 15a729e..76bc15d 100644 --- a/crates/autophagy-store/src/store.rs +++ b/crates/autophagy-store/src/store.rs @@ -585,6 +585,43 @@ impl EventStore { Ok(u64::try_from(count).unwrap_or(0)) } + /// The distinct signature grammar versions present in the exact-signature + /// index, as integers (`operation/v2|…` and `failure/v2|…` both contribute + /// `2`). + /// + /// Every indexed signature carries its grammar version as the first + /// `|`-delimited token after the kind (`operation/v|…`). Efficacy uses + /// this to detect a selector whose grammar is older than the index's — after + /// `reindex --index-tool-input` re-mints every row under the current grammar, + /// a superseded selector matches nothing and its recurrence cannot be + /// measured. Empty when the index holds no signature rows. Rows whose grammar + /// token is unparseable are skipped. + /// + /// # Errors + /// + /// Returns [`StoreError`] when `SQLite` cannot execute the query. + pub fn signature_grammar_versions(&self) -> Result, StoreError> { + // The version token is the substring between the first `/` and the first + // `|`; e.g. `operation/v2|shell|…` yields `v2`. + let mut statement = self.connection.prepare( + "SELECT DISTINCT substr( + signature, + instr(signature, '/') + 1, + instr(signature, '|') - instr(signature, '/') - 1 + ) + FROM event_signatures + WHERE instr(signature, '/') > 0 AND instr(signature, '|') > 0", + )?; + let rows = statement.query_map([], |row| row.get::<_, String>(0))?; + let mut versions = BTreeSet::new(); + for token in rows { + if let Some(version) = token?.strip_prefix('v').and_then(|n| n.parse::().ok()) { + versions.insert(version); + } + } + Ok(versions) + } + /// Rebuild the derived search projections from every stored event's /// canonical `event_json`, applying the caller-supplied redaction-approved /// projection. diff --git a/crates/autophagy-store/tests/store.rs b/crates/autophagy-store/tests/store.rs index 9b76d10..e5abb5f 100644 --- a/crates/autophagy-store/tests/store.rs +++ b/crates/autophagy-store/tests/store.rs @@ -1474,6 +1474,46 @@ fn signatures_below_version_counts_only_superseded_grammar() { ); } +#[test] +fn signature_grammar_versions_reports_the_distinct_indexed_grammars() { + let mut store = EventStore::open_in_memory().expect("store"); + let source = source("instance-grammar"); + let seed = |store: &mut EventStore, event_id: &str, sequence: u64, signature: &str| { + let event = tool_failure(event_id, "ses_grammar", "2026-07-16T01:00:00Z", sequence); + store + .insert_event( + &source, + &event, + &SearchProjection { + tool_input_text: None, + searchable_text: None, + signature: Some(signature.to_owned()), + }, + ) + .expect("insert with signature"); + }; + // An empty index reports no grammar versions at all. + assert!( + store + .signature_grammar_versions() + .expect("empty versions") + .is_empty() + ); + // Two grammars coexist (a skipped reindex): both are reported. + seed( + &mut store, + "evt_g1", + 1, + "operation/v1|shell|cd zuzoto && go build|exit:1", + ); + seed(&mut store, "evt_g2", 2, "operation/v2|shell|go test"); + seed(&mut store, "evt_g3", 3, "operation/v2|shell|cargo build"); + assert_eq!( + store.signature_grammar_versions().expect("mixed versions"), + std::collections::BTreeSet::from([1, 2]) + ); +} + #[test] fn exact_signature_lookup_orders_by_recency_then_id() { let store = seed_retrieval_store(); diff --git a/docs/decisions/0015-mutation-efficacy-tracking.md b/docs/decisions/0015-mutation-efficacy-tracking.md index 06c442a..fd2024d 100644 --- a/docs/decisions/0015-mutation-efficacy-tracking.md +++ b/docs/decisions/0015-mutation-efficacy-tracking.md @@ -133,3 +133,37 @@ persisted and nothing leaves the machine. told to `reindex` through the existing status hint (ADR 0014). - Migrations advance to schema v3; the table and its evidence cascade are immutable from here. + +## Addendum (2026-07-19): `selector_grammar_mismatch` insufficient-data reason + +A mutation installed under an older signature grammar exposed a misleading +report. Its trigger selector embeds a grammar version +(`failure/v1|shell|cd zuzoto && go build ./... 2>&1|exit:1`), but the database it +was evaluated against had its `event_signatures` index re-minted to grammar `v2` +by `reindex --index-tool-input`. The v1 operation key then matched zero rows even +though the pre-install failures existed (indexed under v2), so the report read +`insufficient data · 0 → 0 failures · no prior baseline · needs more observed +occurrences` — as if the failures had never happened. + +The honest statement is that the selector's grammar is older than the index's and +the measurement cannot see those events. A v1 selector cannot be silently +translated to v2: the v2 normalization is not derivable from the v1 string. + +**Decision.** Add a fourth `insufficient_reasons` variant, +`selector_grammar_mismatch`, to efficacy result **v0.1**. It fires when a +selector's grammar version is older than the newest grammar present in the index +*and* the index holds no rows of that older grammar (a partial or skipped reindex +that still carries the old grammar does not trip it — the selector can still match +those rows). The text output states the mismatch explicitly and directs the user +to re-propose from current findings. + +**Why this is not a version bump.** The change is a purely additive enum member. +Existing efficacy/0.1 reports never carried this value and remain valid; every +existing fixture still passes. Per the repo rule that a behavior-changing schema +edit needs a decision record plus updated schema, Rust types, and fixtures, this +addendum is the decision record: `result.schema.json` gains the enum value, +`InsufficientReason` gains the variant, and `valid/selector_grammar_mismatch.json` +is added (a real report generated from the reindexed showcase database). A new +`Store::signature_grammar_versions` supplies the index grammar versions the +evaluator compares against; no migration is required (it is a read-only query over +the existing `event_signatures` table). diff --git a/docs/specs/efficacy/0.1/README.md b/docs/specs/efficacy/0.1/README.md index 8e09701..abe0165 100644 --- a/docs/specs/efficacy/0.1/README.md +++ b/docs/specs/efficacy/0.1/README.md @@ -71,6 +71,15 @@ computed by these exact, inspectable thresholds: - `sparse_occurrences` — fewer than **2** total occurrences across both windows. - `partial_index_coverage` — `coverage_bps` below **5000** (50%). + - `selector_grammar_mismatch` — a trigger selector's signature grammar is + older than the newest grammar present in the index, and the index holds no + rows of that older grammar (as after `reindex --index-tool-input` re-mints + every row under the current grammar). The selector's operation key then + matches zero events, so the counts are blind to the failures it was minted + against — not evidence they stopped. A selector cannot be silently + translated to the current grammar (the current grammar's normalization + cannot be derived from the old string), so the honest resolution is to + re-propose the mutation from current findings. 2. Otherwise, from the change in weekly rate: - With no pre-window baseline (pre rate 0), a nonzero post rate is `regressed` (a new recurrence appeared); `rate_delta_bps` is omitted. diff --git a/docs/specs/efficacy/0.1/result.schema.json b/docs/specs/efficacy/0.1/result.schema.json index 4d7d709..2ad3364 100644 --- a/docs/specs/efficacy/0.1/result.schema.json +++ b/docs/specs/efficacy/0.1/result.schema.json @@ -65,7 +65,8 @@ "enum": [ "post_window_too_short", "sparse_occurrences", - "partial_index_coverage" + "partial_index_coverage", + "selector_grammar_mismatch" ] } }, diff --git a/docs/specs/efficacy/0.1/valid/selector_grammar_mismatch.json b/docs/specs/efficacy/0.1/valid/selector_grammar_mismatch.json new file mode 100644 index 0000000..9f8a96e --- /dev/null +++ b/docs/specs/efficacy/0.1/valid/selector_grammar_mismatch.json @@ -0,0 +1,49 @@ +{ + "spec_version": "efficacy/0.1", + "efficacy_id": "eff_55a7d8070bfb052639e237db6623daea7c3cf13198d4426e7a2e184e35cf6d83", + "mutation_id": "mut_b13f28789cdae90dd6837572975bca5bf73e6b45f6e36730f7ff4d37cefa8e41", + "mutation_version": "0.1.0", + "signature_selectors": [ + "failure/v1|shell|cd zuzoto && go build ./... 2>&1|exit:1" + ], + "matching_rule": "failure_signature_recurrence", + "installed_at": "2026-07-18T21:52:19.074217Z", + "evaluated_at": "2026-11-16T00:00:00Z", + "windows": { + "pre": { + "start": "2026-03-20T19:44:38.148434Z", + "end": "2026-07-18T21:52:19.074217Z", + "duration_seconds": 10375660, + "occurrences": 0, + "distinct_sessions": 0, + "rate_per_week_milli": 0 + }, + "post": { + "start": "2026-07-18T21:52:19.074217Z", + "end": "2026-11-16T00:00:00Z", + "duration_seconds": 10375660, + "occurrences": 0, + "distinct_sessions": 0, + "rate_per_week_milli": 0 + } + }, + "coverage": { + "classifiable_failures": 705, + "total_failures": 876, + "coverage_bps": 8047, + "complete": false + }, + "verdict": "insufficient_data", + "insufficient_reasons": [ + "selector_grammar_mismatch", + "sparse_occurrences" + ], + "evidence": { + "pre_event_ids": [], + "post_event_ids": [], + "pre_event_count": 0, + "post_event_count": 0, + "listing_cap": 50 + }, + "model_used": false +}