Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 61 additions & 6 deletions crates/autophagy-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1491,6 +1491,11 @@ const fn efficacy_reason_phrase(reason: InsufficientReason) -> &'static str {
InsufficientReason::PostWindowTooShort => "a longer post-install window",
InsufficientReason::SparseOccurrences => "more observed occurrences",
InsufficientReason::PartialIndexCoverage => "fuller command-index coverage",
// Rendered as its own actionable sentence by
// `selector_grammar_mismatch_phrase`, never folded into a "needs …" list.
InsufficientReason::SelectorGrammarMismatch => {
"a selector minted under the current signature grammar"
}
}
}

Expand All @@ -1515,17 +1520,65 @@ fn efficacy_summary_line(report: &EfficacyReport) -> String {
percent(report.coverage.coverage_bps)
));
if report.verdict == Verdict::InsufficientData && !report.insufficient_reasons.is_empty() {
let needs = report
// A grammar mismatch is the dominant, actionable cause: the selector
// matches zero rows, so the other reasons (sparse, short window) are just
// consequences of that blindness. Report only the mismatch sentence.
if report
.insufficient_reasons
.iter()
.map(|reason| efficacy_reason_phrase(*reason))
.collect::<Vec<_>>()
.join(", ");
parts.push(format!("needs {needs}"));
.contains(&InsufficientReason::SelectorGrammarMismatch)
{
parts.push(selector_grammar_mismatch_phrase(
&report.signature_selectors,
));
} else {
let needs = report
.insufficient_reasons
.iter()
.map(|reason| efficacy_reason_phrase(*reason))
.collect::<Vec<_>>()
.join(", ");
parts.push(format!("needs {needs}"));
}
}
parts.join(" · ")
}

/// The actionable sentence for a [`InsufficientReason::SelectorGrammarMismatch`]:
/// which grammar the selector uses, which grammar the index now carries, and
/// what to do about it. The index grammar is the current signature spec version
/// (`reindex --index-tool-input` re-mints every row under it).
fn selector_grammar_mismatch_phrase(selectors: &[String]) -> String {
let index_grammar = autophagy_events::signature::SIGNATURE_SPEC_VERSION;
let mut selector_grammars = selectors
.iter()
.filter_map(|selector| selector_grammar_token(selector))
.filter(|grammar| *grammar != index_grammar)
.collect::<Vec<_>>();
selector_grammars.sort_unstable();
selector_grammars.dedup();
let selector_grammar = if selector_grammars.is_empty() {
"an older version".to_owned()
} else {
selector_grammars.join("/")
};
format!(
"selector uses signature grammar {selector_grammar} but the index is {index_grammar} \
— re-propose from current findings to measure efficacy"
)
}

/// The `v<n>` grammar token of a trigger selector (`failure/v1|…` → `v1`), or
/// `None` when the selector carries no such token.
fn selector_grammar_token(selector: &str) -> Option<&str> {
let after_kind = selector.split_once('/')?.1;
let token = after_kind.split('|').next()?;
if token.starts_with('v') && token[1..].chars().all(|c| c.is_ascii_digit()) && token.len() > 1 {
Some(token)
} else {
None
}
}

/// Present the verdict enum in prose.
const fn efficacy_verdict_label(verdict: Verdict) -> &'static str {
match verdict {
Expand Down Expand Up @@ -2830,6 +2883,7 @@ fn execute_mutation_action(
.unwrap_or("0.0.0")
.to_owned();
let bounds = WindowBounds::derive(installed_at, now)?;
let index_grammar_versions = store.signature_grammar_versions()?;
let gathered = store.efficacy_occurrences(&selectors, bounds.pre_start, bounds.now)?;
let mut occurrences = Vec::with_capacity(gathered.occurrences.len());
for occurrence in gathered.occurrences {
Expand All @@ -2855,6 +2909,7 @@ fn execute_mutation_action(
classifiable_failures: gathered.classifiable_failures,
total_failures: gathered.total_failures,
},
index_grammar_versions,
};
let evaluation = evaluate_efficacy(&observations, installed_at, now)?;
let report = serde_json::to_value(&evaluation)?;
Expand Down
52 changes: 50 additions & 2 deletions crates/autophagy-efficacy/src/evaluate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,17 @@ pub fn evaluate(
let post_stats = window_stats(&post, bounds.installed_at, bounds.now, post_duration);

let coverage = coverage(observations);
let insufficient_reasons =
insufficient_reasons(post_duration, &pre_stats, &post_stats, &coverage);
let grammar_mismatch = selector_grammar_mismatch(
&observations.signature_selectors,
&observations.index_grammar_versions,
);
let insufficient_reasons = insufficient_reasons(
post_duration,
&pre_stats,
&post_stats,
&coverage,
grammar_mismatch,
);
let rate_delta_bps = rate_delta_bps(&pre_stats, &post_stats);
let verdict = verdict(
&insufficient_reasons,
Expand Down Expand Up @@ -205,8 +214,15 @@ fn insufficient_reasons(
pre: &WindowStats,
post: &WindowStats,
coverage: &Coverage,
grammar_mismatch: bool,
) -> Vec<InsufficientReason> {
let mut reasons = Vec::new();
// Listed first: a grammar mismatch is the dominant, actionable cause — the
// other reasons below are usually just consequences of the selector matching
// zero events.
if grammar_mismatch {
reasons.push(InsufficientReason::SelectorGrammarMismatch);
}
if post_duration_seconds < MIN_POST_WINDOW_SECONDS {
reasons.push(InsufficientReason::PostWindowTooShort);
}
Expand All @@ -219,6 +235,38 @@ fn insufficient_reasons(
reasons
}

/// Parse the signature grammar version out of a trigger selector.
///
/// Both selector grammars carry the version as the first `|`-delimited token
/// after the kind: `failure/v1|…` and `operation/v2|…` yield `1` and `2`.
/// Returns `None` for a selector that does not carry a `v<n>` grammar token.
fn selector_grammar_version(selector: &str) -> Option<u32> {
let after_kind = selector.split_once('/')?.1;
let token = after_kind.split('|').next()?;
token.strip_prefix('v')?.parse::<u32>().ok()
}

/// Whether any selector's signature grammar is older than the index's.
///
/// The measurement can only see events indexed under a selector's own grammar.
/// When the newest grammar present in the index is newer than a selector's, and
/// the index holds no rows of that older grammar (as happens after
/// `reindex --index-tool-input` re-mints every row under the current grammar),
/// the selector matches nothing — the "0 → 0, no prior baseline" this would
/// otherwise produce is misleading, so it is reported as insufficient data.
///
/// An index that still carries the selector's own grammar (a partial or skipped
/// reindex) does not trip this: the selector can still match those rows.
fn selector_grammar_mismatch(selectors: &[String], index_grammar_versions: &BTreeSet<u32>) -> bool {
let Some(&newest) = index_grammar_versions.iter().next_back() else {
return false;
};
selectors.iter().any(|selector| {
selector_grammar_version(selector)
.is_some_and(|version| version < newest && !index_grammar_versions.contains(&version))
})
}

/// Signed relative change in weekly rate, in basis points, when a nonzero
/// baseline exists. Because the windows are equal-length, this reduces to the
/// relative change in raw counts.
Expand Down
16 changes: 16 additions & 0 deletions crates/autophagy-efficacy/src/model.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
use std::collections::BTreeSet;

use serde::{Deserialize, Serialize};
use time::OffsetDateTime;

Expand Down Expand Up @@ -64,6 +66,14 @@ pub struct EfficacyObservations {
pub occurrences: Vec<FailureOccurrence>,
/// Index-coverage diagnostics over the span.
pub coverage: CoverageInput,
/// Signature grammar versions present in the exact-signature index for this
/// evaluation (for example `{2}` when the index was re-minted under grammar
/// `v2`). Empty when the index holds no signature rows. A selector whose
/// grammar version is older than the newest present here, and for which the
/// index holds no rows of that older grammar, cannot match any events — the
/// evaluator reports [`InsufficientReason::SelectorGrammarMismatch`] rather
/// than a misleading "0 → 0, no prior baseline".
pub index_grammar_versions: BTreeSet<u32>,
}

impl EfficacyObservations {
Expand Down Expand Up @@ -159,6 +169,12 @@ pub enum InsufficientReason {
SparseOccurrences,
/// Too many in-span failures lack an index row to trust the counts.
PartialIndexCoverage,
/// A trigger selector uses a signature grammar older than the index's, and
/// the index holds no rows of that older grammar, so the selector cannot
/// match the events it was minted against. The measurement is blind to those
/// occurrences (not evidence that they stopped): re-propose the mutation from
/// current findings to measure efficacy under the current grammar.
SelectorGrammarMismatch,
}

/// Complete deterministic post-install efficacy report.
Expand Down
109 changes: 108 additions & 1 deletion crates/autophagy-efficacy/tests/efficacy.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
//! Deterministic evaluation math, verdict thresholds, and schema conformance.

use std::collections::BTreeSet;

use autophagy_efficacy::{
CoverageInput, EfficacyObservations, EfficacyReport, FailureOccurrence, InsufficientReason,
MatchingRule, Verdict, WindowBounds, evaluate,
Expand All @@ -12,6 +14,7 @@ const VALID: &[&str] = &[
include_str!("../../../docs/specs/efficacy/0.1/valid/improved.json"),
include_str!("../../../docs/specs/efficacy/0.1/valid/regressed_no_baseline.json"),
include_str!("../../../docs/specs/efficacy/0.1/valid/insufficient_data.json"),
include_str!("../../../docs/specs/efficacy/0.1/valid/selector_grammar_mismatch.json"),
];
const INVALID: &[&str] = &[
include_str!("../../../docs/specs/efficacy/0.1/invalid/bad_spec_version.json"),
Expand All @@ -38,14 +41,30 @@ fn occurrence(event_id: &str, session_id: &str, timestamp: &str) -> FailureOccur
fn observations(
occurrences: Vec<FailureOccurrence>,
coverage: CoverageInput,
) -> EfficacyObservations {
// Default: a current-grammar (v2) selector against a v2 index — no mismatch.
selectors_against_index(
vec!["failure/v2|shell|go build|exit:1".to_owned()],
occurrences,
coverage,
BTreeSet::from([2]),
)
}

fn selectors_against_index(
signature_selectors: Vec<String>,
occurrences: Vec<FailureOccurrence>,
coverage: CoverageInput,
index_grammar_versions: BTreeSet<u32>,
) -> EfficacyObservations {
EfficacyObservations {
mutation_id: "mut_efficacy-fixture".to_owned(),
mutation_version: "0.1.0".to_owned(),
signature_selectors: vec!["failure/v2|shell|go build|exit:1".to_owned()],
signature_selectors,
matching_rule: MatchingRule::FailureSignatureRecurrence,
occurrences,
coverage,
index_grammar_versions,
}
}

Expand Down Expand Up @@ -205,6 +224,94 @@ fn partial_index_coverage_forces_insufficient_data() {
assert!(!report.coverage.complete);
}

#[test]
fn a_v1_selector_against_a_v2_index_is_a_grammar_mismatch() {
// The real defect: the mutation's trigger selector is grammar v1, but the
// index was re-minted to v2, so the v1 operation key matches zero rows. The
// eight pre-install failures exist — they are simply indexed under v2 — so
// "0 → 0, no prior baseline" would be misleading.
let report = evaluate(
&selectors_against_index(
vec!["failure/v1|shell|cd zuzoto && go build ./... 2>&1|exit:1".to_owned()],
Vec::new(),
CoverageInput {
classifiable_failures: 161,
total_failures: 200,
},
BTreeSet::from([2]),
),
at("2026-07-18T00:00:00Z"),
at("2026-11-16T00:00:00Z"),
)
.expect("evaluate");
assert_eq!(report.verdict, Verdict::InsufficientData);
assert!(
report
.insufficient_reasons
.contains(&InsufficientReason::SelectorGrammarMismatch)
);
}

#[test]
fn matching_grammars_do_not_trip_the_mismatch_reason() {
// A current-grammar selector against a current-grammar index measures
// normally: no grammar-mismatch reason regardless of the verdict.
let occurrences = vec![
occurrence("evt_m1", "ses_a", "2026-01-10T00:00:00Z"),
occurrence("evt_m2", "ses_b", "2026-02-01T00:00:00Z"),
occurrence("evt_m3", "ses_b", "2026-03-01T00:00:00Z"),
];
let report = evaluate(
&selectors_against_index(
vec!["failure/v2|shell|go build|exit:1".to_owned()],
occurrences,
CoverageInput {
classifiable_failures: 50,
total_failures: 50,
},
BTreeSet::from([2]),
),
at("2026-04-01T00:00:00Z"),
at("2026-07-01T00:00:00Z"),
)
.expect("evaluate");
assert_eq!(report.verdict, Verdict::Improved);
assert!(
!report
.insufficient_reasons
.contains(&InsufficientReason::SelectorGrammarMismatch)
);
}

#[test]
fn an_old_selector_still_present_in_the_index_is_not_a_mismatch() {
// A partial or skipped reindex leaves the selector's own grammar in the
// index: it can still match those rows, so this is measured, not flagged.
let occurrences = vec![
occurrence("evt_o1", "ses_a", "2026-01-10T00:00:00Z"),
occurrence("evt_o2", "ses_b", "2026-02-01T00:00:00Z"),
];
let report = evaluate(
&selectors_against_index(
vec!["failure/v1|shell|go build|exit:1".to_owned()],
occurrences,
CoverageInput {
classifiable_failures: 40,
total_failures: 40,
},
BTreeSet::from([1, 2]),
),
at("2026-04-01T00:00:00Z"),
at("2026-07-01T00:00:00Z"),
)
.expect("evaluate");
assert!(
!report
.insufficient_reasons
.contains(&InsufficientReason::SelectorGrammarMismatch)
);
}

#[test]
fn identical_inputs_produce_an_identical_content_id() {
let make = || {
Expand Down
37 changes: 37 additions & 0 deletions crates/autophagy-store/src/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,43 @@ impl EventStore {
Ok(u64::try_from(count).unwrap_or(0))
}

/// The distinct signature grammar versions present in the exact-signature
/// index, as integers (`operation/v2|…` and `failure/v2|…` both contribute
/// `2`).
///
/// Every indexed signature carries its grammar version as the first
/// `|`-delimited token after the kind (`operation/v<n>|…`). Efficacy uses
/// this to detect a selector whose grammar is older than the index's — after
/// `reindex --index-tool-input` re-mints every row under the current grammar,
/// a superseded selector matches nothing and its recurrence cannot be
/// measured. Empty when the index holds no signature rows. Rows whose grammar
/// token is unparseable are skipped.
///
/// # Errors
///
/// Returns [`StoreError`] when `SQLite` cannot execute the query.
pub fn signature_grammar_versions(&self) -> Result<BTreeSet<u32>, StoreError> {
// The version token is the substring between the first `/` and the first
// `|`; e.g. `operation/v2|shell|…` yields `v2`.
let mut statement = self.connection.prepare(
"SELECT DISTINCT substr(
signature,
instr(signature, '/') + 1,
instr(signature, '|') - instr(signature, '/') - 1
)
FROM event_signatures
WHERE instr(signature, '/') > 0 AND instr(signature, '|') > 0",
)?;
let rows = statement.query_map([], |row| row.get::<_, String>(0))?;
let mut versions = BTreeSet::new();
for token in rows {
if let Some(version) = token?.strip_prefix('v').and_then(|n| n.parse::<u32>().ok()) {
versions.insert(version);
}
}
Ok(versions)
}

/// Rebuild the derived search projections from every stored event's
/// canonical `event_json`, applying the caller-supplied redaction-approved
/// projection.
Expand Down
Loading