From 979cd47a958fe5dee444564b903cc41642fccf27 Mon Sep 17 00:00:00 2001 From: Karn Date: Sun, 19 Jul 2026 02:31:26 +0530 Subject: [PATCH] feat: track post-install mutation efficacy with deterministic recurrence windows Add an observational, model-free measurement of whether the failure signature an installed mutation addresses recurs less after installation. Compares two equal-length windows anchored on installed_at, deterministic given (database-state, installed_at, evaluated_at), with no causal claim and no lifecycle coupling. - docs/decisions/0015: observational stance, window rule, verdict thresholds, matching rule, no lifecycle coupling, privacy. - docs/specs/efficacy/0.1: result.schema.json + valid/invalid fixtures. - migration 0003 (schema v3): STRICT mutation_efficacy + _evidence tables with a cascade trigger that drops only the report (never the candidate). - new crate autophagy-efficacy: pure deterministic evaluation, unit-tested. - store: register_efficacy / list_efficacy / efficacy_occurrences / efficacy_status_summary; content-hash immutable, no state transition. - cli: `autophagy mutations efficacy `, `mutations show` latest-efficacy line, `status` installed-mutations summary; humanized text, full JSON. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 13 + Cargo.toml | 1 + crates/autophagy-cli/Cargo.toml | 1 + crates/autophagy-cli/src/main.rs | 228 +++++++++- crates/autophagy-cli/src/status.rs | 29 ++ crates/autophagy-cli/tests/cli.rs | 31 ++ crates/autophagy-efficacy/Cargo.toml | 22 + crates/autophagy-efficacy/src/evaluate.rs | 310 ++++++++++++++ crates/autophagy-efficacy/src/lib.rs | 25 ++ crates/autophagy-efficacy/src/model.rs | 200 +++++++++ crates/autophagy-efficacy/src/validate.rs | 143 +++++++ crates/autophagy-efficacy/tests/efficacy.rs | 310 ++++++++++++++ .../migrations/0003_mutation_efficacy.sql | 54 +++ crates/autophagy-store/src/error.rs | 15 + crates/autophagy-store/src/lib.rs | 19 +- crates/autophagy-store/src/migration.rs | 5 + crates/autophagy-store/src/model.rs | 95 +++++ crates/autophagy-store/src/store.rs | 382 ++++++++++++++++- .../autophagy-store/tests/legacy_adoption.rs | 12 +- crates/autophagy-store/tests/store.rs | 396 +++++++++++++++++- .../0015-mutation-efficacy-tracking.md | 135 ++++++ docs/specs/efficacy/0.1/README.md | 90 ++++ .../efficacy/0.1/invalid/bad_efficacy_id.json | 56 +++ .../0.1/invalid/bad_insufficient_reason.json | 58 +++ .../0.1/invalid/bad_matching_rule.json | 56 +++ .../0.1/invalid/bad_spec_version.json | 56 +++ .../efficacy/0.1/invalid/model_used_true.json | 56 +++ .../efficacy/0.1/invalid/unknown_field.json | 57 +++ .../efficacy/0.1/invalid/unknown_verdict.json | 56 +++ docs/specs/efficacy/0.1/result.schema.json | 122 ++++++ docs/specs/efficacy/0.1/valid/improved.json | 56 +++ .../efficacy/0.1/valid/insufficient_data.json | 49 +++ .../0.1/valid/regressed_no_baseline.json | 51 +++ 33 files changed, 3152 insertions(+), 37 deletions(-) create mode 100644 crates/autophagy-efficacy/Cargo.toml create mode 100644 crates/autophagy-efficacy/src/evaluate.rs create mode 100644 crates/autophagy-efficacy/src/lib.rs create mode 100644 crates/autophagy-efficacy/src/model.rs create mode 100644 crates/autophagy-efficacy/src/validate.rs create mode 100644 crates/autophagy-efficacy/tests/efficacy.rs create mode 100644 crates/autophagy-store/migrations/0003_mutation_efficacy.sql create mode 100644 docs/decisions/0015-mutation-efficacy-tracking.md create mode 100644 docs/specs/efficacy/0.1/README.md create mode 100644 docs/specs/efficacy/0.1/invalid/bad_efficacy_id.json create mode 100644 docs/specs/efficacy/0.1/invalid/bad_insufficient_reason.json create mode 100644 docs/specs/efficacy/0.1/invalid/bad_matching_rule.json create mode 100644 docs/specs/efficacy/0.1/invalid/bad_spec_version.json create mode 100644 docs/specs/efficacy/0.1/invalid/model_used_true.json create mode 100644 docs/specs/efficacy/0.1/invalid/unknown_field.json create mode 100644 docs/specs/efficacy/0.1/invalid/unknown_verdict.json create mode 100644 docs/specs/efficacy/0.1/result.schema.json create mode 100644 docs/specs/efficacy/0.1/valid/improved.json create mode 100644 docs/specs/efficacy/0.1/valid/insufficient_data.json create mode 100644 docs/specs/efficacy/0.1/valid/regressed_no_baseline.json diff --git a/Cargo.lock b/Cargo.lock index 9aac4c4..20b6c7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,6 +171,7 @@ dependencies = [ "autophagy-adapter-opencode", "autophagy-adapter-pi", "autophagy-core", + "autophagy-efficacy", "autophagy-events", "autophagy-install", "autophagy-mutations", @@ -204,6 +205,18 @@ dependencies = [ "thiserror", ] +[[package]] +name = "autophagy-efficacy" +version = "0.1.0" +dependencies = [ + "jsonschema", + "serde", + "serde_json", + "sha2", + "thiserror", + "time", +] + [[package]] name = "autophagy-events" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 452008d..83d82bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/autophagy-adapter-test-support", "crates/autophagy-cli", "crates/autophagy-core", + "crates/autophagy-efficacy", "crates/autophagy-events", "crates/autophagy-install", "crates/autophagy-mutations", diff --git a/crates/autophagy-cli/Cargo.toml b/crates/autophagy-cli/Cargo.toml index 60fc757..23df930 100644 --- a/crates/autophagy-cli/Cargo.toml +++ b/crates/autophagy-cli/Cargo.toml @@ -18,6 +18,7 @@ autophagy-adapter-codex = { path = "../../adapters/codex" } autophagy-adapter-opencode = { path = "../../adapters/opencode" } autophagy-adapter-pi = { path = "../../adapters/pi" } autophagy-core = { path = "../autophagy-core" } +autophagy-efficacy = { path = "../autophagy-efficacy" } autophagy-events = { path = "../autophagy-events" } autophagy-install = { path = "../autophagy-install" } autophagy-mutations = { path = "../autophagy-mutations" } diff --git a/crates/autophagy-cli/src/main.rs b/crates/autophagy-cli/src/main.rs index cc00017..dcb9755 100644 --- a/crates/autophagy-cli/src/main.rs +++ b/crates/autophagy-cli/src/main.rs @@ -32,6 +32,10 @@ use autophagy_core::{ ImportOptions, ImportSummary, ReindexError, ReindexOptions, ReindexSummary, import_jsonl, reindex, }; +use autophagy_efficacy::{ + CoverageInput, EfficacyError, EfficacyObservations, EfficacyReport, FailureOccurrence, + InsufficientReason, MatchingRule, Verdict, WindowBounds, evaluate as evaluate_efficacy, +}; use autophagy_events::Event; use autophagy_install::{ InstallError, InstallTarget, InstalledArtifact, SkillPlan, SupervisorError, materialize, @@ -52,11 +56,11 @@ use autophagy_shadow::{ evaluate as evaluate_shadow, extract_observation_draft, }; use autophagy_store::{ - DeleteAllSummary, DeleteSummary, EventStore, InstallationRegistration, - InstallationTransitionOutcome, MutationDetails, MutationRecord, MutationRegisterOutcome, - MutationRegistration, MutationTransitionOutcome, PruneSummary, ReplayRegisterOutcome, - ReplayRegistration, RetrievalHit, RetrievalOutcome, RetrievalQuery, SessionSummary, - ShadowRegisterOutcome, ShadowRegistration, StoreError, + DeleteAllSummary, DeleteSummary, EfficacyRegisterOutcome, EfficacyRegistration, EventStore, + InstallationRegistration, InstallationTransitionOutcome, MutationDetails, MutationRecord, + MutationRegisterOutcome, MutationRegistration, MutationTransitionOutcome, PruneSummary, + ReplayRegisterOutcome, ReplayRegistration, RetrievalHit, RetrievalOutcome, RetrievalQuery, + SessionSummary, ShadowRegisterOutcome, ShadowRegistration, StoreError, }; use autophagy_synthesis::{ AgentCliProvider, Capability, DeterministicReferenceProvider, EndpointLocality, ManifestError, @@ -622,6 +626,16 @@ enum MutationAction { #[arg(long, alias = "observations", value_name = "PATH")] suite: PathBuf, }, + /// Measure whether the addressed failure recurs less since install. + Efficacy { + /// Stable mutation identity. + mutation_id: String, + + /// Evaluation clock (RFC 3339). Defaults to the current time; override + /// for a reproducible or backdated report. + #[arg(long, value_name = "RFC3339")] + now: Option, + }, /// Install one shadow-passed mutation as a repo-scoped coding-agent skill. Install { /// Stable mutation identity. @@ -765,6 +779,8 @@ enum CommandReport { MutationShadowDraft(MutationShadowDraftReport), #[serde(rename = "mutations_shadow")] MutationShadow(MutationShadowReport), + #[serde(rename = "mutations_efficacy")] + MutationEfficacy(MutationEfficacyReport), #[serde(rename = "mutations_install")] MutationInstall(MutationInstallReport), #[serde(rename = "mutations_uninstall")] @@ -975,6 +991,12 @@ struct MutationShadowReport { requested_id: String, } +#[derive(Debug, Serialize)] +struct MutationEfficacyReport { + evaluation: EfficacyReport, + registration: EfficacyRegisterOutcome, +} + #[derive(Debug, Serialize)] struct MutationInstallReport { installation_id: String, @@ -1017,6 +1039,9 @@ impl CommandReport { Self::Import(summary) => summary.has_issues(), Self::MutationReplay(report) => !report.evaluation.passed, Self::MutationShadow(report) => !report.evaluation.passed, + Self::MutationEfficacy(report) => { + matches!(report.evaluation.verdict, Verdict::Regressed) + } Self::Watch(report) => report.has_issues(), Self::Sessions(_) | Self::Search(_) @@ -1127,6 +1152,8 @@ enum CliError { DataDirectoryUnavailable, #[error("could not format report timestamp: {0}")] TimeFormat(#[from] time::error::Format), + #[error("could not parse a stored timestamp: {0}")] + TimeParse(#[from] time::error::Parse), #[error("deleting all data requires --confirm delete-all")] DeleteAllConfirmation, #[error("challenge is incomplete; missing checks: {0}")] @@ -1140,6 +1167,16 @@ enum CliError { #[error(transparent)] Shadow(#[from] ShadowEvaluationError), #[error(transparent)] + Efficacy(#[from] EfficacyError), + #[error( + "mutation '{0}' is not installed; efficacy measures post-install recurrence, so install it first" + )] + MutationNotInstalled(String), + #[error("mutation '{0}' carries no trigger selectors to measure")] + EfficacyNoSelectors(String), + #[error("could not parse the --now evaluation clock '{0}' as an RFC 3339 timestamp")] + InvalidNowTimestamp(String), + #[error(transparent)] Install(#[from] InstallError), #[error(transparent)] Supervisor(#[from] SupervisorError), @@ -1298,6 +1335,93 @@ fn percent(bps: u32) -> String { format!("{:.1}%", f64::from(bps) / 100.0) } +/// Format a milli-occurrences-per-week rate for humans (`1810` → `1.8/wk`). +#[allow(clippy::cast_precision_loss)] +fn per_week(milli: i64) -> String { + format!("{:.1}/wk", milli as f64 / 1000.0) +} + +/// Format a signed basis-points change as a signed percentage (`-8700` → +/// `-87.0%`). +#[allow(clippy::cast_precision_loss)] +fn signed_percent(bps: i32) -> String { + format!("{:+.1}%", f64::from(bps) / 100.0) +} + +/// Turn an insufficiency reason into a plain-language "needs" phrase. +const fn efficacy_reason_phrase(reason: InsufficientReason) -> &'static str { + match reason { + InsufficientReason::PostWindowTooShort => "a longer post-install window", + InsufficientReason::SparseOccurrences => "more observed occurrences", + InsufficientReason::PartialIndexCoverage => "fuller command-index coverage", + } +} + +/// One humanized post-install recurrence summary from a typed efficacy report: +/// verdict, pre/post rates, relative change, and index coverage. +fn efficacy_summary_line(report: &EfficacyReport) -> String { + let mut parts = vec![format!( + "{} · {} → {} failures ({} → {})", + efficacy_verdict_label(report.verdict), + report.windows.pre.occurrences, + report.windows.post.occurrences, + per_week(report.windows.pre.rate_per_week_milli), + per_week(report.windows.post.rate_per_week_milli), + )]; + if let Some(delta) = report.rate_delta_bps { + parts.push(signed_percent(delta)); + } else if report.windows.pre.occurrences == 0 { + parts.push("no prior baseline".to_owned()); + } + parts.push(format!( + "{} classifiable", + percent(report.coverage.coverage_bps) + )); + if report.verdict == Verdict::InsufficientData && !report.insufficient_reasons.is_empty() { + let needs = report + .insufficient_reasons + .iter() + .map(|reason| efficacy_reason_phrase(*reason)) + .collect::>() + .join(", "); + parts.push(format!("needs {needs}")); + } + parts.join(" · ") +} + +/// Present the verdict enum in prose. +const fn efficacy_verdict_label(verdict: Verdict) -> &'static str { + match verdict { + Verdict::Improved => "improved", + Verdict::Regressed => "regressed", + Verdict::Unchanged => "unchanged", + Verdict::InsufficientData => "insufficient data", + } +} + +/// Render the `mutations efficacy` result as one humanized line. +fn write_mutation_efficacy( + writer: &mut impl Write, + report: &MutationEfficacyReport, +) -> io::Result<()> { + let duplicate = matches!( + report.registration, + EfficacyRegisterOutcome::Duplicate { .. } + ); + let window_days = report.evaluation.windows.post.duration_seconds / 86_400; + writeln!( + writer, + "{}\t{}\t{window_days}d window\tmodel_used=false{}", + report.evaluation.efficacy_id, + efficacy_summary_line(&report.evaluation), + if duplicate { + " · already recorded" + } else { + "" + }, + ) +} + /// Rewrite an absolute project path into a `~`-relative form when it lives under /// the user's home directory, so session rows stay short without becoming /// ambiguous. Paths outside home (and non-UTF-8 homes) are returned unchanged. @@ -2441,6 +2565,82 @@ fn execute_mutation_action( requested_id, })) } + MutationAction::Efficacy { mutation_id, now } => { + let mutation_id = resolve_mutation_id(&store, &mutation_id)?; + let installation = store.get_installation(&mutation_id)?; + if installation.state != "installed" { + return Err(CliError::MutationNotInstalled(mutation_id)); + } + let installed_at = OffsetDateTime::parse(&installation.installed_at, &Rfc3339)?; + let now = match now { + Some(raw) => OffsetDateTime::parse(&raw, &Rfc3339) + .map_err(|_| CliError::InvalidNowTimestamp(raw))?, + None => OffsetDateTime::now_utc(), + }; + let details = store.get_mutation(&mutation_id)?; + let package = &details.mutation.package; + let selectors = package + .get("triggers") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(|trigger| trigger.get("selector").and_then(serde_json::Value::as_str)) + .map(str::to_owned) + .collect::>(); + if selectors.is_empty() { + return Err(CliError::EfficacyNoSelectors(mutation_id)); + } + let mutation_version = package + .get("version") + .and_then(serde_json::Value::as_str) + .unwrap_or("0.0.0") + .to_owned(); + let bounds = WindowBounds::derive(installed_at, now)?; + let gathered = store.efficacy_occurrences(&selectors, bounds.pre_start, bounds.now)?; + let mut occurrences = Vec::with_capacity(gathered.occurrences.len()); + for occurrence in gathered.occurrences { + let occurred_at = OffsetDateTime::parse(&occurrence.occurred_at, &Rfc3339)?; + occurrences.push(FailureOccurrence { + event_id: occurrence.event_id, + session_id: occurrence.session_id, + occurred_at, + }); + } + let mut source_event_ids = occurrences + .iter() + .map(|occurrence| occurrence.event_id.clone()) + .collect::>(); + source_event_ids.sort(); + let observations = EfficacyObservations { + mutation_id: mutation_id.clone(), + mutation_version, + signature_selectors: selectors, + matching_rule: MatchingRule::FailureSignatureRecurrence, + occurrences, + coverage: CoverageInput { + classifiable_failures: gathered.classifiable_failures, + total_failures: gathered.total_failures, + }, + }; + let evaluation = evaluate_efficacy(&observations, installed_at, now)?; + let report = serde_json::to_value(&evaluation)?; + let verdict = report + .get("verdict") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_owned(); + let registration = store.register_efficacy(&EfficacyRegistration { + efficacy_id: evaluation.efficacy_id.clone(), + mutation_id: evaluation.mutation_id.clone(), + verdict, + report, + source_event_ids, + })?; + Ok(CommandReport::MutationEfficacy(MutationEfficacyReport { + evaluation, + registration, + })) + } MutationAction::Install { mutation_id, repository, @@ -2741,6 +2941,21 @@ fn write_report( installation.relative_path )?; } + // The latest efficacy report, if any: a one-line post-install + // recurrence summary in the same audit column. + if let Some(latest) = details.efficacies.last() { + let summary = serde_json::from_value::(latest.report.clone()) + .map_or_else( + |_| latest.verdict.clone(), + |report| efficacy_summary_line(&report), + ); + writeln!( + writer, + "{}\tefficacy {}\t{summary}", + compact_time(&latest.created_at, OffsetDateTime::now_utc()), + latest.efficacy_id, + )?; + } } CommandReport::MutationTransition(report) => writeln!( writer, @@ -2788,6 +3003,9 @@ fn write_report( percent(u32::from(report.evaluation.summary.recall_bps)), report.evaluation.summary.false_positives )?, + CommandReport::MutationEfficacy(report) => { + write_mutation_efficacy(&mut writer, report)?; + } CommandReport::MutationInstall(report) => writeln!( writer, "{}\t{}\t{}\t{}\t{}", diff --git a/crates/autophagy-cli/src/status.rs b/crates/autophagy-cli/src/status.rs index b667e26..52ed895 100644 --- a/crates/autophagy-cli/src/status.rs +++ b/crates/autophagy-cli/src/status.rs @@ -13,6 +13,7 @@ use std::{ path::{Path, PathBuf}, }; +use autophagy_store::EfficacyStatusSummary; use serde::Serialize; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; @@ -36,6 +37,10 @@ pub struct StatusReport { pub findings: Option, /// Mutation candidates grouped by lifecycle state. pub candidates: BTreeMap, + /// Post-install efficacy coverage across installed mutations. `None` unless + /// at least one mutation is currently installed. + #[serde(skip_serializing_if = "Option::is_none")] + pub efficacy: Option, /// Background daemon state. pub daemon: DaemonStatus, /// Resolved config file path. @@ -140,6 +145,10 @@ pub fn run( let stale_signatures = store.signatures_below_version(&stale_prefix)?; let activity = store.adapter_activity()?; let candidates = store.mutation_state_counts()?; + // Efficacy is a cheap COUNT-style aggregate; only surface it once something + // is actually installed to measure. + let efficacy_summary = store.efficacy_status_summary()?; + let efficacy = (efficacy_summary.installed > 0).then_some(efficacy_summary); let schema_version = store.schema_version()?; let now = OffsetDateTime::now_utc(); @@ -203,6 +212,7 @@ pub fn run( }, findings, candidates, + efficacy, daemon: DaemonStatus { platform: daemon_report.platform, supported: daemon_report.supported, @@ -395,6 +405,25 @@ pub fn write_text(report: &StatusReport, writer: &mut impl Write) -> std::io::Re .join(" · ") }; writeln!(writer, "{}", row("Lessons", &lessons))?; + if let Some(efficacy) = &report.efficacy { + let measured = efficacy.improved + efficacy.regressed + efficacy.unchanged; + let detail = if measured == 0 && efficacy.insufficient_data == 0 { + "none measured yet — run `autophagy mutations efficacy `".to_owned() + } else { + format!( + "{} improved · {} regressed · {} unchanged · {} inconclusive", + efficacy.improved, + efficacy.regressed, + efficacy.unchanged, + efficacy.insufficient_data, + ) + }; + writeln!( + writer, + "{}", + cont(&format!("{} installed · {detail}", efficacy.installed)) + )?; + } writeln!(writer)?; let daemon = if report.daemon.supported { diff --git a/crates/autophagy-cli/tests/cli.rs b/crates/autophagy-cli/tests/cli.rs index 742f1f2..7fd5b77 100644 --- a/crates/autophagy-cli/tests/cli.rs +++ b/crates/autophagy-cli/tests/cli.rs @@ -645,11 +645,42 @@ fn milestone_demo_digests_exports_deletes_and_prunes_offline() { .contains("Mutation: `mut_") ); + // Post-install efficacy: an observational, model-free recurrence report that + // registers without changing any lifecycle state. Evaluated at install time + // the post-window is empty, so the honest verdict is insufficient_data. + let efficacy = run_json(&database, ["mutations", "efficacy", &failure_id]); + assert_eq!( + efficacy["result"]["evaluation"]["spec_version"], + "efficacy/0.1" + ); + assert_eq!(efficacy["result"]["evaluation"]["model_used"], false); + assert_eq!( + efficacy["result"]["evaluation"]["matching_rule"], + "failure_signature_recurrence" + ); + assert_eq!(efficacy["result"]["registration"]["status"], "inserted"); + let measured = run_json(&database, ["mutations", "show", &failure_id]); + assert_eq!(measured["result"]["mutation"]["state"], "active"); + assert_eq!( + measured["result"]["efficacies"] + .as_array() + .expect("efficacies") + .len(), + 1 + ); + let uninstalled = run_json(&database, ["mutations", "uninstall", &failure_id]); assert_eq!(uninstalled["result"]["mutation_state"], "retired"); assert_eq!(uninstalled["result"]["installation_state"], "uninstalled"); assert!(!installed_path.exists()); + // Efficacy measures post-install recurrence, so a retired mutation is refused. + let refused_efficacy = command(&database) + .args(["mutations", "efficacy", &failure_id]) + .output() + .expect("efficacy after uninstall"); + assert!(!refused_efficacy.status.success()); + let retired = run_json(&database, ["mutations", "show", &failure_id]); assert_eq!(retired["result"]["mutation"]["state"], "retired"); assert_eq!( diff --git a/crates/autophagy-efficacy/Cargo.toml b/crates/autophagy-efficacy/Cargo.toml new file mode 100644 index 0000000..96c160c --- /dev/null +++ b/crates/autophagy-efficacy/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "autophagy-efficacy" +description = "Deterministic post-install recurrence-efficacy evaluation for Autophagy mutations" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +thiserror.workspace = true +time.workspace = true + +[dev-dependencies] +jsonschema = { version = "0.47", default-features = false } + +[lints] +workspace = true diff --git a/crates/autophagy-efficacy/src/evaluate.rs b/crates/autophagy-efficacy/src/evaluate.rs new file mode 100644 index 0000000..e689138 --- /dev/null +++ b/crates/autophagy-efficacy/src/evaluate.rs @@ -0,0 +1,310 @@ +use std::{collections::BTreeSet, fmt::Write as _}; + +use sha2::{Digest, Sha256}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; + +use crate::{ + Coverage, EfficacyObservations, EfficacyReport, EfficacyResultSpecVersion, EfficacyWindows, + Evidence, FailureOccurrence, InsufficientReason, Verdict, WindowStats, + validate::EfficacyValidationErrors, +}; + +/// Minimum post-install observation window. A window shorter than this cannot +/// yield a trustworthy recurrence rate, so the verdict is insufficient. +const MIN_POST_WINDOW_SECONDS: i64 = 7 * 86_400; +/// Minimum combined pre+post occurrences before a comparison is meaningful. +const MIN_TOTAL_OCCURRENCES: u32 = 2; +/// Minimum classifiable-failure coverage (basis points) to trust the counts. +const MIN_COVERAGE_BPS: u32 = 5_000; +/// Neutral band (basis points): a weekly-rate change within ±20% is unchanged. +const UNCHANGED_BAND_BPS: i32 = 2_000; +/// Maximum evidence identifiers listed per window (counts stay exact). +const EVIDENCE_LISTING_CAP: u32 = 50; +/// Seconds in one week. +const WEEK_SECONDS: i64 = 604_800; + +/// The symmetric pre/post comparison windows, as instants. +/// +/// The post-window runs from `installed_at` to the evaluation clock. The +/// pre-window is the equal-length window immediately before `installed_at`, so +/// the two rates are directly comparable. This is the single source of truth for +/// the window math — the caller derives it to bound its store query and +/// [`evaluate`] derives the same bounds to partition the occurrences. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct WindowBounds { + /// Start of the pre-window (`installed_at - post_duration`). + pub pre_start: OffsetDateTime, + /// Boundary between the two windows. + pub installed_at: OffsetDateTime, + /// End of the post-window (the evaluation clock). + pub now: OffsetDateTime, +} + +impl WindowBounds { + /// Derive the symmetric windows from an install timestamp and a clock. + /// + /// # Errors + /// Returns [`EfficacyError::NonMonotonicClock`] when `now` precedes + /// `installed_at`. + pub fn derive( + installed_at: OffsetDateTime, + now: OffsetDateTime, + ) -> Result { + if now < installed_at { + return Err(EfficacyError::NonMonotonicClock { + installed_at: rfc3339(installed_at), + evaluated_at: rfc3339(now), + }); + } + let post_duration = now - installed_at; + Ok(Self { + pre_start: installed_at - post_duration, + installed_at, + now, + }) + } +} + +/// Evaluate post-install recurrence efficacy deterministically, without a model. +/// +/// # Errors +/// Returns semantic validation failures, a non-monotonic clock, or +/// serialization failures encountered while deriving the content identity. +pub fn evaluate( + observations: &EfficacyObservations, + installed_at: OffsetDateTime, + now: OffsetDateTime, +) -> Result { + observations.validate()?; + let bounds = WindowBounds::derive(installed_at, now)?; + + let (pre, post) = partition(&observations.occurrences, &bounds); + let post_duration = (bounds.now - bounds.installed_at).whole_seconds(); + let pre_duration = (bounds.installed_at - bounds.pre_start).whole_seconds(); + + let pre_stats = window_stats(&pre, bounds.pre_start, bounds.installed_at, pre_duration); + let post_stats = window_stats(&post, bounds.installed_at, bounds.now, post_duration); + + let coverage = coverage(observations); + let insufficient_reasons = + insufficient_reasons(post_duration, &pre_stats, &post_stats, &coverage); + let rate_delta_bps = rate_delta_bps(&pre_stats, &post_stats); + let verdict = verdict( + &insufficient_reasons, + &pre_stats, + &post_stats, + rate_delta_bps, + ); + let evidence = evidence(&pre, &post); + + let mut selectors = observations.signature_selectors.clone(); + selectors.sort(); + selectors.dedup(); + + let mut report = EfficacyReport { + spec_version: EfficacyResultSpecVersion::V0_1, + efficacy_id: String::new(), + mutation_id: observations.mutation_id.clone(), + mutation_version: observations.mutation_version.clone(), + signature_selectors: selectors, + matching_rule: observations.matching_rule, + installed_at: rfc3339(bounds.installed_at), + evaluated_at: rfc3339(bounds.now), + windows: EfficacyWindows { + pre: pre_stats, + post: post_stats, + }, + rate_delta_bps, + coverage, + verdict, + insufficient_reasons, + evidence, + model_used: false, + }; + let identity = serde_json::to_vec(&report)?; + report.efficacy_id = prefixed_hash("eff", &identity); + Ok(report) +} + +/// Partition occurrences into the pre-window `[pre_start, installed_at)` and the +/// post-window `[installed_at, now]`, dropping anything outside the span. Each +/// side is returned in canonical `(occurred_at, event_id)` order. +fn partition<'a>( + occurrences: &'a [FailureOccurrence], + bounds: &WindowBounds, +) -> (Vec<&'a FailureOccurrence>, Vec<&'a FailureOccurrence>) { + let mut pre = Vec::new(); + let mut post = Vec::new(); + for occurrence in occurrences { + let at = occurrence.occurred_at; + if at >= bounds.installed_at && at <= bounds.now { + post.push(occurrence); + } else if at >= bounds.pre_start && at < bounds.installed_at { + pre.push(occurrence); + } + } + let order = |left: &&FailureOccurrence, right: &&FailureOccurrence| { + left.occurred_at + .cmp(&right.occurred_at) + .then_with(|| left.event_id.cmp(&right.event_id)) + }; + pre.sort_by(order); + post.sort_by(order); + (pre, post) +} + +fn window_stats( + occurrences: &[&FailureOccurrence], + start: OffsetDateTime, + end: OffsetDateTime, + duration_seconds: i64, +) -> WindowStats { + let count = u32::try_from(occurrences.len()).unwrap_or(u32::MAX); + let distinct_sessions = occurrences + .iter() + .map(|occurrence| occurrence.session_id.as_str()) + .collect::>() + .len(); + WindowStats { + start: rfc3339(start), + end: rfc3339(end), + duration_seconds, + occurrences: count, + distinct_sessions: u32::try_from(distinct_sessions).unwrap_or(u32::MAX), + rate_per_week_milli: rate_per_week_milli(count, duration_seconds), + } +} + +/// Occurrences per week scaled by 1000, so the report carries no floats. +fn rate_per_week_milli(occurrences: u32, duration_seconds: i64) -> i64 { + if duration_seconds <= 0 { + return 0; + } + i64::from(occurrences) * WEEK_SECONDS * 1_000 / duration_seconds +} + +fn coverage(observations: &EfficacyObservations) -> Coverage { + let classifiable = observations.coverage.classifiable_failures; + let total = observations.coverage.total_failures; + let coverage_bps = if total == 0 { + // No failures at all in the span: coverage is trivially complete. + 10_000 + } else { + u32::try_from(u64::from(classifiable) * 10_000 / u64::from(total)).unwrap_or(10_000) + }; + Coverage { + classifiable_failures: classifiable, + total_failures: total, + coverage_bps, + complete: classifiable == total, + } +} + +fn insufficient_reasons( + post_duration_seconds: i64, + pre: &WindowStats, + post: &WindowStats, + coverage: &Coverage, +) -> Vec { + let mut reasons = Vec::new(); + if post_duration_seconds < MIN_POST_WINDOW_SECONDS { + reasons.push(InsufficientReason::PostWindowTooShort); + } + if pre.occurrences.saturating_add(post.occurrences) < MIN_TOTAL_OCCURRENCES { + reasons.push(InsufficientReason::SparseOccurrences); + } + if coverage.coverage_bps < MIN_COVERAGE_BPS { + reasons.push(InsufficientReason::PartialIndexCoverage); + } + reasons +} + +/// Signed relative change in weekly rate, in basis points, when a nonzero +/// baseline exists. Because the windows are equal-length, this reduces to the +/// relative change in raw counts. +fn rate_delta_bps(pre: &WindowStats, post: &WindowStats) -> Option { + if pre.rate_per_week_milli == 0 { + return None; + } + let delta = + (post.rate_per_week_milli - pre.rate_per_week_milli) * 10_000 / pre.rate_per_week_milli; + Some(i32::try_from(delta).unwrap_or(if delta < 0 { i32::MIN } else { i32::MAX })) +} + +fn verdict( + insufficient_reasons: &[InsufficientReason], + pre: &WindowStats, + post: &WindowStats, + rate_delta_bps: Option, +) -> Verdict { + if !insufficient_reasons.is_empty() { + return Verdict::InsufficientData; + } + match rate_delta_bps { + // No pre-window baseline, yet enough occurrences survived the sparse + // gate: the recurrence is new after install. + None => { + if post.occurrences > pre.occurrences { + Verdict::Regressed + } else { + Verdict::Unchanged + } + } + Some(delta) if delta <= -UNCHANGED_BAND_BPS => Verdict::Improved, + Some(delta) if delta >= UNCHANGED_BAND_BPS => Verdict::Regressed, + Some(_) => Verdict::Unchanged, + } +} + +fn evidence(pre: &[&FailureOccurrence], post: &[&FailureOccurrence]) -> Evidence { + let cap = EVIDENCE_LISTING_CAP as usize; + let list = |occurrences: &[&FailureOccurrence]| { + occurrences + .iter() + .take(cap) + .map(|occurrence| occurrence.event_id.clone()) + .collect::>() + }; + Evidence { + pre_event_ids: list(pre), + post_event_ids: list(post), + pre_event_count: u32::try_from(pre.len()).unwrap_or(u32::MAX), + post_event_count: u32::try_from(post.len()).unwrap_or(u32::MAX), + listing_cap: EVIDENCE_LISTING_CAP, + } +} + +fn rfc3339(instant: OffsetDateTime) -> String { + instant + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .unwrap_or_else(|_| String::from("0000-01-01T00:00:00Z")) +} + +fn prefixed_hash(prefix: &str, bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut encoded = String::with_capacity(digest.len() * 2); + for byte in digest { + write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); + } + format!("{prefix}_{encoded}") +} + +/// Error produced before an efficacy report is complete. +#[derive(Debug, thiserror::Error)] +pub enum EfficacyError { + /// The observation set violated its semantic contract. + #[error("invalid efficacy observations: {0}")] + InvalidObservations(#[from] EfficacyValidationErrors), + /// The evaluation clock preceded the install timestamp. + #[error("evaluation clock {evaluated_at} precedes install {installed_at}")] + NonMonotonicClock { + /// Install timestamp. + installed_at: String, + /// Evaluation clock. + evaluated_at: String, + }, + /// The report could not be canonicalized for hashing. + #[error("could not serialize efficacy report: {0}")] + Serialization(#[from] serde_json::Error), +} diff --git a/crates/autophagy-efficacy/src/lib.rs b/crates/autophagy-efficacy/src/lib.rs new file mode 100644 index 0000000..a4b1791 --- /dev/null +++ b/crates/autophagy-efficacy/src/lib.rs @@ -0,0 +1,25 @@ +//! Versioned, deterministic post-install efficacy evaluation. +//! +//! Efficacy v0.1 answers one empirical, model-free question about an installed +//! mutation: does the exact failure signature it addresses recur *less* in the +//! observation window since `installed_at` than it did in an equal window +//! immediately before? It makes no causal claim, invokes no model, and never +//! changes a mutation's lifecycle state — it only measures recurrence. +//! +//! This crate is a pure function of the inputs handed to it. Gathering the +//! failure occurrences and index-coverage counts from the local event store is +//! the store's job; wiring the two together is the CLI's. Given the same +//! observations, `installed_at`, and evaluation clock, [`evaluate`] always +//! produces the same report and the same content-derived `efficacy_id`. + +mod evaluate; +mod model; +mod validate; + +pub use evaluate::{EfficacyError, WindowBounds, evaluate}; +pub use model::{ + Coverage, CoverageInput, EfficacyObservations, EfficacyReport, EfficacyResultSpecVersion, + EfficacyWindows, Evidence, FailureOccurrence, InsufficientReason, MatchingRule, Verdict, + WindowStats, +}; +pub use validate::{EfficacyValidationError, EfficacyValidationErrors}; diff --git a/crates/autophagy-efficacy/src/model.rs b/crates/autophagy-efficacy/src/model.rs new file mode 100644 index 0000000..6bafeef --- /dev/null +++ b/crates/autophagy-efficacy/src/model.rs @@ -0,0 +1,200 @@ +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; + +use crate::EfficacyValidationErrors; + +/// Efficacy report wire-format version. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum EfficacyResultSpecVersion { + /// Initial deterministic recurrence-efficacy report. + #[serde(rename = "efficacy/0.1")] + V0_1, +} + +/// The deterministic rule used to count failure occurrences for a mutation. +/// +/// A mutation's trigger selector is a versioned failure signature +/// (`failure/|||exit:`). The store decomposes it into +/// the outcome-independent operation signature (`operation/||`) +/// plus an exit code, then counts the `tool.failed` events indexed under that +/// operation signature whose exit code matches. Operation-form selectors +/// (`operation/|…`) match any indexed `tool.failed` event for that operation. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MatchingRule { + /// Failure-signature recurrence over the exact-signature index. + FailureSignatureRecurrence, +} + +/// One failure occurrence counted from the local event store. Input only; never +/// serialized into the report (the report cites `event_id`s under +/// [`Evidence`]). +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FailureOccurrence { + /// Exact AEP event identity. + pub event_id: String, + /// Session the failure occurred in. + pub session_id: String, + /// Canonical event timestamp. + pub occurred_at: OffsetDateTime, +} + +/// Raw index-coverage counts over the full evaluated span, gathered by the store. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CoverageInput { + /// `tool.failed` events in the span that carry an exact-signature index row + /// (and are therefore classifiable by signature). + pub classifiable_failures: u32, + /// All `tool.failed` events in the span, indexed or not. + pub total_failures: u32, +} + +/// Everything [`evaluate`](crate::evaluate) needs, handed in by the caller. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EfficacyObservations { + /// Installed mutation identity. + pub mutation_id: String, + /// Immutable mutation semantic version. + pub mutation_version: String, + /// Immutable trigger selectors the recurrence is measured against. + pub signature_selectors: Vec, + /// The rule the store applied to count occurrences. + pub matching_rule: MatchingRule, + /// Every matched failure occurrence within the full evaluated span. + pub occurrences: Vec, + /// Index-coverage diagnostics over the span. + pub coverage: CoverageInput, +} + +impl EfficacyObservations { + /// Enforce efficacy-observation semantic invariants. + /// + /// # Errors + /// Returns every invalid field and cross-field relationship. + pub fn validate(&self) -> Result<(), EfficacyValidationErrors> { + crate::validate::validate(self) + } +} + +/// Integer-only recurrence measurements for one window. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct WindowStats { + /// Inclusive window start (RFC 3339 UTC). + pub start: String, + /// Window end (RFC 3339 UTC): the pre-window ends at `installed_at`, the + /// post-window ends at the evaluation clock. + pub end: String, + /// Window length in whole seconds. + pub duration_seconds: i64, + /// Matched failure occurrences in the window. + pub occurrences: u32, + /// Distinct sessions those occurrences fell across. + pub distinct_sessions: u32, + /// Occurrences per week, scaled by 1000 (milli-occurrences per week) to stay + /// integer and deterministic. `1810` means `1.81` failures per week. + pub rate_per_week_milli: i64, +} + +/// The symmetric pre/post comparison windows. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct EfficacyWindows { + /// Equal-length window immediately before `installed_at`. + pub pre: WindowStats, + /// Observation window since `installed_at`. + pub post: WindowStats, +} + +/// Exact-signature index coverage for the evaluated span. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Coverage { + /// `tool.failed` events classifiable by signature. + pub classifiable_failures: u32, + /// All `tool.failed` events in the span. + pub total_failures: u32, + /// Classifiable fraction in basis points (`10000` = fully indexed). + pub coverage_bps: u32, + /// Whether every in-span failure was classifiable. + pub complete: bool, +} + +/// Exact evidence identifiers for the counted occurrences. +/// +/// Counts are always exact; the listed identifiers are capped at +/// `listing_cap` so a pathological corpus cannot produce an unbounded report. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Evidence { + /// Up to `listing_cap` pre-window occurrence event IDs, in canonical order. + pub pre_event_ids: Vec, + /// Up to `listing_cap` post-window occurrence event IDs, in canonical order. + pub post_event_ids: Vec, + /// Exact pre-window occurrence count (never capped). + pub pre_event_count: u32, + /// Exact post-window occurrence count (never capped). + pub post_event_count: u32, + /// Maximum identifiers listed per window. + pub listing_cap: u32, +} + +/// Deterministic efficacy verdict. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Verdict { + /// The signature recurs meaningfully less after install. + Improved, + /// The signature recurs meaningfully more after install. + Regressed, + /// The recurrence rate is within the neutral band. + Unchanged, + /// Too little observation, evidence, or index coverage to judge. + InsufficientData, +} + +/// Why a verdict is [`Verdict::InsufficientData`]. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InsufficientReason { + /// The post-install observation window is shorter than the minimum. + PostWindowTooShort, + /// Too few total occurrences across both windows to distinguish signal. + SparseOccurrences, + /// Too many in-span failures lack an index row to trust the counts. + PartialIndexCoverage, +} + +/// Complete deterministic post-install efficacy report. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct EfficacyReport { + /// Report contract version. + pub spec_version: EfficacyResultSpecVersion, + /// Stable content-derived report identity (`eff_`). + pub efficacy_id: String, + /// Installed mutation identity. + pub mutation_id: String, + /// Immutable mutation semantic version. + pub mutation_version: String, + /// Trigger selectors the recurrence was measured against, in canonical order. + pub signature_selectors: Vec, + /// Rule used to count occurrences. + pub matching_rule: MatchingRule, + /// Install timestamp anchoring the windows (RFC 3339 UTC). + pub installed_at: String, + /// Evaluation clock echoed for reproducibility (RFC 3339 UTC). + pub evaluated_at: String, + /// The symmetric pre/post windows and their measurements. + pub windows: EfficacyWindows, + /// Signed relative change in weekly rate, in basis points, when a nonzero + /// baseline exists (`-3300` = a 33% reduction). Absent when the pre-window + /// held no occurrences. + #[serde(skip_serializing_if = "Option::is_none")] + pub rate_delta_bps: Option, + /// Index-coverage diagnostics. + pub coverage: Coverage, + /// Deterministic verdict. + pub verdict: Verdict, + /// Every reason the verdict is insufficient; empty otherwise. + pub insufficient_reasons: Vec, + /// Exact evidence identifiers counted in each window. + pub evidence: Evidence, + /// Always false: efficacy is model-free. + pub model_used: bool, +} diff --git a/crates/autophagy-efficacy/src/validate.rs b/crates/autophagy-efficacy/src/validate.rs new file mode 100644 index 0000000..58c6995 --- /dev/null +++ b/crates/autophagy-efficacy/src/validate.rs @@ -0,0 +1,143 @@ +use std::{collections::BTreeSet, fmt}; + +use crate::EfficacyObservations; + +/// One efficacy-observation semantic violation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EfficacyValidationError { + /// Stable machine-readable code. + pub code: &'static str, + /// Field path containing the violation. + pub field: String, + /// Human-readable explanation. + pub message: String, +} + +/// All semantic violations found in an efficacy observation set. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EfficacyValidationErrors(Vec); + +impl EfficacyValidationErrors { + /// Iterate over every violation. + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } +} + +impl fmt::Display for EfficacyValidationErrors { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + for (index, error) in self.0.iter().enumerate() { + if index > 0 { + formatter.write_str("; ")?; + } + write!( + formatter, + "{} [{}]: {}", + error.field, error.code, error.message + )?; + } + Ok(()) + } +} + +impl std::error::Error for EfficacyValidationErrors {} + +pub(crate) fn validate( + observations: &EfficacyObservations, +) -> Result<(), EfficacyValidationErrors> { + let mut errors = Vec::new(); + if !valid_id(&observations.mutation_id, "mut_") { + push(&mut errors, "id", "mutation_id", "must begin with mut_"); + } + if observations.mutation_version.trim().is_empty() { + push( + &mut errors, + "required", + "mutation_version", + "must not be blank", + ); + } + if observations.signature_selectors.is_empty() { + push( + &mut errors, + "required", + "signature_selectors", + "must not be empty", + ); + } + let mut seen_selectors = BTreeSet::new(); + for (index, selector) in observations.signature_selectors.iter().enumerate() { + if selector.trim().is_empty() { + push( + &mut errors, + "invalid", + format!("signature_selectors[{index}]"), + "must not be blank", + ); + } else if !seen_selectors.insert(selector) { + push( + &mut errors, + "duplicate", + format!("signature_selectors[{index}]"), + "must be unique", + ); + } + } + let mut seen_events = BTreeSet::new(); + for (index, occurrence) in observations.occurrences.iter().enumerate() { + let prefix = format!("occurrences[{index}]"); + if !valid_id(&occurrence.event_id, "evt_") { + push( + &mut errors, + "id", + format!("{prefix}.event_id"), + "must begin with evt_", + ); + } else if !seen_events.insert(&occurrence.event_id) { + push( + &mut errors, + "duplicate", + format!("{prefix}.event_id"), + "must not be counted twice", + ); + } + if occurrence.session_id.trim().is_empty() { + push( + &mut errors, + "required", + format!("{prefix}.session_id"), + "must not be blank", + ); + } + } + if observations.coverage.classifiable_failures > observations.coverage.total_failures { + push( + &mut errors, + "invalid", + "coverage.classifiable_failures", + "must not exceed total_failures", + ); + } + if errors.is_empty() { + Ok(()) + } else { + Err(EfficacyValidationErrors(errors)) + } +} + +fn valid_id(value: &str, prefix: &str) -> bool { + value.starts_with(prefix) && value.len() > prefix.len() +} + +fn push( + errors: &mut Vec, + code: &'static str, + field: impl Into, + message: impl Into, +) { + errors.push(EfficacyValidationError { + code, + field: field.into(), + message: message.into(), + }); +} diff --git a/crates/autophagy-efficacy/tests/efficacy.rs b/crates/autophagy-efficacy/tests/efficacy.rs new file mode 100644 index 0000000..e107660 --- /dev/null +++ b/crates/autophagy-efficacy/tests/efficacy.rs @@ -0,0 +1,310 @@ +//! Deterministic evaluation math, verdict thresholds, and schema conformance. + +use autophagy_efficacy::{ + CoverageInput, EfficacyObservations, EfficacyReport, FailureOccurrence, InsufficientReason, + MatchingRule, Verdict, WindowBounds, evaluate, +}; +use serde_json::Value; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; + +const RESULT_SCHEMA: &str = include_str!("../../../docs/specs/efficacy/0.1/result.schema.json"); +const VALID: &[&str] = &[ + include_str!("../../../docs/specs/efficacy/0.1/valid/improved.json"), + include_str!("../../../docs/specs/efficacy/0.1/valid/regressed_no_baseline.json"), + include_str!("../../../docs/specs/efficacy/0.1/valid/insufficient_data.json"), +]; +const INVALID: &[&str] = &[ + include_str!("../../../docs/specs/efficacy/0.1/invalid/bad_spec_version.json"), + include_str!("../../../docs/specs/efficacy/0.1/invalid/bad_efficacy_id.json"), + include_str!("../../../docs/specs/efficacy/0.1/invalid/unknown_verdict.json"), + include_str!("../../../docs/specs/efficacy/0.1/invalid/bad_matching_rule.json"), + include_str!("../../../docs/specs/efficacy/0.1/invalid/model_used_true.json"), + include_str!("../../../docs/specs/efficacy/0.1/invalid/unknown_field.json"), + include_str!("../../../docs/specs/efficacy/0.1/invalid/bad_insufficient_reason.json"), +]; + +fn at(timestamp: &str) -> OffsetDateTime { + OffsetDateTime::parse(timestamp, &Rfc3339).expect("timestamp") +} + +fn occurrence(event_id: &str, session_id: &str, timestamp: &str) -> FailureOccurrence { + FailureOccurrence { + event_id: event_id.to_owned(), + session_id: session_id.to_owned(), + occurred_at: at(timestamp), + } +} + +fn observations( + occurrences: Vec, + coverage: CoverageInput, +) -> EfficacyObservations { + EfficacyObservations { + mutation_id: "mut_efficacy-fixture".to_owned(), + mutation_version: "0.1.0".to_owned(), + signature_selectors: vec!["failure/v2|shell|go build|exit:1".to_owned()], + matching_rule: MatchingRule::FailureSignatureRecurrence, + occurrences, + coverage, + } +} + +#[test] +fn improvement_is_detected_with_a_negative_rate_delta() { + // Three failures in the pre-window, none after install: a clear reduction. + let occurrences = vec![ + occurrence("evt_p1", "ses_a", "2026-01-10T00:00:00Z"), + occurrence("evt_p2", "ses_b", "2026-02-01T00:00:00Z"), + occurrence("evt_p3", "ses_b", "2026-03-01T00:00:00Z"), + ]; + let installed_at = at("2026-04-01T00:00:00Z"); + let now = at("2026-07-01T00:00:00Z"); + let report = evaluate( + &observations( + occurrences, + CoverageInput { + classifiable_failures: 50, + total_failures: 50, + }, + ), + installed_at, + now, + ) + .expect("evaluate"); + + assert_eq!(report.verdict, Verdict::Improved); + assert!(report.insufficient_reasons.is_empty()); + assert_eq!(report.windows.pre.occurrences, 3); + assert_eq!(report.windows.pre.distinct_sessions, 2); + assert_eq!(report.windows.post.occurrences, 0); + assert_eq!(report.rate_delta_bps, Some(-10_000)); + assert_eq!(report.evidence.pre_event_count, 3); + assert_eq!(report.evidence.post_event_count, 0); + assert!(!report.model_used); + assert!(report.efficacy_id.starts_with("eff_")); +} + +#[test] +fn a_new_recurrence_after_install_regresses_without_a_baseline() { + let occurrences = vec![ + occurrence("evt_q1", "ses_a", "2026-05-01T00:00:00Z"), + occurrence("evt_q2", "ses_a", "2026-06-01T00:00:00Z"), + ]; + let report = evaluate( + &observations( + occurrences, + CoverageInput { + classifiable_failures: 20, + total_failures: 20, + }, + ), + at("2026-04-01T00:00:00Z"), + at("2026-07-01T00:00:00Z"), + ) + .expect("evaluate"); + assert_eq!(report.verdict, Verdict::Regressed); + assert_eq!(report.rate_delta_bps, None); + assert_eq!(report.windows.pre.occurrences, 0); + assert_eq!(report.windows.post.occurrences, 2); +} + +#[test] +fn a_small_change_within_the_band_is_unchanged() { + // 4 pre, 4 post — identical rate. + let mut occurrences = Vec::new(); + for (index, day) in ["2026-01-05", "2026-01-15", "2026-02-05", "2026-02-15"] + .into_iter() + .enumerate() + { + occurrences.push(occurrence( + &format!("evt_pre{index}"), + "ses_pre", + &format!("{day}T00:00:00Z"), + )); + } + for (index, day) in ["2026-04-05", "2026-04-15", "2026-05-05", "2026-05-15"] + .into_iter() + .enumerate() + { + occurrences.push(occurrence( + &format!("evt_post{index}"), + "ses_post", + &format!("{day}T00:00:00Z"), + )); + } + let report = evaluate( + &observations( + occurrences, + CoverageInput { + classifiable_failures: 30, + total_failures: 30, + }, + ), + at("2026-03-01T00:00:00Z"), + at("2026-06-01T00:00:00Z"), + ) + .expect("evaluate"); + assert_eq!(report.verdict, Verdict::Unchanged); + assert_eq!(report.rate_delta_bps, Some(0)); +} + +#[test] +fn a_short_post_window_is_insufficient() { + let occurrences = vec![ + occurrence("evt_r1", "ses_a", "2026-06-25T00:00:00Z"), + occurrence("evt_r2", "ses_a", "2026-07-02T00:00:00Z"), + ]; + // Post window is only three days. + let report = evaluate( + &observations( + occurrences, + CoverageInput { + classifiable_failures: 10, + total_failures: 10, + }, + ), + at("2026-06-30T00:00:00Z"), + at("2026-07-03T00:00:00Z"), + ) + .expect("evaluate"); + assert_eq!(report.verdict, Verdict::InsufficientData); + assert!( + report + .insufficient_reasons + .contains(&InsufficientReason::PostWindowTooShort) + ); +} + +#[test] +fn partial_index_coverage_forces_insufficient_data() { + let occurrences = vec![ + occurrence("evt_s1", "ses_a", "2026-01-10T00:00:00Z"), + occurrence("evt_s2", "ses_a", "2026-02-10T00:00:00Z"), + occurrence("evt_s3", "ses_a", "2026-03-10T00:00:00Z"), + ]; + // Only 4 of 40 in-span failures are classifiable: 1000 bps, well below 5000. + let report = evaluate( + &observations( + occurrences, + CoverageInput { + classifiable_failures: 4, + total_failures: 40, + }, + ), + at("2026-04-01T00:00:00Z"), + at("2026-07-01T00:00:00Z"), + ) + .expect("evaluate"); + assert_eq!(report.verdict, Verdict::InsufficientData); + assert!( + report + .insufficient_reasons + .contains(&InsufficientReason::PartialIndexCoverage) + ); + assert_eq!(report.coverage.coverage_bps, 1_000); + assert!(!report.coverage.complete); +} + +#[test] +fn identical_inputs_produce_an_identical_content_id() { + let make = || { + evaluate( + &observations( + vec![ + occurrence("evt_p1", "ses_a", "2026-01-10T00:00:00Z"), + occurrence("evt_p2", "ses_a", "2026-02-10T00:00:00Z"), + occurrence("evt_p3", "ses_a", "2026-03-10T00:00:00Z"), + ], + CoverageInput { + classifiable_failures: 50, + total_failures: 50, + }, + ), + at("2026-04-01T00:00:00Z"), + at("2026-07-01T00:00:00Z"), + ) + .expect("evaluate") + }; + let first = make(); + let second = make(); + assert_eq!(first.efficacy_id, second.efficacy_id); + // A later evaluation clock yields a different content id (history, not a + // collision): the same corpus at a new `now` is a new report. + let later = evaluate( + &observations( + vec![ + occurrence("evt_p1", "ses_a", "2026-01-10T00:00:00Z"), + occurrence("evt_p2", "ses_a", "2026-02-10T00:00:00Z"), + occurrence("evt_p3", "ses_a", "2026-03-10T00:00:00Z"), + ], + CoverageInput { + classifiable_failures: 50, + total_failures: 50, + }, + ), + at("2026-04-01T00:00:00Z"), + at("2026-07-02T00:00:00Z"), + ) + .expect("evaluate"); + assert_ne!(first.efficacy_id, later.efficacy_id); +} + +#[test] +fn a_backwards_clock_is_rejected() { + let result = WindowBounds::derive(at("2026-07-01T00:00:00Z"), at("2026-06-01T00:00:00Z")); + assert!(result.is_err()); +} + +#[test] +fn a_computed_report_conforms_to_its_own_schema() { + let report = evaluate( + &observations( + vec![ + occurrence("evt_p1", "ses_a", "2026-01-10T00:00:00Z"), + occurrence("evt_p2", "ses_b", "2026-02-10T00:00:00Z"), + occurrence("evt_p3", "ses_b", "2026-03-10T00:00:00Z"), + ], + CoverageInput { + classifiable_failures: 48, + total_failures: 50, + }, + ), + at("2026-04-01T00:00:00Z"), + at("2026-07-01T00:00:00Z"), + ) + .expect("evaluate"); + let schema: Value = serde_json::from_str(RESULT_SCHEMA).expect("schema JSON"); + let validator = jsonschema::validator_for(&schema).expect("compile schema"); + let instance = serde_json::to_value(&report).expect("serialize report"); + assert!( + validator.is_valid(&instance), + "schema rejected a freshly computed report" + ); +} + +#[test] +fn fixtures_round_trip_through_the_schema_and_types() { + let schema: Value = serde_json::from_str(RESULT_SCHEMA).expect("schema JSON"); + assert_eq!( + schema["properties"]["spec_version"]["const"], + "efficacy/0.1" + ); + let validator = jsonschema::validator_for(&schema).expect("compile schema"); + + for fixture in VALID { + let instance: Value = serde_json::from_str(fixture).expect("valid fixture JSON"); + assert!(validator.is_valid(&instance), "schema rejected {fixture}"); + // Every valid fixture also round-trips through the Rust type. + let report: EfficacyReport = + serde_json::from_str(fixture).expect("deserialize into report"); + let reserialized = serde_json::to_value(&report).expect("reserialize"); + assert!( + validator.is_valid(&reserialized), + "round-trip broke {fixture}" + ); + } + for fixture in INVALID { + let instance: Value = serde_json::from_str(fixture).expect("invalid fixture JSON"); + assert!(!validator.is_valid(&instance), "schema accepted {fixture}"); + } +} diff --git a/crates/autophagy-store/migrations/0003_mutation_efficacy.sql b/crates/autophagy-store/migrations/0003_mutation_efficacy.sql new file mode 100644 index 0000000..fa031c7 --- /dev/null +++ b/crates/autophagy-store/migrations/0003_mutation_efficacy.sql @@ -0,0 +1,54 @@ +-- Post-install mutation efficacy tracking (observational, non-causal). +-- +-- Once a mutation is installed as a repo-scoped skill, the only honest question +-- left is empirical: does the exact failure signature it addresses recur less +-- after `installed_at` than before? This table stores the deterministic, +-- model-free answer to that question — a versioned `efficacy/0.1` report keyed +-- by a content fingerprint of its exact inputs (the mutation, its selectors, the +-- install timestamp, and the evaluation clock). See ADR 0015. +-- +-- History is intentional. Efficacy is a moving measurement: each evaluation runs +-- against more post-install observation time, so multiple rows accumulate per +-- mutation over time and the table deliberately does NOT constrain to one row +-- per mutation. That accumulation is the point of tracking. +-- +-- Unlike replay and shadow, efficacy carries no lifecycle gate: it never +-- advances or retires a mutation, so registration performs no state transition. +-- It is ordered after the v2 findings cache and, like every migration from the +-- first release onward, is immutable — add new migrations, never edit this one. + +CREATE TABLE mutation_efficacy ( + efficacy_id TEXT PRIMARY KEY CHECK (efficacy_id LIKE 'eff_%'), + mutation_id TEXT NOT NULL REFERENCES mutation_candidates(mutation_id) ON DELETE CASCADE, + report_json TEXT NOT NULL CHECK (json_valid(report_json)), + content_hash BLOB NOT NULL CHECK (length(content_hash) = 32), + verdict TEXT NOT NULL CHECK ( + verdict IN ('improved', 'regressed', 'unchanged', 'insufficient_data') + ), + created_at TEXT NOT NULL +) STRICT; + +-- Exact evidence identifiers for every failure event counted in either window, +-- mirroring mutation_shadow_evidence. Rows cascade on event deletion. +CREATE TABLE mutation_efficacy_evidence ( + efficacy_id TEXT NOT NULL REFERENCES mutation_efficacy(efficacy_id) ON DELETE CASCADE, + event_id TEXT NOT NULL REFERENCES events(event_id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + PRIMARY KEY (efficacy_id, ordinal), + UNIQUE (efficacy_id, event_id) +) STRICT; + +CREATE INDEX mutation_efficacy_candidate_time + ON mutation_efficacy(mutation_id, created_at, efficacy_id); + +-- Removing any cited evidence event drops the efficacy report that cited it, so +-- a stored report can never reference a deleted event. This mirrors the shadow +-- and replay evidence triggers in shape, but deletes only the report — never the +-- mutation candidate. Efficacy is observational and gates no lifecycle state, so +-- losing an efficacy report must not retire an otherwise-valid mutation (ADR +-- 0015). Deleting the report cascades to its remaining evidence rows. +CREATE TRIGGER mutation_efficacy_evidence_removed +AFTER DELETE ON mutation_efficacy_evidence +BEGIN + DELETE FROM mutation_efficacy WHERE efficacy_id = OLD.efficacy_id; +END; diff --git a/crates/autophagy-store/src/error.rs b/crates/autophagy-store/src/error.rs index 486ba23..e95cddc 100644 --- a/crates/autophagy-store/src/error.rs +++ b/crates/autophagy-store/src/error.rs @@ -80,6 +80,12 @@ pub enum StoreError { /// Rejected zero-based evidence position. ordinal: usize, }, + /// An efficacy evidence position cannot fit `SQLite`'s integer representation. + #[error("efficacy evidence ordinal {ordinal} exceeds SQLite's integer range")] + EfficacyEvidenceOrdinalOutOfRange { + /// Rejected zero-based evidence position. + ordinal: usize, + }, /// An incremental cursor cannot fit `SQLite`'s signed integer representation. #[error("cursor {field} value {value} exceeds SQLite's integer range")] CursorOutOfRange { @@ -164,6 +170,15 @@ pub enum StoreError { /// Conflicting shadow identity. shadow_id: String, }, + /// Efficacy identity, mutation identity, hash, or verdict disagreed with its report. + #[error("efficacy registration does not match its versioned report")] + InvalidEfficacyRegistration, + /// Immutable efficacy content changed under the same ID. + #[error("efficacy '{efficacy_id}' already exists with different report content")] + EfficacyContentConflict { + /// Conflicting efficacy identity. + efficacy_id: String, + }, /// Installation registration violated the supported target contract. #[error("installation registration is invalid")] InvalidInstallationRegistration, diff --git a/crates/autophagy-store/src/lib.rs b/crates/autophagy-store/src/lib.rs index 142f135..fd06392 100644 --- a/crates/autophagy-store/src/lib.rs +++ b/crates/autophagy-store/src/lib.rs @@ -12,14 +12,15 @@ mod util; pub use error::StoreError; pub use model::{ - AdapterActivity, DeleteAllSummary, DeleteSummary, DetectionFingerprint, InsertOutcome, - InstallationRegistration, InstallationTransitionOutcome, MutationDetails, - MutationInstallationRecord, MutationRecord, MutationRegisterOutcome, MutationRegistration, - MutationReplayRecord, MutationShadowRecord, MutationTransition, MutationTransitionOutcome, - PruneSummary, RankingExplanation, RankingSignal, RankingSignalKind, RebuildSummary, - ReplayRegisterOutcome, ReplayRegistration, RetrievalFilter, RetrievalFilterField, RetrievalHit, - RetrievalMatchKind, RetrievalOutcome, RetrievalQuery, SearchHit, SearchProjection, - SessionSummary, ShadowRegisterOutcome, ShadowRegistration, SourceCursor, SourceIdentity, - StoreStats, + AdapterActivity, DeleteAllSummary, DeleteSummary, DetectionFingerprint, EfficacyOccurrence, + EfficacyOccurrences, EfficacyRegisterOutcome, EfficacyRegistration, EfficacyStatusSummary, + InsertOutcome, InstallationRegistration, InstallationTransitionOutcome, MutationDetails, + MutationEfficacyRecord, MutationInstallationRecord, MutationRecord, MutationRegisterOutcome, + MutationRegistration, MutationReplayRecord, MutationShadowRecord, MutationTransition, + MutationTransitionOutcome, PruneSummary, RankingExplanation, RankingSignal, RankingSignalKind, + RebuildSummary, ReplayRegisterOutcome, ReplayRegistration, RetrievalFilter, + RetrievalFilterField, RetrievalHit, RetrievalMatchKind, RetrievalOutcome, RetrievalQuery, + SearchHit, SearchProjection, SessionSummary, ShadowRegisterOutcome, ShadowRegistration, + SourceCursor, SourceIdentity, StoreStats, }; pub use store::EventStore; diff --git a/crates/autophagy-store/src/migration.rs b/crates/autophagy-store/src/migration.rs index fa91379..82a312e 100644 --- a/crates/autophagy-store/src/migration.rs +++ b/crates/autophagy-store/src/migration.rs @@ -37,6 +37,11 @@ const MIGRATIONS: &[Migration] = &[ description: "derived detection-findings cache", sql: include_str!("../migrations/0002_findings_cache.sql"), }, + Migration { + version: 3, + description: "post-install mutation efficacy tracking", + sql: include_str!("../migrations/0003_mutation_efficacy.sql"), + }, ]; /// SHA-256 checksums of the eight pre-release development migrations, in order diff --git a/crates/autophagy-store/src/model.rs b/crates/autophagy-store/src/model.rs index a089ad6..e94deb4 100644 --- a/crates/autophagy-store/src/model.rs +++ b/crates/autophagy-store/src/model.rs @@ -534,6 +534,8 @@ pub struct MutationDetails { pub shadows: Vec, /// Installation and rollback audit records. pub installations: Vec, + /// Post-install efficacy reports in creation order (oldest first). + pub efficacies: Vec, } /// Idempotent lifecycle transition result. @@ -661,6 +663,99 @@ pub enum ShadowRegisterOutcome { }, } +/// Owned post-install efficacy report ready for persistence. +/// +/// Registration is append-only and, unlike replay and shadow, performs no +/// lifecycle transition: efficacy observes recurrence, it never gates promotion. +#[derive(Clone, Debug, PartialEq)] +pub struct EfficacyRegistration { + /// Stable content-derived efficacy identity. + pub efficacy_id: String, + /// Measured mutation identity. + pub mutation_id: String, + /// Deterministic verdict (`improved`, `regressed`, `unchanged`, or + /// `insufficient_data`). + pub verdict: String, + /// Complete versioned efficacy report. + pub report: Value, + /// Exact failure-occurrence events cited across both windows, deduplicated. + pub source_event_ids: Vec, +} + +/// One persisted immutable efficacy report. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct MutationEfficacyRecord { + /// Stable efficacy report identity. + pub efficacy_id: String, + /// Measured mutation identity. + pub mutation_id: String, + /// Deterministic verdict. + pub verdict: String, + /// Complete versioned efficacy report. + pub report: Value, + /// Canonical persistence timestamp. + pub created_at: String, +} + +/// Idempotent efficacy persistence result. No lifecycle field: efficacy never +/// changes mutation state. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum EfficacyRegisterOutcome { + /// A new efficacy report was stored. + Inserted { + /// Stable efficacy report identity. + efficacy_id: String, + }, + /// The identical report was already stored. + Duplicate { + /// Stable efficacy report identity. + efficacy_id: String, + }, +} + +/// One matched failure occurrence gathered for efficacy evaluation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EfficacyOccurrence { + /// Exact AEP event identity. + pub event_id: String, + /// Session the failure occurred in. + pub session_id: String, + /// Canonical event timestamp (RFC 3339 UTC). + pub occurred_at: String, +} + +/// Failure occurrences and index-coverage counts for one evaluated span, +/// gathered from the event store for the pure efficacy evaluator to consume. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EfficacyOccurrences { + /// Matched failure occurrences within the span, deduplicated by event. + pub occurrences: Vec, + /// In-span `tool.failed` events carrying an exact-signature index row. + pub classifiable_failures: u32, + /// All in-span `tool.failed` events. + pub total_failures: u32, + /// Selectors that could not be parsed into the failure-matching rule. + pub unparsed_selectors: Vec, +} + +/// Efficacy coverage of the installed mutations, for the `status` summary. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +pub struct EfficacyStatusSummary { + /// Currently-installed mutations. + pub installed: u32, + /// Installed mutations whose latest verdict is `improved`. + pub improved: u32, + /// Installed mutations whose latest verdict is `regressed`. + pub regressed: u32, + /// Installed mutations whose latest verdict is `unchanged`. + pub unchanged: u32, + /// Installed mutations whose latest verdict is `insufficient_data`. + pub insufficient_data: u32, + /// Installed mutations with no efficacy report yet. + pub not_measured: u32, +} + /// Audited input for a completed filesystem materialization. #[derive(Clone, Debug, PartialEq)] pub struct InstallationRegistration { diff --git a/crates/autophagy-store/src/store.rs b/crates/autophagy-store/src/store.rs index 2367f83..3f39089 100644 --- a/crates/autophagy-store/src/store.rs +++ b/crates/autophagy-store/src/store.rs @@ -5,18 +5,19 @@ use rusqlite::{ Connection, OptionalExtension, Transaction, TransactionBehavior, params, params_from_iter, types::Value as SqlValue, }; -use time::OffsetDateTime; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use crate::{ - AdapterActivity, DeleteAllSummary, DeleteSummary, DetectionFingerprint, InsertOutcome, - InstallationRegistration, InstallationTransitionOutcome, MutationDetails, - MutationInstallationRecord, MutationRecord, MutationRegisterOutcome, MutationRegistration, - MutationReplayRecord, MutationShadowRecord, MutationTransition, MutationTransitionOutcome, - PruneSummary, RankingExplanation, RankingSignal, RankingSignalKind, RebuildSummary, - ReplayRegisterOutcome, ReplayRegistration, RetrievalFilter, RetrievalFilterField, RetrievalHit, - RetrievalMatchKind, RetrievalQuery, SearchHit, SearchProjection, SessionSummary, - ShadowRegisterOutcome, ShadowRegistration, SourceCursor, SourceIdentity, StoreError, - StoreStats, migration, util, + AdapterActivity, DeleteAllSummary, DeleteSummary, DetectionFingerprint, EfficacyOccurrence, + EfficacyOccurrences, EfficacyRegisterOutcome, EfficacyRegistration, EfficacyStatusSummary, + InsertOutcome, InstallationRegistration, InstallationTransitionOutcome, MutationDetails, + MutationEfficacyRecord, MutationInstallationRecord, MutationRecord, MutationRegisterOutcome, + MutationRegistration, MutationReplayRecord, MutationShadowRecord, MutationTransition, + MutationTransitionOutcome, PruneSummary, RankingExplanation, RankingSignal, RankingSignalKind, + RebuildSummary, ReplayRegisterOutcome, ReplayRegistration, RetrievalFilter, + RetrievalFilterField, RetrievalHit, RetrievalMatchKind, RetrievalQuery, SearchHit, + SearchProjection, SessionSummary, ShadowRegisterOutcome, ShadowRegistration, SourceCursor, + SourceIdentity, StoreError, StoreStats, migration, util, }; /// Score contribution for an exact normalized-signature match, in basis points. @@ -1235,15 +1236,49 @@ impl EventStore { let installations = rows .map(|row| installation_record(row?)) .collect::, StoreError>>()?; + let efficacies = self.efficacy_records(mutation_id)?; Ok(MutationDetails { mutation, transitions, replays, shadows, installations, + efficacies, }) } + /// Every efficacy report for one mutation, oldest first. + fn efficacy_records( + &self, + mutation_id: &str, + ) -> Result, StoreError> { + let mut statement = self.connection.prepare( + "SELECT efficacy_id, mutation_id, verdict, report_json, created_at + FROM mutation_efficacy WHERE mutation_id = ?1 + ORDER BY created_at, efficacy_id", + )?; + let rows = statement.query_map([mutation_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + })?; + rows.map(|row| { + let (efficacy_id, mutation_id, verdict, report, created_at) = row?; + Ok(MutationEfficacyRecord { + efficacy_id, + mutation_id, + verdict, + report: serde_json::from_str(&report)?, + created_at, + }) + }) + .collect::, StoreError>>() + } + /// Persist one immutable replay report and advance a challenged candidate only on pass. /// /// Failed reports remain auditable without changing lifecycle state. Identical @@ -1472,6 +1507,230 @@ impl EventStore { }) } + /// Persist one immutable post-install efficacy report. + /// + /// Efficacy is observational: registration performs no lifecycle transition + /// and never advances or retires the mutation. History accumulates — a + /// mutation may carry many reports over time — but re-registering an + /// identical report (same `efficacy_id`, same bytes) is a no-op, and reusing + /// an `efficacy_id` for different content is a conflict. + /// + /// # Errors + /// Returns [`StoreError`] for inconsistent metadata, content conflicts, + /// a missing mutation, or database failures. + pub fn register_efficacy( + &mut self, + registration: &EfficacyRegistration, + ) -> Result { + if !efficacy_report_matches_registration(registration) { + return Err(StoreError::InvalidEfficacyRegistration); + } + let report_json = serde_json::to_string(®istration.report)?; + let content_hash = util::sha256(report_json.as_bytes()); + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate)?; + if let Some(existing_hash) = transaction + .query_row( + "SELECT content_hash FROM mutation_efficacy WHERE efficacy_id = ?1", + [®istration.efficacy_id], + |row| row.get::<_, Vec>(0), + ) + .optional()? + { + if existing_hash.as_slice() != content_hash { + return Err(StoreError::EfficacyContentConflict { + efficacy_id: registration.efficacy_id.clone(), + }); + } + return Ok(EfficacyRegisterOutcome::Duplicate { + efficacy_id: registration.efficacy_id.clone(), + }); + } + // Confirm the mutation exists but never inspect or change its state: + // efficacy is decoupled from the lifecycle by design (ADR 0015). + transaction + .query_row( + "SELECT 1 FROM mutation_candidates WHERE mutation_id = ?1", + [®istration.mutation_id], + |_| Ok(()), + ) + .optional()? + .ok_or_else(|| StoreError::MutationNotFound { + mutation_id: registration.mutation_id.clone(), + })?; + let now = util::now_timestamp()?; + transaction.execute( + "INSERT INTO mutation_efficacy( + efficacy_id, mutation_id, report_json, content_hash, verdict, created_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + registration.efficacy_id, + registration.mutation_id, + report_json, + content_hash.as_slice(), + registration.verdict, + now, + ], + )?; + for (ordinal, event_id) in registration.source_event_ids.iter().enumerate() { + let stored_ordinal = i64::try_from(ordinal) + .map_err(|_| StoreError::EfficacyEvidenceOrdinalOutOfRange { ordinal })?; + transaction.execute( + "INSERT INTO mutation_efficacy_evidence(efficacy_id, event_id, ordinal) + VALUES (?1, ?2, ?3)", + params![registration.efficacy_id, event_id, stored_ordinal], + )?; + } + transaction.commit()?; + Ok(EfficacyRegisterOutcome::Inserted { + efficacy_id: registration.efficacy_id.clone(), + }) + } + + /// Every efficacy report for one mutation, oldest first. + /// + /// # Errors + /// Returns [`StoreError`] for database or deserialization failures. + pub fn list_efficacy( + &self, + mutation_id: &str, + ) -> Result, StoreError> { + self.efficacy_records(mutation_id) + } + + /// Gather matched failure occurrences and index-coverage counts for one + /// evaluated span, for the pure efficacy evaluator to consume. + /// + /// Each selector is decomposed by the deterministic failure-matching rule: + /// a `failure/|||exit:` selector yields the operation + /// signature `operation/||` and an exit code, and matches + /// the `tool.failed` events indexed under that operation signature with that + /// exit code. An `operation/|…` selector matches any indexed `tool.failed` + /// event for that operation. Selectors that parse as neither are returned in + /// `unparsed_selectors` and contribute no occurrences. + /// + /// Timestamps are compared as parsed instants (never lexically), so mixed + /// sub-second precision in stored events cannot misclassify a boundary. + /// + /// # Errors + /// Returns [`StoreError`] for database failures. + pub fn efficacy_occurrences( + &self, + selectors: &[String], + span_start: OffsetDateTime, + span_end: OffsetDateTime, + ) -> Result { + let mut matched: BTreeMap = BTreeMap::new(); + let mut unparsed_selectors = Vec::new(); + let mut statement = self.connection.prepare( + "SELECT e.event_id, e.session_id, e.occurred_at, e.exit_code + FROM event_signatures s + JOIN events e ON e.row_id = s.event_row_id + WHERE s.signature = ?1 AND e.event_type = 'tool.failed'", + )?; + for selector in selectors { + let Some(rule) = FailureSelector::parse(selector) else { + unparsed_selectors.push(selector.clone()); + continue; + }; + let rows = statement.query_map([&rule.operation_signature], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + )) + })?; + for row in rows { + let (event_id, session_id, occurred_at, exit_code) = row?; + if let Some(required) = rule.exit_code { + if exit_code != Some(required) { + continue; + } + } + if !within_span(&occurred_at, span_start, span_end) { + continue; + } + matched + .entry(event_id.clone()) + .or_insert(EfficacyOccurrence { + event_id, + session_id, + occurred_at, + }); + } + } + let (classifiable_failures, total_failures) = + self.failure_coverage(span_start, span_end)?; + Ok(EfficacyOccurrences { + occurrences: matched.into_values().collect(), + classifiable_failures, + total_failures, + unparsed_selectors, + }) + } + + /// Count in-span `tool.failed` events, and how many carry an index row. + fn failure_coverage( + &self, + span_start: OffsetDateTime, + span_end: OffsetDateTime, + ) -> Result<(u32, u32), StoreError> { + let mut statement = self.connection.prepare( + "SELECT e.occurred_at, + EXISTS(SELECT 1 FROM event_signatures s WHERE s.event_row_id = e.row_id) + FROM events e WHERE e.event_type = 'tool.failed'", + )?; + let rows = statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, bool>(1)?)) + })?; + let mut classifiable = 0u32; + let mut total = 0u32; + for row in rows { + let (occurred_at, indexed) = row?; + if within_span(&occurred_at, span_start, span_end) { + total = total.saturating_add(1); + if indexed { + classifiable = classifiable.saturating_add(1); + } + } + } + Ok((classifiable, total)) + } + + /// Summarize efficacy coverage across currently-installed mutations, using + /// each mutation's most recent efficacy verdict. `None`-style aggregation is + /// avoided: mutations with no report count toward `not_measured`. + /// + /// # Errors + /// Returns [`StoreError`] for database failures. + pub fn efficacy_status_summary(&self) -> Result { + let mut statement = self.connection.prepare( + "SELECT ( + SELECT verdict FROM mutation_efficacy me + WHERE me.mutation_id = i.mutation_id + ORDER BY created_at DESC, efficacy_id DESC LIMIT 1 + ) + FROM mutation_installations i WHERE i.state = 'installed'", + )?; + let rows = statement.query_map([], |row| row.get::<_, Option>(0))?; + let mut summary = EfficacyStatusSummary::default(); + for row in rows { + summary.installed = summary.installed.saturating_add(1); + match row?.as_deref() { + Some("improved") => summary.improved = summary.improved.saturating_add(1), + Some("regressed") => summary.regressed = summary.regressed.saturating_add(1), + Some("unchanged") => summary.unchanged = summary.unchanged.saturating_add(1), + Some("insufficient_data") => { + summary.insufficient_data = summary.insufficient_data.saturating_add(1); + } + _ => summary.not_measured = summary.not_measured.saturating_add(1), + } + } + Ok(summary) + } + /// Record a completed Codex repo-skill materialization and activate the mutation. /// /// # Errors @@ -2265,6 +2524,109 @@ fn shadow_report_matches_registration(registration: &ShadowRegistration) -> bool && report_event_ids == registered_event_ids } +/// A mutation trigger selector decomposed into an occurrence-matching query. +/// +/// Both selector grammars minted by the signature spec are supported: +/// `failure/|||exit:` (the common case) yields the +/// operation signature and an exit-code filter; `operation/||` +/// yields the operation signature with no exit filter. Command text may itself +/// contain `|` (shell pipes), so the failure form is parsed by trimming the +/// trailing `|exit:` rather than splitting on `|`. +struct FailureSelector { + operation_signature: String, + exit_code: Option, +} + +impl FailureSelector { + fn parse(selector: &str) -> Option { + if let Some(rest) = selector.strip_prefix("failure/") { + // rest = "|||exit:" + let (version, tail) = rest.split_once('|')?; + let (body, exit) = tail.rsplit_once("|exit:")?; + let exit_code = exit.parse::().ok()?; + // Require a tool|command boundary inside the body. + body.split_once('|')?; + Some(Self { + operation_signature: format!("operation/{version}|{body}"), + exit_code: Some(exit_code), + }) + } else if selector.starts_with("operation/") { + Some(Self { + operation_signature: selector.to_owned(), + exit_code: None, + }) + } else { + None + } + } +} + +/// Whether an RFC 3339 timestamp string falls in `[start, end]`, compared as +/// parsed instants. Unparseable timestamps are treated as out of span. +fn within_span(occurred_at: &str, start: OffsetDateTime, end: OffsetDateTime) -> bool { + match OffsetDateTime::parse(occurred_at, &Rfc3339) { + Ok(at) => at >= start && at <= end, + Err(_) => false, + } +} + +fn efficacy_report_matches_registration(registration: &EfficacyRegistration) -> bool { + if registration + .report + .get("efficacy_id") + .and_then(serde_json::Value::as_str) + != Some(®istration.efficacy_id) + || registration + .report + .get("mutation_id") + .and_then(serde_json::Value::as_str) + != Some(®istration.mutation_id) + || registration + .report + .get("verdict") + .and_then(serde_json::Value::as_str) + != Some(®istration.verdict) + { + return false; + } + let Some(evidence) = registration.report.get("evidence") else { + return false; + }; + let count = |field: &str| { + evidence + .get(field) + .and_then(serde_json::Value::as_u64) + .map(|value| usize::try_from(value).unwrap_or(usize::MAX)) + }; + let (Some(pre), Some(post)) = (count("pre_event_count"), count("post_event_count")) else { + return false; + }; + // The report lists a capped subset of evidence ids; the registration carries + // the exact, deduplicated union. Require the exact counts to agree and every + // listed id to appear in the registered set. + let registered = registration + .source_event_ids + .iter() + .map(String::as_str) + .collect::>(); + if registered.len() != registration.source_event_ids.len() + || registered.len() != pre.saturating_add(post) + { + return false; + } + ["pre_event_ids", "post_event_ids"] + .into_iter() + .flat_map(|field| { + evidence + .get(field) + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(serde_json::Value::as_str) + }) + .all(|id| registered.contains(id)) +} + fn valid_installation_registration(registration: &InstallationRegistration) -> bool { let target_path_prefix = match registration.target.as_str() { "codex_repo_skill" => ".agents/skills/", diff --git a/crates/autophagy-store/tests/legacy_adoption.rs b/crates/autophagy-store/tests/legacy_adoption.rs index 729d9a6..18e7cd4 100644 --- a/crates/autophagy-store/tests/legacy_adoption.rs +++ b/crates/autophagy-store/tests/legacy_adoption.rs @@ -100,16 +100,16 @@ fn known_legacy_v8_database_is_adopted_and_data_preserved() { ); // First open through the store adopts the baseline, collapsing the eight - // legacy rows to the v1 baseline, then applies the post-baseline chain (v2) - // on top — landing on the current released schema. + // legacy rows to the v1 baseline, then applies the post-baseline chain + // (v2, v3) on top — landing on the current released schema. { let store = EventStore::open(&path).expect("open adopts legacy database"); - assert_eq!(store.schema_version().expect("schema version"), 2); + assert_eq!(store.schema_version().expect("schema version"), 3); } assert_eq!( ledger_state(&path), - (2, 2, 2), + (3, 3, 3), "legacy chain replaced by the released baseline chain and user_version reset" ); assert_eq!( @@ -121,9 +121,9 @@ fn known_legacy_v8_database_is_adopted_and_data_preserved() { // Second open is a pure no-op: same schema, same ledger, data intact. { let store = EventStore::open(&path).expect("second open is a no-op"); - assert_eq!(store.schema_version().expect("schema version"), 2); + assert_eq!(store.schema_version().expect("schema version"), 3); } - assert_eq!(ledger_state(&path), (2, 2, 2)); + assert_eq!(ledger_state(&path), (3, 3, 3)); assert_eq!(data_counts(&path), before); } diff --git a/crates/autophagy-store/tests/store.rs b/crates/autophagy-store/tests/store.rs index c16bbce..edf8585 100644 --- a/crates/autophagy-store/tests/store.rs +++ b/crates/autophagy-store/tests/store.rs @@ -6,11 +6,12 @@ use autophagy_events::{ Artifact, ArtifactKind, Event, EventId, EventKind, SessionId, SpecVersion, ToolCall, }; use autophagy_store::{ - DeleteAllSummary, DeleteSummary, EventStore, InsertOutcome, InstallationRegistration, - InstallationTransitionOutcome, MutationRegisterOutcome, MutationRegistration, PruneSummary, - RebuildSummary, ReplayRegisterOutcome, ReplayRegistration, RetrievalMatchKind, - RetrievalOutcome, RetrievalQuery, SearchProjection, ShadowRegisterOutcome, ShadowRegistration, - SourceCursor, SourceIdentity, StoreError, StoreStats, + DeleteAllSummary, DeleteSummary, EfficacyRegisterOutcome, EfficacyRegistration, EventStore, + InsertOutcome, InstallationRegistration, InstallationTransitionOutcome, + MutationRegisterOutcome, MutationRegistration, PruneSummary, RebuildSummary, + ReplayRegisterOutcome, ReplayRegistration, RetrievalMatchKind, RetrievalOutcome, + RetrievalQuery, SearchProjection, ShadowRegisterOutcome, ShadowRegistration, SourceCursor, + SourceIdentity, StoreError, StoreStats, }; use serde_json::{Value, json}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; @@ -30,7 +31,7 @@ fn migrations_persist_and_reopen_cleanly() { { let mut store = EventStore::open(&database).expect("store should open"); - assert_eq!(store.schema_version().expect("schema version"), 2); + assert_eq!(store.schema_version().expect("schema version"), 3); assert!(matches!( store .insert_event(&source, &event, &SearchProjection::default()) @@ -40,7 +41,7 @@ fn migrations_persist_and_reopen_cleanly() { } let reopened = EventStore::open(&database).expect("store should reopen"); - assert_eq!(reopened.schema_version().expect("schema version"), 2); + assert_eq!(reopened.schema_version().expect("schema version"), 3); assert_eq!( reopened .get_event(event.event_id.as_str()) @@ -1681,3 +1682,384 @@ fn signature_index_preserves_idempotency_quarantine_and_deletion() { ["evt_b1"] ); } + +// -- Post-install efficacy tracking ------------------------------------------ + +const EFF_OP_SIG: &str = "operation/v2|shell|go build"; +const EFF_SELECTOR: &str = "failure/v2|shell|go build|exit:1"; + +/// Seed a store with an installed mutation and failure/success events indexed +/// under one operation signature, for efficacy-occurrence tests. +fn seed_efficacy_store() -> EventStore { + let mut store = EventStore::open_in_memory().expect("store"); + let source = source("instance-efficacy"); + let events = [ + // Matching failure (exit 1), indexed. + ( + retrieval_event( + "evt_ef_fail1", + "ses_pre", + EventKind::ToolFailed, + "go build", + Some(1), + "/workspace/project", + "2026-03-01T00:00:00Z", + 0, + ), + retrieval_projection("build failed", EFF_OP_SIG), + ), + // Same operation, exit 2: not a selector match, but a classifiable failure. + ( + retrieval_event( + "evt_ef_fail2", + "ses_pre", + EventKind::ToolFailed, + "go build", + Some(2), + "/workspace/project", + "2026-03-02T00:00:00Z", + 1, + ), + retrieval_projection("build failed differently", EFF_OP_SIG), + ), + // Success under the same operation: never a failure occurrence. + ( + retrieval_event( + "evt_ef_ok", + "ses_pre", + EventKind::ToolCompleted, + "go build", + Some(0), + "/workspace/project", + "2026-03-03T00:00:00Z", + 2, + ), + retrieval_projection("build ok", EFF_OP_SIG), + ), + // Matching failure in a second session, but never indexed (no signature). + ( + retrieval_event( + "evt_ef_fail3", + "ses_post", + EventKind::ToolFailed, + "go build", + Some(1), + "/workspace/project", + "2026-03-04T00:00:00Z", + 0, + ), + SearchProjection::default(), + ), + ]; + for (event, projection) in &events { + store + .insert_event(&source, event, projection) + .expect("insert efficacy event"); + } + store +} + +fn span(start: &str, end: &str) -> (OffsetDateTime, OffsetDateTime) { + ( + OffsetDateTime::parse(start, &Rfc3339).expect("start"), + OffsetDateTime::parse(end, &Rfc3339).expect("end"), + ) +} + +/// Register `mut_registry` through its full lifecycle into the `active` state. +fn seed_installed_mutation(store: &mut EventStore) { + for (event_id, session_id, timestamp, sequence) in [ + ( + "evt_mutation-support-a", + "ses_reg_a", + "2026-02-01T00:00:00Z", + 0, + ), + ( + "evt_mutation-support-b", + "ses_reg_b", + "2026-02-01T00:01:00Z", + 0, + ), + ( + "evt_mutation-counter", + "ses_reg_c", + "2026-02-01T00:02:00Z", + 0, + ), + ("evt_replay-only", "ses_reg_d", "2026-02-01T00:03:00Z", 0), + ] { + store + .insert_event( + &source("instance-efficacy"), + &session_event( + event_id, + session_id, + EventKind::DecisionRecorded, + timestamp, + sequence, + ), + &SearchProjection::default(), + ) + .expect("evidence event"); + } + store + .register_mutation(&mutation_registration( + "mut_registry", + "fnd_registry", + "eqv_registry", + )) + .expect("register"); + store + .challenge_mutation("mut_registry", &json!({"checks":["sessions_comparable"]})) + .expect("challenge"); + store + .register_replay(&replay_registration("rpl_passing", "rsh_passing", true)) + .expect("replay"); + store + .register_shadow(&shadow_registration("shr_passing", "shh_passing", true)) + .expect("shadow"); + store + .register_installation(&claude_code_installation_registration()) + .expect("install"); +} + +fn efficacy_registration( + efficacy_id: &str, + verdict: &str, + pre_event_ids: &[&str], + post_event_ids: &[&str], +) -> EfficacyRegistration { + let source_event_ids: Vec = pre_event_ids + .iter() + .chain(post_event_ids) + .map(|id| (*id).to_owned()) + .collect(); + EfficacyRegistration { + efficacy_id: efficacy_id.to_owned(), + mutation_id: "mut_registry".to_owned(), + verdict: verdict.to_owned(), + report: json!({ + "efficacy_id": efficacy_id, + "mutation_id": "mut_registry", + "verdict": verdict, + "evidence": { + "pre_event_ids": pre_event_ids, + "post_event_ids": post_event_ids, + "pre_event_count": pre_event_ids.len(), + "post_event_count": post_event_ids.len(), + "listing_cap": 50, + }, + }), + source_event_ids, + } +} + +#[test] +fn efficacy_occurrences_apply_the_failure_matching_rule_and_report_coverage() { + let store = seed_efficacy_store(); + let (start, end) = span("2026-01-01T00:00:00Z", "2026-12-31T00:00:00Z"); + let occurrences = store + .efficacy_occurrences(&[EFF_SELECTOR.to_owned()], start, end) + .expect("occurrences"); + + // Only the exit-1 failure indexed under the operation signature matches. The + // exit-2 failure, the success, and the unindexed failure are all excluded. + let ids: Vec<&str> = occurrences + .occurrences + .iter() + .map(|occurrence| occurrence.event_id.as_str()) + .collect(); + assert_eq!(ids, ["evt_ef_fail1"]); + assert!(occurrences.unparsed_selectors.is_empty()); + + // Coverage counts every in-span tool.failed event (3) and how many carry an + // index row (2): the unindexed failure lowers coverage without vanishing. + assert_eq!(occurrences.total_failures, 3); + assert_eq!(occurrences.classifiable_failures, 2); +} + +#[test] +fn efficacy_occurrences_bound_by_span_and_flag_unparsed_selectors() { + let store = seed_efficacy_store(); + // A tight span excludes the March failure entirely. + let (start, end) = span("2026-06-01T00:00:00Z", "2026-06-30T00:00:00Z"); + let occurrences = store + .efficacy_occurrences( + &[EFF_SELECTOR.to_owned(), "not-a-signature".to_owned()], + start, + end, + ) + .expect("occurrences"); + assert!(occurrences.occurrences.is_empty()); + assert_eq!(occurrences.total_failures, 0); + assert_eq!(occurrences.unparsed_selectors, ["not-a-signature"]); +} + +#[test] +fn register_efficacy_is_append_only_immutable_and_lifecycle_free() { + let mut store = EventStore::open_in_memory().expect("store"); + seed_installed_mutation(&mut store); + let state_before = store + .get_mutation("mut_registry") + .expect("details") + .mutation + .state; + + let registration = + efficacy_registration("eff_one", "improved", &["evt_mutation-support-a"], &[]); + assert_eq!( + store.register_efficacy(®istration).expect("insert"), + EfficacyRegisterOutcome::Inserted { + efficacy_id: "eff_one".to_owned(), + } + ); + + // Re-registering identical content is a no-op. + assert_eq!( + store.register_efficacy(®istration).expect("duplicate"), + EfficacyRegisterOutcome::Duplicate { + efficacy_id: "eff_one".to_owned(), + } + ); + + // Reusing the id for different content conflicts. + let conflicting = + efficacy_registration("eff_one", "regressed", &["evt_mutation-support-b"], &[]); + assert!(matches!( + store.register_efficacy(&conflicting), + Err(StoreError::EfficacyContentConflict { .. }) + )); + + // Efficacy never touches the lifecycle: the mutation stays exactly as it was. + let state_after = store + .get_mutation("mut_registry") + .expect("details") + .mutation + .state; + assert_eq!(state_before, "active"); + assert_eq!(state_after, "active"); + + // A second, distinct report accumulates — history, not replacement. + let second = efficacy_registration("eff_two", "unchanged", &["evt_mutation-support-b"], &[]); + store.register_efficacy(&second).expect("second"); + let listed = store.list_efficacy("mut_registry").expect("list"); + assert_eq!(listed.len(), 2); + assert_eq!(listed[0].efficacy_id, "eff_one"); + assert_eq!(listed[1].efficacy_id, "eff_two"); +} + +#[test] +fn register_efficacy_rejects_reports_inconsistent_with_the_registration() { + let mut store = EventStore::open_in_memory().expect("store"); + seed_installed_mutation(&mut store); + let mut registration = + efficacy_registration("eff_bad", "improved", &["evt_mutation-support-a"], &[]); + // The declared count no longer matches the source events. + registration + .source_event_ids + .push("evt_mutation-support-b".to_owned()); + assert!(matches!( + store.register_efficacy(®istration), + Err(StoreError::InvalidEfficacyRegistration) + )); +} + +#[test] +fn efficacy_evidence_removal_drops_the_report_but_keeps_the_mutation() { + let mut store = EventStore::open_in_memory().expect("store"); + // Register a candidate mutation with its own evidence, then a separate + // efficacy-only event the report will cite. + for (event_id, session_id) in [ + ("evt_mutation-support-a", "ses_ev_a"), + ("evt_mutation-support-b", "ses_ev_b"), + ("evt_mutation-counter", "ses_ev_c"), + ("evt_eff_only", "ses_eff_only"), + ] { + store + .insert_event( + &source("instance-efficacy"), + &session_event( + event_id, + session_id, + EventKind::DecisionRecorded, + "2026-02-01T00:00:00Z", + 0, + ), + &SearchProjection::default(), + ) + .expect("evidence event"); + } + store + .register_mutation(&mutation_registration( + "mut_registry", + "fnd_registry", + "eqv_registry", + )) + .expect("register"); + store + .register_efficacy(&efficacy_registration( + "eff_cascade", + "improved", + &["evt_eff_only"], + &[], + )) + .expect("insert"); + assert_eq!(store.list_efficacy("mut_registry").expect("list").len(), 1); + + // Deleting the cited event's session cascades to the efficacy evidence row, + // whose trigger drops the report — but the mutation candidate, whose own + // evidence is untouched, survives (efficacy gates no lifecycle state). + store + .delete_session("ses_eff_only") + .expect("delete session"); + assert!( + store + .list_efficacy("mut_registry") + .expect("list") + .is_empty() + ); + assert_eq!( + store + .get_mutation("mut_registry") + .expect("details") + .mutation + .state, + "candidate" + ); +} + +#[test] +fn efficacy_status_summary_counts_latest_verdict_per_installed_mutation() { + let mut store = EventStore::open_in_memory().expect("store"); + seed_installed_mutation(&mut store); + + // Before any report, the single installed mutation is not measured. + let summary = store.efficacy_status_summary().expect("summary"); + assert_eq!(summary.installed, 1); + assert_eq!(summary.not_measured, 1); + assert_eq!(summary.improved, 0); + + // The latest report wins: register unchanged, then improved. + store + .register_efficacy(&efficacy_registration( + "eff_1_unchanged", + "unchanged", + &["evt_mutation-support-a"], + &[], + )) + .expect("older"); + store + .register_efficacy(&efficacy_registration( + "eff_2_improved", + "improved", + &["evt_mutation-support-b"], + &[], + )) + .expect("newer"); + let summary = store.efficacy_status_summary().expect("summary"); + assert_eq!(summary.installed, 1); + assert_eq!(summary.improved, 1); + assert_eq!(summary.unchanged, 0); + assert_eq!(summary.not_measured, 0); +} diff --git a/docs/decisions/0015-mutation-efficacy-tracking.md b/docs/decisions/0015-mutation-efficacy-tracking.md new file mode 100644 index 0000000..06c442a --- /dev/null +++ b/docs/decisions/0015-mutation-efficacy-tracking.md @@ -0,0 +1,135 @@ +# ADR 0015: Track post-install mutation efficacy with recurrence windows + +- Status: accepted +- Date: 2026-07-19 + +## Context + +Autophagy carries a mutation from a detected finding through challenge, replay, +shadow, and finally installation as a repo-scoped skill. Every gate up to +installation is a *prospective* judgement: replay reasons about annotated +counterfactuals, shadow measures would-be trigger precision. None of them answers +the only question that matters once a mutation is live: **does the failure +signature it addresses actually recur less after it was installed?** + +That question is answerable locally, deterministically, and without a model. The +store already holds everything needed: `mutation_installations.installed_at` (an +RFC 3339, NOT NULL install timestamp), the exact normalized-signature index +(`event_signatures`), and `events.occurred_at` / `event_type` / `exit_code`. A +mutation's immutable trigger selector names the exact failure signature to count. + +Two engineering constraints bound the design. The default path stays local and +offline — this is a COUNT-style query plus pure arithmetic, no model. And every +derived finding must retain exact evidence identifiers — the report cites the +`event_id`s it counted. Adding a stored table takes an ordered, immutable +migration (0003) and this record. + +## Decision + +Add an **observational, non-causal** efficacy measurement. It compares how often +the addressed failure signature recurred in two equal-length windows anchored on +`installed_at`, and never touches the mutation lifecycle. + +**Windows.** The post-window is `[installed_at, evaluated_at]`; the pre-window is +the equal-length window immediately before install, +`[installed_at − post_duration, installed_at)`. Equal lengths make the two rates +directly comparable. The evaluation clock is passed in and echoed in the report, +so a report is reproducible from `(database-state, installed_at, evaluated_at)`. + +**Matching rule (`failure_signature_recurrence`).** Resolved empirically against +the real development database (see below). The mutation's trigger selector is a +`failure/|||exit:` signature, but the exact-signature +index stores only the *operation* projection `operation/||` — +the failure grammar never appears in `event_signatures`. So a failure selector +cannot be matched against the index directly. Instead the store decomposes the +selector into its operation signature and exit code, and counts the events that +are indexed under that operation signature **and** are `tool.failed` **and** carry +the matching exit code. (Command text can contain `|`, so the failure form is +parsed by trimming the trailing `|exit:`, never by splitting on `|`.) An +`operation/|…` selector matches any indexed `tool.failed` for that operation. + +**Coverage diagnostics.** The exact-signature index is partial by construction +(old imports predate indexing; unprojectable commands carry no row). The report +therefore always states `total_failures` (every in-span `tool.failed`) versus +`classifiable_failures` (those with an index row), and never silently +undercounts: coverage below 50% forces an `insufficient_data` verdict with reason +`partial_index_coverage`. + +**Verdict thresholds** (deterministic and inspectable, documented in +[`docs/specs/efficacy/0.1`](../specs/efficacy/0.1/README.md)): `insufficient_data` +when the post-window is under 7 days, when fewer than 2 total occurrences are +observed, or when coverage is below 50% — each triggered reason is listed. +Otherwise, from the change in weekly rate: a reduction of ≥20% is `improved`, an +increase of ≥20% is `regressed`, the ±20% band is `unchanged`; a brand-new +recurrence with no pre-window baseline is `regressed`. + +**No lifecycle coupling.** Replay and shadow advance the mutation state on pass; +efficacy never does. `register_efficacy` writes an append-only report and +performs no state transition. History accumulates — each evaluation sees more +post-install time, so many reports per mutation are expected and the table does +not constrain to one row per mutation. This is also why the evidence-removal +trigger deletes only the efficacy *report* when a cited event is removed, rather +than deleting the mutation candidate the way the shadow and replay evidence +triggers do: losing an efficacy snapshot must never retire an otherwise-valid, +installed mutation. + +**Crate boundary.** A new `autophagy-efficacy` crate holds the pure, deterministic +evaluation (mirroring `autophagy-shadow`); the store gathers occurrences and +coverage; the CLI wires them (`autophagy mutations efficacy `). This keeps +the evaluator a pure function of its inputs and sits it at the replay/shadow tier +of the dependency graph. The report is versioned `efficacy/0.1` with a +content-derived `efficacy_id` (`eff_`), so re-evaluating at the same clock +is idempotent and a new clock yields a new report — exactly the history semantics +above. + +## Real-data verification + +Resolved and verified against a throwaway copy of the author's real database +(69,400 events, 34,911 indexed signatures) — never the live database or config. + +- **Selector-form gotcha, confirmed empirically.** Both registered candidates + carry `failure/v1|shell|…|exit:1` selectors, while `event_signatures` holds + `operation/v1|…` rows exclusively (34,911 of 34,911). For the `go build` + candidate the operation signature resolves to 56 indexed rows: 28 `tool.called`, + 25 `tool.completed` (exit 0), and **3 `tool.failed` (exit 1)** — precisely the + three failures the finding cites. The decompose-and-rejoin rule counts those + three and nothing else. +- **End-to-end drive.** One candidate was driven on the copy through challenge → + replay (12 scenarios, passed) → shadow (12 observations, precision 100%, + passed) → install into a scratch git repo, producing a real + `installed_at = 2026-07-18T20:54:57Z`. +- **Honest empty window.** Evaluated at the real install instant, the post-window + is 0 days, so the verdict is `insufficient_data` + (`post_window_too_short`, `sparse_occurrences`) — the correct answer when + observation has not yet begun. +- **Verdict math on real evidence.** Evaluated at a later clock (the same real + `installed_at`, `--now 2026-11-10`), the three historical `go build` failures + fall in the pre-window and none recur after install: + + ``` + improved · 3 → 0 failures (0.2/wk → 0.0/wk) · -100.0% · 80.4% classifiable + ``` + + The report cites the exact three failure `event_id`s, and the 80.4% coverage + figure (304 of 425 in-span `tool.failed` events indexed) demonstrates the + partial-index diagnostic surfacing rather than hiding. + +## Privacy + +Strictly subtractive on derived data. The report cites exact `event_id`s and +counts only — never command text, never event content. Occurrences are matched +through the same redaction-approved signature index that already gates search, so +efficacy can only ever see what redaction already approved. No new source text is +persisted and nothing leaves the machine. + +## Consequences + +- A fourth deterministic, model-free measurement joins detection, replay, and + shadow, closing the loop from "installed" to "did it help". +- `status` gains a one-line installed-mutation efficacy summary (only when + installations exist); `mutations show` gains a latest-efficacy line. +- Efficacy is only as complete as the signature index; the coverage diagnostic + makes that limit explicit rather than silent, and a low-coverage database is + told to `reindex` through the existing status hint (ADR 0014). +- Migrations advance to schema v3; the table and its evidence cascade are + immutable from here. diff --git a/docs/specs/efficacy/0.1/README.md b/docs/specs/efficacy/0.1/README.md new file mode 100644 index 0000000..8e09701 --- /dev/null +++ b/docs/specs/efficacy/0.1/README.md @@ -0,0 +1,90 @@ +# Efficacy v0.1 + +Efficacy v0.1 measures, deterministically and observationally, whether the +failure signature an *installed* mutation addresses recurs less after it was +installed. It makes **no causal claim**, invokes **no model**, and never changes +a mutation's lifecycle state. The normative contract is +[`result.schema.json`](result.schema.json); `valid/` and `invalid/` hold +fixtures the evaluator is tested against. + +Every report sets `model_used: false` and cites exact local AEP event IDs — never +event content. + +## Windows + +Efficacy compares two equal-length windows anchored on `installed_at`: + +- **post-window** `[installed_at, evaluated_at]` — the observation period since + install. +- **pre-window** `[installed_at − post_duration, installed_at)` — the equal-length + window immediately before install. + +Both windows report raw `occurrences`, `distinct_sessions`, and a weekly rate. +The rate is `rate_per_week_milli` — occurrences per week scaled by 1000, so the +report carries no floats (`1810` means `1.81` failures/week). The evaluation +clock (`evaluated_at`) is passed in and echoed, so a report is fully reproducible +from `(database-state, installed_at, evaluated_at)`. + +## Matching rule + +`matching_rule: failure_signature_recurrence` is the only rule in v0.1. A +mutation's immutable trigger selector is a versioned failure signature; the store +decomposes it and counts the matching failure events: + +- A `failure/|||exit:` selector is split into the + outcome-independent operation signature `operation/||` and an + exit code. It matches an event when the event is a `tool.failed`, its exit code + equals ``, and it is indexed under that operation signature. (Command + text may contain `|`, so the failure form is parsed by trimming the trailing + `|exit:`, never by splitting on `|`.) +- An `operation/||` selector matches any `tool.failed` event + indexed under that operation signature, regardless of exit code. + +The exact-signature index is the same redaction-approved projection that gates +free-text search, so a failure event is only counted when its operation +signature was indexed. This is why coverage is reported explicitly (below) rather +than assumed complete. + +## Coverage diagnostics + +The exact-signature index is partial by construction — old imports may predate +indexing, and events whose command cannot be projected carry no signature row. +`coverage` reports, over the full evaluated span: + +- `total_failures` — every `tool.failed` event in the span. +- `classifiable_failures` — how many of those carry an index row (and can + therefore be matched by signature at all). +- `coverage_bps` — `classifiable / total`, in basis points. +- `complete` — whether every in-span failure was classifiable. + +Coverage never silently undercounts: when it is too low to trust, the verdict is +`insufficient_data` with reason `partial_index_coverage`. + +## Verdict + +`verdict` is one of `improved`, `regressed`, `unchanged`, `insufficient_data`, +computed by these exact, inspectable thresholds: + +1. `insufficient_data` when **any** of these hold (all triggered reasons are + listed in `insufficient_reasons`): + - `post_window_too_short` — the post-window is shorter than **7 days**. + - `sparse_occurrences` — fewer than **2** total occurrences across both + windows. + - `partial_index_coverage` — `coverage_bps` below **5000** (50%). +2. Otherwise, from the change in weekly rate: + - With no pre-window baseline (pre rate 0), a nonzero post rate is + `regressed` (a new recurrence appeared); `rate_delta_bps` is omitted. + - With a nonzero baseline, `rate_delta_bps` is the signed relative change + (`-3300` = a 33% reduction). A change of **−2000 bps or more** (≥20% + reduction) is `improved`; **+2000 bps or more** is `regressed`; anything + inside the ±20% band is `unchanged`. + +Because the windows are equal length, the rate comparison reduces to comparing +raw counts, but rates are reported so the numbers stay legible. + +## Evidence + +`evidence` lists the exact `event_id`s counted in each window, in canonical +order. Counts (`pre_event_count`, `post_event_count`) are always exact; the +listed identifiers are capped at `listing_cap` (50) so a pathological corpus +cannot produce an unbounded report. diff --git a/docs/specs/efficacy/0.1/invalid/bad_efficacy_id.json b/docs/specs/efficacy/0.1/invalid/bad_efficacy_id.json new file mode 100644 index 0000000..9a37dcc --- /dev/null +++ b/docs/specs/efficacy/0.1/invalid/bad_efficacy_id.json @@ -0,0 +1,56 @@ +{ + "spec_version": "efficacy/0.1", + "efficacy_id": "shr_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "mutation_id": "mut_a87b7c83aa9bf78169c01c2ad73b02c1", + "mutation_version": "0.1.0", + "signature_selectors": [ + "failure/v2|shell|cd $PROJECT && go build ./... 2>&1|exit:1" + ], + "matching_rule": "failure_signature_recurrence", + "installed_at": "2026-04-15T00:00:00Z", + "evaluated_at": "2026-07-15T00:00:00Z", + "windows": { + "pre": { + "start": "2026-01-14T00:00:00Z", + "end": "2026-04-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 6, + "distinct_sessions": 3, + "rate_per_week_milli": 461 + }, + "post": { + "start": "2026-04-15T00:00:00Z", + "end": "2026-07-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 1, + "distinct_sessions": 1, + "rate_per_week_milli": 76 + } + }, + "rate_delta_bps": -8351, + "coverage": { + "classifiable_failures": 118, + "total_failures": 124, + "coverage_bps": 9516, + "complete": false + }, + "verdict": "improved", + "insufficient_reasons": [], + "evidence": { + "pre_event_ids": [ + "evt_pi_4a98a0e1", + "evt_pi_b3d59052", + "evt_pi_5ffbc3d1", + "evt_pi_1cab776a", + "evt_pi_2a7bcdcb", + "evt_pi_0a991c55" + ], + "post_event_ids": [ + "evt_pi_ed88be62" + ], + "pre_event_count": 6, + "post_event_count": 1, + "listing_cap": 50 + }, + "model_used": false +} diff --git a/docs/specs/efficacy/0.1/invalid/bad_insufficient_reason.json b/docs/specs/efficacy/0.1/invalid/bad_insufficient_reason.json new file mode 100644 index 0000000..d968575 --- /dev/null +++ b/docs/specs/efficacy/0.1/invalid/bad_insufficient_reason.json @@ -0,0 +1,58 @@ +{ + "spec_version": "efficacy/0.1", + "efficacy_id": "eff_e88853216dc0a71e56da0adc54560d5531fcd9aa04734773fd13de947615ed58", + "mutation_id": "mut_a87b7c83aa9bf78169c01c2ad73b02c1", + "mutation_version": "0.1.0", + "signature_selectors": [ + "failure/v2|shell|cd $PROJECT && go build ./... 2>&1|exit:1" + ], + "matching_rule": "failure_signature_recurrence", + "installed_at": "2026-04-15T00:00:00Z", + "evaluated_at": "2026-07-15T00:00:00Z", + "windows": { + "pre": { + "start": "2026-01-14T00:00:00Z", + "end": "2026-04-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 6, + "distinct_sessions": 3, + "rate_per_week_milli": 461 + }, + "post": { + "start": "2026-04-15T00:00:00Z", + "end": "2026-07-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 1, + "distinct_sessions": 1, + "rate_per_week_milli": 76 + } + }, + "rate_delta_bps": -8351, + "coverage": { + "classifiable_failures": 118, + "total_failures": 124, + "coverage_bps": 9516, + "complete": false + }, + "verdict": "improved", + "insufficient_reasons": [ + "not_a_reason" + ], + "evidence": { + "pre_event_ids": [ + "evt_pi_4a98a0e1", + "evt_pi_b3d59052", + "evt_pi_5ffbc3d1", + "evt_pi_1cab776a", + "evt_pi_2a7bcdcb", + "evt_pi_0a991c55" + ], + "post_event_ids": [ + "evt_pi_ed88be62" + ], + "pre_event_count": 6, + "post_event_count": 1, + "listing_cap": 50 + }, + "model_used": false +} diff --git a/docs/specs/efficacy/0.1/invalid/bad_matching_rule.json b/docs/specs/efficacy/0.1/invalid/bad_matching_rule.json new file mode 100644 index 0000000..147db23 --- /dev/null +++ b/docs/specs/efficacy/0.1/invalid/bad_matching_rule.json @@ -0,0 +1,56 @@ +{ + "spec_version": "efficacy/0.1", + "efficacy_id": "eff_e88853216dc0a71e56da0adc54560d5531fcd9aa04734773fd13de947615ed58", + "mutation_id": "mut_a87b7c83aa9bf78169c01c2ad73b02c1", + "mutation_version": "0.1.0", + "signature_selectors": [ + "failure/v2|shell|cd $PROJECT && go build ./... 2>&1|exit:1" + ], + "matching_rule": "model_judgment", + "installed_at": "2026-04-15T00:00:00Z", + "evaluated_at": "2026-07-15T00:00:00Z", + "windows": { + "pre": { + "start": "2026-01-14T00:00:00Z", + "end": "2026-04-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 6, + "distinct_sessions": 3, + "rate_per_week_milli": 461 + }, + "post": { + "start": "2026-04-15T00:00:00Z", + "end": "2026-07-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 1, + "distinct_sessions": 1, + "rate_per_week_milli": 76 + } + }, + "rate_delta_bps": -8351, + "coverage": { + "classifiable_failures": 118, + "total_failures": 124, + "coverage_bps": 9516, + "complete": false + }, + "verdict": "improved", + "insufficient_reasons": [], + "evidence": { + "pre_event_ids": [ + "evt_pi_4a98a0e1", + "evt_pi_b3d59052", + "evt_pi_5ffbc3d1", + "evt_pi_1cab776a", + "evt_pi_2a7bcdcb", + "evt_pi_0a991c55" + ], + "post_event_ids": [ + "evt_pi_ed88be62" + ], + "pre_event_count": 6, + "post_event_count": 1, + "listing_cap": 50 + }, + "model_used": false +} diff --git a/docs/specs/efficacy/0.1/invalid/bad_spec_version.json b/docs/specs/efficacy/0.1/invalid/bad_spec_version.json new file mode 100644 index 0000000..4432d36 --- /dev/null +++ b/docs/specs/efficacy/0.1/invalid/bad_spec_version.json @@ -0,0 +1,56 @@ +{ + "spec_version": "efficacy/0.2", + "efficacy_id": "eff_e88853216dc0a71e56da0adc54560d5531fcd9aa04734773fd13de947615ed58", + "mutation_id": "mut_a87b7c83aa9bf78169c01c2ad73b02c1", + "mutation_version": "0.1.0", + "signature_selectors": [ + "failure/v2|shell|cd $PROJECT && go build ./... 2>&1|exit:1" + ], + "matching_rule": "failure_signature_recurrence", + "installed_at": "2026-04-15T00:00:00Z", + "evaluated_at": "2026-07-15T00:00:00Z", + "windows": { + "pre": { + "start": "2026-01-14T00:00:00Z", + "end": "2026-04-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 6, + "distinct_sessions": 3, + "rate_per_week_milli": 461 + }, + "post": { + "start": "2026-04-15T00:00:00Z", + "end": "2026-07-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 1, + "distinct_sessions": 1, + "rate_per_week_milli": 76 + } + }, + "rate_delta_bps": -8351, + "coverage": { + "classifiable_failures": 118, + "total_failures": 124, + "coverage_bps": 9516, + "complete": false + }, + "verdict": "improved", + "insufficient_reasons": [], + "evidence": { + "pre_event_ids": [ + "evt_pi_4a98a0e1", + "evt_pi_b3d59052", + "evt_pi_5ffbc3d1", + "evt_pi_1cab776a", + "evt_pi_2a7bcdcb", + "evt_pi_0a991c55" + ], + "post_event_ids": [ + "evt_pi_ed88be62" + ], + "pre_event_count": 6, + "post_event_count": 1, + "listing_cap": 50 + }, + "model_used": false +} diff --git a/docs/specs/efficacy/0.1/invalid/model_used_true.json b/docs/specs/efficacy/0.1/invalid/model_used_true.json new file mode 100644 index 0000000..a43778d --- /dev/null +++ b/docs/specs/efficacy/0.1/invalid/model_used_true.json @@ -0,0 +1,56 @@ +{ + "spec_version": "efficacy/0.1", + "efficacy_id": "eff_e88853216dc0a71e56da0adc54560d5531fcd9aa04734773fd13de947615ed58", + "mutation_id": "mut_a87b7c83aa9bf78169c01c2ad73b02c1", + "mutation_version": "0.1.0", + "signature_selectors": [ + "failure/v2|shell|cd $PROJECT && go build ./... 2>&1|exit:1" + ], + "matching_rule": "failure_signature_recurrence", + "installed_at": "2026-04-15T00:00:00Z", + "evaluated_at": "2026-07-15T00:00:00Z", + "windows": { + "pre": { + "start": "2026-01-14T00:00:00Z", + "end": "2026-04-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 6, + "distinct_sessions": 3, + "rate_per_week_milli": 461 + }, + "post": { + "start": "2026-04-15T00:00:00Z", + "end": "2026-07-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 1, + "distinct_sessions": 1, + "rate_per_week_milli": 76 + } + }, + "rate_delta_bps": -8351, + "coverage": { + "classifiable_failures": 118, + "total_failures": 124, + "coverage_bps": 9516, + "complete": false + }, + "verdict": "improved", + "insufficient_reasons": [], + "evidence": { + "pre_event_ids": [ + "evt_pi_4a98a0e1", + "evt_pi_b3d59052", + "evt_pi_5ffbc3d1", + "evt_pi_1cab776a", + "evt_pi_2a7bcdcb", + "evt_pi_0a991c55" + ], + "post_event_ids": [ + "evt_pi_ed88be62" + ], + "pre_event_count": 6, + "post_event_count": 1, + "listing_cap": 50 + }, + "model_used": true +} diff --git a/docs/specs/efficacy/0.1/invalid/unknown_field.json b/docs/specs/efficacy/0.1/invalid/unknown_field.json new file mode 100644 index 0000000..bbc64f2 --- /dev/null +++ b/docs/specs/efficacy/0.1/invalid/unknown_field.json @@ -0,0 +1,57 @@ +{ + "spec_version": "efficacy/0.1", + "efficacy_id": "eff_e88853216dc0a71e56da0adc54560d5531fcd9aa04734773fd13de947615ed58", + "mutation_id": "mut_a87b7c83aa9bf78169c01c2ad73b02c1", + "mutation_version": "0.1.0", + "signature_selectors": [ + "failure/v2|shell|cd $PROJECT && go build ./... 2>&1|exit:1" + ], + "matching_rule": "failure_signature_recurrence", + "installed_at": "2026-04-15T00:00:00Z", + "evaluated_at": "2026-07-15T00:00:00Z", + "windows": { + "pre": { + "start": "2026-01-14T00:00:00Z", + "end": "2026-04-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 6, + "distinct_sessions": 3, + "rate_per_week_milli": 461 + }, + "post": { + "start": "2026-04-15T00:00:00Z", + "end": "2026-07-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 1, + "distinct_sessions": 1, + "rate_per_week_milli": 76 + } + }, + "rate_delta_bps": -8351, + "coverage": { + "classifiable_failures": 118, + "total_failures": 124, + "coverage_bps": 9516, + "complete": false + }, + "verdict": "improved", + "insufficient_reasons": [], + "evidence": { + "pre_event_ids": [ + "evt_pi_4a98a0e1", + "evt_pi_b3d59052", + "evt_pi_5ffbc3d1", + "evt_pi_1cab776a", + "evt_pi_2a7bcdcb", + "evt_pi_0a991c55" + ], + "post_event_ids": [ + "evt_pi_ed88be62" + ], + "pre_event_count": 6, + "post_event_count": 1, + "listing_cap": 50 + }, + "model_used": false, + "surprise": "nope" +} diff --git a/docs/specs/efficacy/0.1/invalid/unknown_verdict.json b/docs/specs/efficacy/0.1/invalid/unknown_verdict.json new file mode 100644 index 0000000..b67c133 --- /dev/null +++ b/docs/specs/efficacy/0.1/invalid/unknown_verdict.json @@ -0,0 +1,56 @@ +{ + "spec_version": "efficacy/0.1", + "efficacy_id": "eff_e88853216dc0a71e56da0adc54560d5531fcd9aa04734773fd13de947615ed58", + "mutation_id": "mut_a87b7c83aa9bf78169c01c2ad73b02c1", + "mutation_version": "0.1.0", + "signature_selectors": [ + "failure/v2|shell|cd $PROJECT && go build ./... 2>&1|exit:1" + ], + "matching_rule": "failure_signature_recurrence", + "installed_at": "2026-04-15T00:00:00Z", + "evaluated_at": "2026-07-15T00:00:00Z", + "windows": { + "pre": { + "start": "2026-01-14T00:00:00Z", + "end": "2026-04-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 6, + "distinct_sessions": 3, + "rate_per_week_milli": 461 + }, + "post": { + "start": "2026-04-15T00:00:00Z", + "end": "2026-07-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 1, + "distinct_sessions": 1, + "rate_per_week_milli": 76 + } + }, + "rate_delta_bps": -8351, + "coverage": { + "classifiable_failures": 118, + "total_failures": 124, + "coverage_bps": 9516, + "complete": false + }, + "verdict": "worse", + "insufficient_reasons": [], + "evidence": { + "pre_event_ids": [ + "evt_pi_4a98a0e1", + "evt_pi_b3d59052", + "evt_pi_5ffbc3d1", + "evt_pi_1cab776a", + "evt_pi_2a7bcdcb", + "evt_pi_0a991c55" + ], + "post_event_ids": [ + "evt_pi_ed88be62" + ], + "pre_event_count": 6, + "post_event_count": 1, + "listing_cap": 50 + }, + "model_used": false +} diff --git a/docs/specs/efficacy/0.1/result.schema.json b/docs/specs/efficacy/0.1/result.schema.json new file mode 100644 index 0000000..4d7d709 --- /dev/null +++ b/docs/specs/efficacy/0.1/result.schema.json @@ -0,0 +1,122 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://autophagy.sh/specs/efficacy/0.1/result.schema.json", + "title": "Autophagy Efficacy Result v0.1", + "type": "object", + "additionalProperties": false, + "required": [ + "spec_version", + "efficacy_id", + "mutation_id", + "mutation_version", + "signature_selectors", + "matching_rule", + "installed_at", + "evaluated_at", + "windows", + "coverage", + "verdict", + "insufficient_reasons", + "evidence", + "model_used" + ], + "properties": { + "spec_version": { "const": "efficacy/0.1" }, + "efficacy_id": { "type": "string", "pattern": "^eff_[a-f0-9]{64}$" }, + "mutation_id": { "type": "string", "pattern": "^mut_[A-Za-z0-9._:-]+$" }, + "mutation_version": { "type": "string", "minLength": 1 }, + "signature_selectors": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 1 } + }, + "matching_rule": { "enum": ["failure_signature_recurrence"] }, + "installed_at": { "type": "string", "format": "date-time" }, + "evaluated_at": { "type": "string", "format": "date-time" }, + "windows": { + "type": "object", + "additionalProperties": false, + "required": ["pre", "post"], + "properties": { + "pre": { "$ref": "#/$defs/window" }, + "post": { "$ref": "#/$defs/window" } + } + }, + "rate_delta_bps": { "type": "integer" }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": ["classifiable_failures", "total_failures", "coverage_bps", "complete"], + "properties": { + "classifiable_failures": { "type": "integer", "minimum": 0 }, + "total_failures": { "type": "integer", "minimum": 0 }, + "coverage_bps": { "type": "integer", "minimum": 0, "maximum": 10000 }, + "complete": { "type": "boolean" } + } + }, + "verdict": { + "enum": ["improved", "regressed", "unchanged", "insufficient_data"] + }, + "insufficient_reasons": { + "type": "array", + "uniqueItems": true, + "items": { + "enum": [ + "post_window_too_short", + "sparse_occurrences", + "partial_index_coverage" + ] + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "pre_event_ids", + "post_event_ids", + "pre_event_count", + "post_event_count", + "listing_cap" + ], + "properties": { + "pre_event_ids": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string", "pattern": "^evt_[A-Za-z0-9._:-]+$" } + }, + "post_event_ids": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string", "pattern": "^evt_[A-Za-z0-9._:-]+$" } + }, + "pre_event_count": { "type": "integer", "minimum": 0 }, + "post_event_count": { "type": "integer", "minimum": 0 }, + "listing_cap": { "type": "integer", "minimum": 1 } + } + }, + "model_used": { "const": false } + }, + "$defs": { + "window": { + "type": "object", + "additionalProperties": false, + "required": [ + "start", + "end", + "duration_seconds", + "occurrences", + "distinct_sessions", + "rate_per_week_milli" + ], + "properties": { + "start": { "type": "string", "format": "date-time" }, + "end": { "type": "string", "format": "date-time" }, + "duration_seconds": { "type": "integer", "minimum": 0 }, + "occurrences": { "type": "integer", "minimum": 0 }, + "distinct_sessions": { "type": "integer", "minimum": 0 }, + "rate_per_week_milli": { "type": "integer", "minimum": 0 } + } + } + } +} diff --git a/docs/specs/efficacy/0.1/valid/improved.json b/docs/specs/efficacy/0.1/valid/improved.json new file mode 100644 index 0000000..9b529ad --- /dev/null +++ b/docs/specs/efficacy/0.1/valid/improved.json @@ -0,0 +1,56 @@ +{ + "spec_version": "efficacy/0.1", + "efficacy_id": "eff_e88853216dc0a71e56da0adc54560d5531fcd9aa04734773fd13de947615ed58", + "mutation_id": "mut_a87b7c83aa9bf78169c01c2ad73b02c1", + "mutation_version": "0.1.0", + "signature_selectors": [ + "failure/v2|shell|cd $PROJECT && go build ./... 2>&1|exit:1" + ], + "matching_rule": "failure_signature_recurrence", + "installed_at": "2026-04-15T00:00:00Z", + "evaluated_at": "2026-07-15T00:00:00Z", + "windows": { + "pre": { + "start": "2026-01-14T00:00:00Z", + "end": "2026-04-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 6, + "distinct_sessions": 3, + "rate_per_week_milli": 461 + }, + "post": { + "start": "2026-04-15T00:00:00Z", + "end": "2026-07-15T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 1, + "distinct_sessions": 1, + "rate_per_week_milli": 76 + } + }, + "rate_delta_bps": -8351, + "coverage": { + "classifiable_failures": 118, + "total_failures": 124, + "coverage_bps": 9516, + "complete": false + }, + "verdict": "improved", + "insufficient_reasons": [], + "evidence": { + "pre_event_ids": [ + "evt_pi_4a98a0e1", + "evt_pi_b3d59052", + "evt_pi_5ffbc3d1", + "evt_pi_1cab776a", + "evt_pi_2a7bcdcb", + "evt_pi_0a991c55" + ], + "post_event_ids": [ + "evt_pi_ed88be62" + ], + "pre_event_count": 6, + "post_event_count": 1, + "listing_cap": 50 + }, + "model_used": false +} diff --git a/docs/specs/efficacy/0.1/valid/insufficient_data.json b/docs/specs/efficacy/0.1/valid/insufficient_data.json new file mode 100644 index 0000000..91f4f20 --- /dev/null +++ b/docs/specs/efficacy/0.1/valid/insufficient_data.json @@ -0,0 +1,49 @@ +{ + "spec_version": "efficacy/0.1", + "efficacy_id": "eff_8cd0afb35166df2ffa12898b16bccbdd5d5b730f71fd02bd8d23c19acce4b84c", + "mutation_id": "mut_b13f28789cdae90dd6837572975bca5b", + "mutation_version": "0.1.0", + "signature_selectors": [ + "failure/v2|shell|cd zuzoto && go build ./... 2>&1|exit:1" + ], + "matching_rule": "failure_signature_recurrence", + "installed_at": "2026-07-14T00:00:00Z", + "evaluated_at": "2026-07-15T00:00:00Z", + "windows": { + "pre": { + "start": "2026-07-13T00:00:00Z", + "end": "2026-07-14T00:00:00Z", + "duration_seconds": 86400, + "occurrences": 0, + "distinct_sessions": 0, + "rate_per_week_milli": 0 + }, + "post": { + "start": "2026-07-14T00:00:00Z", + "end": "2026-07-15T00:00:00Z", + "duration_seconds": 86400, + "occurrences": 0, + "distinct_sessions": 0, + "rate_per_week_milli": 0 + } + }, + "coverage": { + "classifiable_failures": 0, + "total_failures": 0, + "coverage_bps": 10000, + "complete": true + }, + "verdict": "insufficient_data", + "insufficient_reasons": [ + "post_window_too_short", + "sparse_occurrences" + ], + "evidence": { + "pre_event_ids": [], + "post_event_ids": [], + "pre_event_count": 0, + "post_event_count": 0, + "listing_cap": 50 + }, + "model_used": false +} diff --git a/docs/specs/efficacy/0.1/valid/regressed_no_baseline.json b/docs/specs/efficacy/0.1/valid/regressed_no_baseline.json new file mode 100644 index 0000000..25c5a13 --- /dev/null +++ b/docs/specs/efficacy/0.1/valid/regressed_no_baseline.json @@ -0,0 +1,51 @@ +{ + "spec_version": "efficacy/0.1", + "efficacy_id": "eff_293df1750a2cb9ff4634b4e9121473fd95e4c7ae37ca3f9fe757c1416a532f67", + "mutation_id": "mut_c0ffee00000000000000000000000000", + "mutation_version": "0.2.1", + "signature_selectors": [ + "operation/v2|shell|pytest -q" + ], + "matching_rule": "failure_signature_recurrence", + "installed_at": "2026-02-01T00:00:00Z", + "evaluated_at": "2026-05-02T00:00:00Z", + "windows": { + "pre": { + "start": "2025-11-02T00:00:00Z", + "end": "2026-02-01T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 0, + "distinct_sessions": 0, + "rate_per_week_milli": 0 + }, + "post": { + "start": "2026-02-01T00:00:00Z", + "end": "2026-05-02T00:00:00Z", + "duration_seconds": 7862400, + "occurrences": 4, + "distinct_sessions": 2, + "rate_per_week_milli": 307 + } + }, + "coverage": { + "classifiable_failures": 40, + "total_failures": 40, + "coverage_bps": 10000, + "complete": true + }, + "verdict": "regressed", + "insufficient_reasons": [], + "evidence": { + "pre_event_ids": [], + "post_event_ids": [ + "evt_a1", + "evt_b7", + "evt_c3", + "evt_d9" + ], + "pre_event_count": 0, + "post_event_count": 4, + "listing_cap": 50 + }, + "model_used": false +}