diff --git a/Cargo.lock b/Cargo.lock index 9c4eed3..8ff02f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -209,6 +209,7 @@ name = "autophagy-events" version = "0.1.0" dependencies = [ "jsonschema", + "regex", "serde", "serde_json", "thiserror", diff --git a/crates/autophagy-cli/src/main.rs b/crates/autophagy-cli/src/main.rs index d7e3030..aea590c 100644 --- a/crates/autophagy-cli/src/main.rs +++ b/crates/autophagy-cli/src/main.rs @@ -183,7 +183,7 @@ enum Commands { query: Option, /// Exact normalized operation signature, such as - /// `operation/v1|shell|cargo test`. + /// `operation/v2|shell|cargo test`. #[arg(long, value_name = "SIGNATURE")] signature: Option, diff --git a/crates/autophagy-cli/src/status.rs b/crates/autophagy-cli/src/status.rs index 02452d2..202c0f2 100644 --- a/crates/autophagy-cli/src/status.rs +++ b/crates/autophagy-cli/src/status.rs @@ -62,6 +62,10 @@ pub struct IndexStatus { pub signatures: u64, /// Whether redacted tool input has been indexed for exact recall. pub tool_input_indexed: bool, + /// Indexed signatures minted under a superseded signature grammar. When + /// non-zero, the index predates the current grammar and a + /// `reindex --index-tool-input` re-mints every row (ADR 0014). + pub stale_signatures: u64, } /// One adapter's import activity and freshness. @@ -121,6 +125,15 @@ pub fn run( let size_bytes = std::fs::metadata(&db_path).ok().map(|meta| meta.len()); let stats = store.stats()?; let signatures = store.signature_count()?; + // Every indexed signature is an `operation/|…` key. Rows that do not + // carry the current grammar's prefix were minted under a superseded grammar + // and no longer match freshly minted signatures (ADR 0014); `reindex` heals + // them. + let stale_prefix = format!( + "operation/{}|", + autophagy_events::signature::SIGNATURE_SPEC_VERSION + ); + let stale_signatures = store.signatures_below_version(&stale_prefix)?; let activity = store.adapter_activity()?; let candidates = store.mutation_state_counts()?; let schema_version = store.schema_version()?; @@ -176,6 +189,7 @@ pub fn run( index: IndexStatus { signatures, tool_input_indexed: signatures > 0, + stale_signatures, }, adapters, detector: DetectorStatus { @@ -269,6 +283,16 @@ pub fn write_text(report: &StatusReport, writer: &mut impl Write) -> std::io::Re "commands not indexed — run `autophagy reindex --index-tool-input`".to_owned() }; writeln!(writer, "{}", row("Search", &search))?; + if report.index.stale_signatures > 0 { + writeln!( + writer, + "{}", + cont(&format!( + "{} signatures use an older grammar — run `autophagy reindex --index-tool-input` to re-mint and detect recurring shapes", + group(i64::try_from(report.index.stale_signatures).unwrap_or(i64::MAX)) + )) + )?; + } writeln!(writer)?; if report.adapters.is_empty() { diff --git a/crates/autophagy-core/tests/generic_jsonl.rs b/crates/autophagy-core/tests/generic_jsonl.rs index be9274c..8ba0dce 100644 --- a/crates/autophagy-core/tests/generic_jsonl.rs +++ b/crates/autophagy-core/tests/generic_jsonl.rs @@ -8,7 +8,7 @@ use autophagy_store::{EventStore, RetrievalMatchKind, RetrievalQuery, StoreStats const MIXED: &str = include_str!("fixtures/mixed.jsonl"); const RETRIEVAL: &str = include_str!("../../../evals/fixtures/retrieval/deterministic.jsonl"); -const BUILD_SIG: &str = "operation/v1|shell|cargo build"; +const BUILD_SIG: &str = "operation/v2|shell|cargo build"; fn hit_ids(hits: &[autophagy_store::RetrievalHit]) -> Vec { hits.iter().map(|hit| hit.event_id.clone()).collect() diff --git a/crates/autophagy-events/Cargo.toml b/crates/autophagy-events/Cargo.toml index e5822eb..c90b04b 100644 --- a/crates/autophagy-events/Cargo.toml +++ b/crates/autophagy-events/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true publish = false [dependencies] +regex.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true diff --git a/crates/autophagy-events/src/signature.rs b/crates/autophagy-events/src/signature.rs index fbb9ce3..ea0f5bd 100644 --- a/crates/autophagy-events/src/signature.rs +++ b/crates/autophagy-events/src/signature.rs @@ -1,25 +1,61 @@ //! Deterministic, model-free normalization of tool operations into stable //! signatures. //! -//! A normalized signature collapses incidental variation (tool aliases, -//! whitespace, and the concrete project prefix) so that the same underlying -//! operation produces the same string across sessions. The normalization is a -//! pure function of an [`Event`]; it never consults a model, the filesystem, or -//! the network. Both the deterministic pattern detectors and the retrieval -//! signature index build on this single implementation so their identities stay -//! byte-for-byte consistent. +//! A normalized signature collapses incidental variation so that the same +//! underlying operation produces the same string across sessions. Two kinds of +//! variation are removed: +//! +//! - **Aliases and layout** — tool aliases (`bash`/`exec`/`shell` → `shell`), +//! repeated whitespace, and the concrete project prefix (→ `$PROJECT`). +//! - **Volatile tokens** — absolute and home-relative (`~/…`) paths, URLs, +//! UUIDs, long hex runs, and long digit runs are replaced with stable +//! placeholders (`«path»`, `«url»`, +//! `«uuid»`, `«hex»`, `«n»`). Real agent commands are long one-off compounds +//! full of scratchpad directories, session UUIDs, and timestamps; without this +//! pass two semantically identical failures never share a byte-identical +//! signature and never register as recurring. Command *structure* — binaries, +//! subcommands, flags, and shell operators — is deliberately preserved, so +//! `cargo test -p a` and `cargo test -p b` stay distinct while `cd /x && go +//! build` and `cd /y && go build` collapse to one shape. +//! +//! The normalization is a pure, total function of an [`Event`]; it never +//! consults a model, the filesystem, the network, the locale, or the clock, and +//! normalizing an already-normalized command is a fixed point (idempotent). Both +//! the deterministic pattern detectors and the retrieval signature index build +//! on this single implementation so their identities stay byte-for-byte +//! consistent. +//! +//! # Versioning +//! +//! Every minted selector embeds [`SIGNATURE_SPEC_VERSION`]. The volatile-token +//! normalization was introduced in `v2`; `v1` selectors embedded literal command +//! text. The two grammars are intentionally non-interoperable: a `v1` selector +//! never matches a freshly minted `v2` signature. Already-registered mutations +//! keep their immutable `v1` trigger selectors as valid audit records; a stored +//! or indexed signature is re-minted under `v2` by rebuilding the projection +//! (`autophagy reindex --index-tool-input`). See ADR 0014. + +use std::sync::LazyLock; +use regex::{Captures, Regex}; use serde_json::Value; use crate::Event; +/// Current signature-grammar version token embedded in every minted selector +/// (`operation/|…`, `failure/|…`). +/// +/// Bump this when the normalization in [`normalize_operation`] changes so a new +/// grammar cannot be silently confused with signatures minted under the old one. +pub const SIGNATURE_SPEC_VERSION: &str = "v2"; + /// A normalized tool operation: its canonical tool name and command text. /// /// Construct one with [`normalize_operation`]. The stable string projections /// ([`operation_key`](OperationSignature::operation_key) and /// [`failure_signature`](OperationSignature::failure_signature)) are versioned -/// so a future normalization change can introduce a new prefix without -/// silently reinterpreting stored signatures. +/// so a future normalization change can introduce a new prefix without silently +/// reinterpreting stored signatures. #[derive(Clone, Debug, Eq, PartialEq)] pub struct OperationSignature { tool: String, @@ -33,23 +69,31 @@ impl OperationSignature { &self.tool } - /// Normalized command text with the project prefix replaced by `$PROJECT`. + /// Normalized command text: the project prefix replaced by `$PROJECT` and + /// every volatile token replaced by a stable placeholder. #[must_use] pub fn command(&self) -> &str { &self.command } - /// Outcome-independent operation identity: `operation/v1||`. + /// Outcome-independent operation identity: + /// `operation/||`. #[must_use] pub fn operation_key(&self) -> String { - format!("operation/v1|{}|{}", self.tool, self.command) + format!( + "operation/{SIGNATURE_SPEC_VERSION}|{}|{}", + self.tool, self.command + ) } /// Failure identity including an exit code: - /// `failure/v1|||exit:`. + /// `failure/|||exit:`. #[must_use] pub fn failure_signature(&self, exit_code: i64) -> String { - format!("failure/v1|{}|{}|exit:{exit_code}", self.tool, self.command) + format!( + "failure/{SIGNATURE_SPEC_VERSION}|{}|{}|exit:{exit_code}", + self.tool, self.command + ) } /// Deterministic human-readable label: `: `. @@ -79,6 +123,15 @@ pub fn normalize_operation(event: &Event) -> Option { }) } +/// Normalize a raw command string exactly as [`normalize_operation`] does. +/// +/// Exposed for direct testing and for callers that already hold command text +/// outside an [`Event`]; it applies the identical, deterministic pass. +#[must_use] +pub fn normalize_command_text(command: &str, project: Option<&str>) -> String { + normalize_command(command, project) +} + fn normalize_tool(tool: &str) -> String { match tool.trim().to_ascii_lowercase().as_str() { "bash" | "exec" | "exec_command" | "shell" | "terminal" => "shell".to_owned(), @@ -98,12 +151,82 @@ fn command(input: &Value) -> Option { } } +// Shell characters that bound a token: a volatile path starts right after one of +// them (or at the start of the command) and never crosses one. Kept in a single +// fragment so the boundary and the path-interior classes stay in sync. +const BOUNDARY_CLASS: &str = r#"\s=:'"(){}\[\],&|;<>$`"#; + +/// URL-like tokens: `scheme://…`. Their host, path, and query are all volatile. +static URL: LazyLock = LazyLock::new(|| { + Regex::new(r#"[a-zA-Z][a-zA-Z0-9+.\-]*://[^\s'"(){}\[\],&|;<>`]*"#).expect("valid url regex") +}); + +/// Characters allowed inside a filesystem path token. +const PATH_INTERIOR: &str = r#"[^\s'"(){}\[\],&|;<>$`#]"#; + +/// POSIX absolute paths with at least two segments (`/a/b…`). Requiring two +/// segments avoids collapsing single-slash tokens such as sed flags (`/g`) or a +/// bare division operator. Group 1 preserves the boundary delimiter (start, +/// whitespace, `=`, a quote, …) so the shell structure around the path survives. +static ABSOLUTE_PATH: LazyLock = LazyLock::new(|| { + Regex::new(&format!( + r"(^|[{BOUNDARY_CLASS}])(/{PATH_INTERIOR}+/{PATH_INTERIOR}*)" + )) + .expect("valid absolute-path regex") +}); + +/// Home-relative paths (`~/…`). The `~/` prefix is unambiguous, so a single +/// segment is enough (`~/cio`); no two-segment guard is needed. +static HOME_PATH: LazyLock = LazyLock::new(|| { + Regex::new(&format!(r"(^|[{BOUNDARY_CLASS}])(~/{PATH_INTERIOR}*)")) + .expect("valid home-path regex") +}); + +/// RFC 4122-shaped UUIDs. +static UUID: LazyLock = LazyLock::new(|| { + Regex::new(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b") + .expect("valid uuid regex") +}); + +/// Runs of eight or more hex characters (git SHAs, object ids, content hashes). +/// A run with no hex *letter* is a plain number and is left for [`DIGITS`]. +static HEX: LazyLock = + LazyLock::new(|| Regex::new(r"[0-9a-fA-F]{8,}").expect("valid hex regex")); + +/// Runs of four or more digits (timestamps, ports, line numbers, pids). +static DIGITS: LazyLock = + LazyLock::new(|| Regex::new(r"[0-9]{4,}").expect("valid digit regex")); + fn normalize_command(command: &str, project: Option<&str>) -> String { + // 1. Replace the concrete project prefix with a stable marker so the same + // operation reads identically across repositories. let command = project.map_or_else( || command.to_owned(), |project| command.replace(project, "$PROJECT"), ); - command.split_whitespace().collect::>().join(" ") + // 2. Collapse every run of whitespace to a single space. + let command = command.split_whitespace().collect::>().join(" "); + // 3. Replace volatile tokens with stable placeholders, most specific first, + // preserving surrounding command structure. Ordering matters: URLs before + // paths (a URL's `//host` is not a filesystem path), and hex before digits + // (a hex id may embed digit runs). None of the placeholders can re-match a + // later rule, so the pass is idempotent. + let command = URL.replace_all(&command, "«url»"); + let command = ABSOLUTE_PATH.replace_all(&command, "${1}«path»"); + let command = HOME_PATH.replace_all(&command, "${1}«path»"); + let command = UUID.replace_all(&command, "«uuid»"); + let command = HEX.replace_all(&command, |caps: &Captures<'_>| { + let matched = &caps[0]; + if matched.bytes().any(|byte| byte.is_ascii_alphabetic()) { + "«hex»".to_owned() + } else { + // A pure-digit run is a number, not a hash: leave it for DIGITS so it + // reads as «n» rather than «hex». + matched.to_owned() + } + }); + let command = DIGITS.replace_all(&command, "«n»"); + command.into_owned() } #[cfg(test)] @@ -113,7 +236,7 @@ mod tests { use serde_json::json; use time::OffsetDateTime; - use super::normalize_operation; + use super::{SIGNATURE_SPEC_VERSION, normalize_command_text, normalize_operation}; use crate::{Event, EventId, EventKind, SessionId, SpecVersion, ToolCall}; fn tool_event(input: serde_json::Value, project: Option<&str>) -> Event { @@ -139,6 +262,15 @@ mod tests { } } + fn norm(command: &str) -> String { + normalize_command_text(command, None) + } + + #[test] + fn version_token_is_v2() { + assert_eq!(SIGNATURE_SPEC_VERSION, "v2"); + } + #[test] fn normalizes_tool_alias_whitespace_and_project_prefix() { let event = tool_event( @@ -147,14 +279,16 @@ mod tests { ); let operation = normalize_operation(&event).expect("operation"); assert_eq!(operation.tool(), "shell"); + // The single-segment `$PROJECT/crate` suffix is in-repo structure, not a + // volatile absolute path, so it is preserved. assert_eq!(operation.command(), "cargo test $PROJECT/crate"); assert_eq!( operation.operation_key(), - "operation/v1|shell|cargo test $PROJECT/crate" + "operation/v2|shell|cargo test $PROJECT/crate" ); assert_eq!( operation.failure_signature(2), - "failure/v1|shell|cargo test $PROJECT/crate|exit:2" + "failure/v2|shell|cargo test $PROJECT/crate|exit:2" ); } @@ -162,7 +296,7 @@ mod tests { fn reads_structured_command_input() { let event = tool_event(json!({"command": "pytest -q"}), None); let operation = normalize_operation(&event).expect("operation"); - assert_eq!(operation.operation_key(), "operation/v1|shell|pytest -q"); + assert_eq!(operation.operation_key(), "operation/v2|shell|pytest -q"); } #[test] @@ -170,4 +304,184 @@ mod tests { assert!(normalize_operation(&tool_event(json!({"other": 1}), None)).is_none()); assert!(normalize_operation(&tool_event(json!(" "), None)).is_none()); } + + // --- Per-rule normalization ------------------------------------------- + + #[test] + fn collapses_absolute_paths_but_keeps_structure() { + assert_eq!( + norm("cd /home/user/project && go build"), + "cd «path» && go build" + ); + assert_eq!( + norm("cat /private/tmp/claude-501/x/scratchpad/cookies.txt"), + "cat «path»" + ); + // Boundary delimiters around the path survive: `-f`, `[`, `]`. + assert_eq!(norm("test [ -f /var/run/a/b.pid ]"), "test [ -f «path» ]"); + // A path attached to an assignment keeps the `KEY=` structure. + assert_eq!( + norm("JAR=/private/tmp/x/y/cookies.txt curl"), + "JAR=«path» curl" + ); + } + + #[test] + fn collapses_home_relative_paths() { + // Distinct repositories under the home directory share one shape. + assert_eq!( + norm("cd ~/code/ui && gh pr checks"), + "cd «path» && gh pr checks" + ); + assert_eq!( + norm("cd ~/code/ui && gh pr checks"), + norm("cd ~/code/cdp && gh pr checks") + ); + // A single home-relative segment still normalizes. + assert_eq!(norm("ls ~/cio"), "ls «path»"); + } + + #[test] + fn different_paths_same_structure_collapse_together() { + assert_eq!( + norm("cd /a/first && go build ./..."), + norm("cd /b/second && go build ./...") + ); + } + + #[test] + fn preserves_command_structure_across_binaries_and_flags() { + // No volatile tokens: distinct invocations must stay distinct. + assert_ne!( + norm("cargo test -p autophagy-store"), + norm("cargo test -p autophagy-cli") + ); + assert_eq!( + norm("cargo test -p autophagy-store"), + "cargo test -p autophagy-store" + ); + } + + #[test] + fn collapses_urls() { + assert_eq!( + norm("curl -s -X POST https://api.example.com/v1/x?token=abc"), + "curl -s -X POST «url»" + ); + } + + #[test] + fn collapses_uuids_hex_and_digits() { + assert_eq!( + norm("kill e7fb40d6-5a91-4164-b6d2-4494d95a30b4"), + "kill «uuid»" + ); + assert_eq!(norm("git show a7a903af82db154a3"), "git show «hex»"); + assert_eq!(norm("nc localhost 8080"), "nc localhost «n»"); + // Short numbers are stable structure and are left intact. + assert_eq!(norm("sleep 5"), "sleep 5"); + assert_eq!(norm("head -3 file"), "head -3 file"); + } + + #[test] + fn pure_digit_run_reads_as_number_not_hex() { + // Eight digits are hex-shaped but carry no hex letter: they must read as + // a number, and hex-shaped ids with a letter must read as hex. + assert_eq!(norm("echo 20260717"), "echo «n»"); + assert_eq!(norm("echo deadbeef12"), "echo «hex»"); + } + + #[test] + fn near_miss_audit_examples_recur_across_sessions() { + let a = norm( + "until [ -f /private/tmp/claude-501/-Users-x/e7fb40d6-5a91-4164-b6d2-4494d95a30b4/tasks/a7a903af82db154a3.done ]; do sleep 5; done", + ); + let b = norm( + "until [ -f /private/tmp/claude-501/-Users-y/cb114e1e-bf4e-4b5e-9b34-bc95f435d061/tasks/9f2c1e77bd0a4c11.done ]; do sleep 5; done", + ); + assert_eq!(a, b, "same shell shape must produce one signature"); + assert_eq!(a, "until [ -f «path» ]; do sleep 5; done"); + } + + #[test] + fn normalization_is_idempotent() { + for command in [ + "cd /a/b && go build ./...", + "curl https://x.example/y?z=1 && kill e7fb40d6-5a91-4164-b6d2-4494d95a30b4", + "JAR=/private/tmp/x/y/c.txt run 12345 a7a903af82db154a3", + "cargo test -p autophagy-store", + ] { + let once = norm(command); + let twice = normalize_command_text(&once, None); + assert_eq!( + once, twice, + "normalizing twice must equal once for `{command}`" + ); + } + } + + #[test] + fn minted_signatures_and_fixtures_match_the_v2_schema() { + let schema: serde_json::Value = + serde_json::from_str(include_str!("../../../docs/specs/signature/v2/schema.json")) + .expect("schema JSON"); + let validator = jsonschema::validator_for(&schema).expect("compile schema"); + + // Signatures minted by the code under test satisfy the published grammar. + let event = tool_event(json!("cd /a/b && go build ./..."), None); + let operation = normalize_operation(&event).expect("operation"); + for minted in [operation.operation_key(), operation.failure_signature(1)] { + assert!( + validator.is_valid(&serde_json::Value::String(minted.clone())), + "schema rejected minted signature {minted}" + ); + } + + // Every published valid fixture is accepted; every invalid one rejected. + for fixture in [ + include_str!("../../../docs/specs/signature/v2/valid/operation.json"), + include_str!("../../../docs/specs/signature/v2/valid/operation_with_pipe.json"), + include_str!("../../../docs/specs/signature/v2/valid/failure.json"), + include_str!("../../../docs/specs/signature/v2/valid/recovery.json"), + include_str!("../../../docs/specs/signature/v2/valid/correction.json"), + ] { + let instance: serde_json::Value = serde_json::from_str(fixture).expect("fixture JSON"); + assert!( + validator.is_valid(&instance), + "schema rejected valid {instance}" + ); + } + for fixture in [ + include_str!("../../../docs/specs/signature/v2/invalid/superseded_v1_version.json"), + include_str!("../../../docs/specs/signature/v2/invalid/failure_missing_exit.json"), + include_str!("../../../docs/specs/signature/v2/invalid/operation_empty_tool.json"), + include_str!("../../../docs/specs/signature/v2/invalid/failure_noninteger_exit.json"), + ] { + let instance: serde_json::Value = serde_json::from_str(fixture).expect("fixture JSON"); + assert!( + !validator.is_valid(&instance), + "schema accepted invalid {instance}" + ); + } + } + + #[test] + fn distinct_structures_stay_distinct_after_normalization() { + let commands = [ + "cargo test -p a", + "cargo test -p b", + "cargo build", + "go build ./...", + "cd «path» && go build", + ]; + for (i, left) in commands.iter().enumerate() { + for right in &commands[i + 1..] { + assert_ne!( + norm(left), + norm(right), + "`{left}` and `{right}` must differ" + ); + } + } + } } diff --git a/crates/autophagy-install/tests/codex_skill.rs b/crates/autophagy-install/tests/codex_skill.rs index 5071e3a..413accb 100644 --- a/crates/autophagy-install/tests/codex_skill.rs +++ b/crates/autophagy-install/tests/codex_skill.rs @@ -191,7 +191,7 @@ fn command_failure_package() -> MutationPackage { .find_map(|outcome| match outcome { GenerationOutcome::Candidate { package } if package.mutation_id - == "mut_d6b7a340eb2fb6f18bee4a20932b9c954adb4975f3ea8136bf0bd264b3ec431c" => + == "mut_6b51ef819f54c0275db19b15907b0b23c39598241c912828bb64cd5bf824a0ee" => { Some(*package) } diff --git a/crates/autophagy-mutations/src/generate.rs b/crates/autophagy-mutations/src/generate.rs index a0eccfe..01894bc 100644 --- a/crates/autophagy-mutations/src/generate.rs +++ b/crates/autophagy-mutations/src/generate.rs @@ -95,8 +95,12 @@ pub fn equivalence_key(package: &MutationPackage) -> String { format!("eqv_{encoded}") } +// Selector prefixes track `autophagy_events::signature::SIGNATURE_SPEC_VERSION` +// (currently `v2`). A finding minted under an older grammar (`failure/v1|…`) +// simply does not parse here and yields no v2 candidate — the compatibility +// story in ADR 0014: v1 records stay valid but are never re-derived as v2. fn failure_candidate(finding: &EvidencePacket) -> Option { - let signature = finding.signature.strip_prefix("failure/v1|")?; + let signature = finding.signature.strip_prefix("failure/v2|")?; let (operation, exit_code) = signature.rsplit_once("|exit:")?; let (tool, command) = operation.split_once('|')?; if tool.trim().is_empty() || command.trim().is_empty() || exit_code.parse::().is_err() { @@ -130,7 +134,7 @@ fn failure_candidate(finding: &EvidencePacket) -> Option { } fn correction_candidate(finding: &EvidencePacket) -> Option { - let rule = finding.signature.strip_prefix("correction/v1|")?.trim(); + let rule = finding.signature.strip_prefix("correction/v2|")?.trim(); if rule.is_empty() { return None; } @@ -156,7 +160,7 @@ fn correction_candidate(finding: &EvidencePacket) -> Option { } fn recovery_candidate(finding: &EvidencePacket) -> Option { - let signature = finding.signature.strip_prefix("recovery/v1|")?; + let signature = finding.signature.strip_prefix("recovery/v2|")?; let (target, recovery) = signature.split_once("|via|")?; let (target_operation, exit_code) = target.rsplit_once("|exit:")?; let (_target_tool, target_command) = target_operation.split_once('|')?; diff --git a/crates/autophagy-mutations/tests/candidates.rs b/crates/autophagy-mutations/tests/candidates.rs index 4d5a176..304cd0b 100644 --- a/crates/autophagy-mutations/tests/candidates.rs +++ b/crates/autophagy-mutations/tests/candidates.rs @@ -51,11 +51,22 @@ fn weak_or_malformed_findings_produce_insufficient_evidence() { )); let mut malformed = fixture_findings().remove(0); - malformed.signature = "failure/v1|missing-exit".to_owned(); + malformed.signature = "failure/v2|missing-exit".to_owned(); assert!(matches!( generate_candidate(&malformed), GenerationOutcome::InsufficientEvidence { .. } )); + + // Compatibility (ADR 0014): a well-formed selector minted under the retired + // v1 grammar no longer parses as a v2 selector, so it generates no candidate. + // Already-registered v1 mutations stay valid records; they are simply never + // re-derived under v2. + let mut legacy = fixture_findings().remove(0); + legacy.signature = "failure/v1|shell|cargo build|exit:101".to_owned(); + assert!(matches!( + generate_candidate(&legacy), + GenerationOutcome::InsufficientEvidence { .. } + )); } #[test] diff --git a/crates/autophagy-patterns/src/correction.rs b/crates/autophagy-patterns/src/correction.rs index 0f578dd..5c79ec6 100644 --- a/crates/autophagy-patterns/src/correction.rs +++ b/crates/autophagy-patterns/src/correction.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use autophagy_events::{Event, EventKind}; +use autophagy_events::{Event, EventKind, signature::SIGNATURE_SPEC_VERSION}; use crate::{ Candidate, DetectorConfig, DetectorKind, EvidencePacket, EvidenceReference, @@ -36,7 +36,7 @@ pub(crate) fn detect( let Some(recurrence) = score(&evidence, opposite) else { continue; }; - let packet_signature = format!("correction/v1|{signature}"); + let packet_signature = format!("correction/{SIGNATURE_SPEC_VERSION}|{signature}"); let title = format!("Repeated user correction: {signature}"); if qualifies(&recurrence, config) { findings.push(EvidencePacket { diff --git a/crates/autophagy-patterns/src/lib.rs b/crates/autophagy-patterns/src/lib.rs index 123b262..afd2ee9 100644 --- a/crates/autophagy-patterns/src/lib.rs +++ b/crates/autophagy-patterns/src/lib.rs @@ -29,7 +29,11 @@ const OBSERVATION_LIMIT: usize = 5; /// Bump this whenever detector output or the signature normalization in /// `autophagy-events` changes, so every previously persisted cache entry is /// automatically treated as stale rather than served from an outdated pass. -pub const DETECTION_SPEC_VERSION: &str = "detection/v1"; +/// +/// `v2` accompanies the `v2` signature grammar: detectors now group operations +/// by their volatile-token-normalized shape (see `autophagy_events::signature` +/// and ADR 0014), so every cache entry minted under the `v1` grammar must miss. +pub const DETECTION_SPEC_VERSION: &str = "detection/v2"; /// Thresholds shared by deterministic recurrence detectors. #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/crates/autophagy-patterns/src/recovery.rs b/crates/autophagy-patterns/src/recovery.rs index 27fe9d6..23799e1 100644 --- a/crates/autophagy-patterns/src/recovery.rs +++ b/crates/autophagy-patterns/src/recovery.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, BTreeSet}; -use autophagy_events::{Event, EventKind}; +use autophagy_events::{Event, EventKind, signature::SIGNATURE_SPEC_VERSION}; use crate::{ Candidate, DetectorConfig, DetectorKind, EvidencePacket, EvidenceReference, @@ -167,15 +167,17 @@ fn preceding_failure<'a>( } fn recovery_signature(failure: &FailureOperation, recovery: &FailureOperation) -> String { + let failure_prefix = format!("failure/{SIGNATURE_SPEC_VERSION}|"); + let operation_prefix = format!("operation/{SIGNATURE_SPEC_VERSION}|"); format!( - "recovery/v1|{}|via|{}", + "recovery/{SIGNATURE_SPEC_VERSION}|{}|via|{}", failure .signature - .strip_prefix("failure/v1|") + .strip_prefix(&failure_prefix) .unwrap_or(&failure.signature), recovery .success_key - .strip_prefix("operation/v1|") + .strip_prefix(&operation_prefix) .unwrap_or(&recovery.success_key) ) } diff --git a/crates/autophagy-patterns/tests/detectors.rs b/crates/autophagy-patterns/tests/detectors.rs index 1f27cef..d219820 100644 --- a/crates/autophagy-patterns/tests/detectors.rs +++ b/crates/autophagy-patterns/tests/detectors.rs @@ -117,7 +117,7 @@ fn repeated_successful_recovery_preserves_composite_lineage() { assert_eq!(recovery.counterexamples.len(), 2); assert_eq!( recovery.signature, - "recovery/v1|shell|mise run check|exit:1|via|shell|mise run codegen" + "recovery/v2|shell|mise run check|exit:1|via|shell|mise run codegen" ); assert!( recovery @@ -165,6 +165,52 @@ fn shell_event(index: u32, session: &str, kind: EventKind, command: &str, exit: } } +/// Cross-path recurrence: the v2 signature grammar (ADR 0014) normalizes +/// volatile absolute paths, so the *same* failing command shape recurring under +/// three different scratchpad paths across three sessions now qualifies as one +/// repeated-command-failure finding. Under the v1 literal grammar these three +/// byte-distinct commands never grouped and produced zero findings — the exact +/// yield gap this change closes. +#[test] +fn volatile_paths_recur_as_one_failure_under_v2() { + let commands = [ + ( + "ses_a", + "cat /private/tmp/session-11112222/notes.txt && go build ./...", + ), + ( + "ses_b", + "cat /private/tmp/session-33334444/notes.txt && go build ./...", + ), + ( + "ses_c", + "cat /private/tmp/session-55556666/notes.txt && go build ./...", + ), + ]; + let events = commands + .iter() + .enumerate() + .map(|(index, (session, command))| { + #[allow(clippy::cast_possible_truncation)] + shell_event(index as u32, session, EventKind::ToolFailed, command, 1) + }) + .collect::>(); + + let findings = detect(&events, DetectorConfig::default()); + let failure = findings + .iter() + .find(|finding| finding.detector == DetectorKind::RepeatedCommandFailure) + .expect("distinct-path failures share one normalized signature and qualify"); + assert_eq!(failure.score.occurrences, 3); + assert_eq!(failure.score.distinct_sessions, 3); + assert_eq!( + failure.signature, + "failure/v2|shell|cat «path» && go build ./...|exit:1" + ); + // The three byte-distinct raw commands are all retained as exact evidence. + assert_eq!(failure.evidence.len(), 3); +} + /// Regression for the real-data threshold bug: an operation that succeeds far /// more often than it fails must still qualify as a repeated failure, because /// qualification is about cross-session recurrence, not overall failure share. diff --git a/crates/autophagy-replay/tests/replay.rs b/crates/autophagy-replay/tests/replay.rs index 3441c36..7e55b93 100644 --- a/crates/autophagy-replay/tests/replay.rs +++ b/crates/autophagy-replay/tests/replay.rs @@ -282,7 +282,7 @@ fn command_failure_package_from_events(events: &[Event]) -> MutationPackage { .find_map(|outcome| match outcome { GenerationOutcome::Candidate { package } if package.mutation_id - == "mut_d6b7a340eb2fb6f18bee4a20932b9c954adb4975f3ea8136bf0bd264b3ec431c" => + == "mut_6b51ef819f54c0275db19b15907b0b23c39598241c912828bb64cd5bf824a0ee" => { Some(*package) } diff --git a/crates/autophagy-shadow/tests/shadow.rs b/crates/autophagy-shadow/tests/shadow.rs index 82a166c..56f4e59 100644 --- a/crates/autophagy-shadow/tests/shadow.rs +++ b/crates/autophagy-shadow/tests/shadow.rs @@ -94,7 +94,7 @@ fn command_failure_package() -> MutationPackage { .find_map(|outcome| match outcome { GenerationOutcome::Candidate { package } if package.mutation_id - == "mut_d6b7a340eb2fb6f18bee4a20932b9c954adb4975f3ea8136bf0bd264b3ec431c" => + == "mut_6b51ef819f54c0275db19b15907b0b23c39598241c912828bb64cd5bf824a0ee" => { Some(*package) } diff --git a/crates/autophagy-store/src/store.rs b/crates/autophagy-store/src/store.rs index bd2f99d..7aa8a8d 100644 --- a/crates/autophagy-store/src/store.rs +++ b/crates/autophagy-store/src/store.rs @@ -557,6 +557,30 @@ impl EventStore { Ok(u64::try_from(count).unwrap_or(0)) } + /// Number of indexed signatures **not** minted under `current_prefix`. + /// + /// Every indexed signature is an `operation/|…` key, so passing the + /// caller's current operation prefix (for example `operation/v2|`) counts the + /// rows left over from a superseded signature grammar. A non-zero result + /// means the exact-signature index was built under an older grammar and no + /// longer matches freshly minted signatures; rebuilding it with + /// `reindex --index-tool-input` re-mints every row under the current grammar. + /// The count is diagnostic only — the stale rows stay usable for exact recall + /// of their own (old) signature until a rebuild replaces them. + /// + /// # Errors + /// + /// Returns [`StoreError`] when `SQLite` cannot execute the query. + pub fn signatures_below_version(&self, current_prefix: &str) -> Result { + let pattern = format!("{}%", escape_like(current_prefix)); + let count: i64 = self.connection.query_row( + "SELECT count(*) FROM event_signatures WHERE signature NOT LIKE ?1 ESCAPE '\\'", + params![pattern], + |row| row.get(0), + )?; + Ok(u64::try_from(count).unwrap_or(0)) + } + /// Rebuild the derived search projections from every stored event's /// canonical `event_json`, applying the caller-supplied redaction-approved /// projection. @@ -2665,3 +2689,16 @@ fn compare_bm25(left: Option, right: Option) -> std::cmp::Ordering { (None, None) => std::cmp::Ordering::Equal, } } + +/// Escape the SQL `LIKE` metacharacters (`\`, `%`, `_`) in a literal prefix so it +/// matches only itself. Paired with an explicit `ESCAPE '\'` clause. +fn escape_like(literal: &str) -> String { + let mut escaped = String::with_capacity(literal.len()); + for ch in literal.chars() { + if matches!(ch, '\\' | '%' | '_') { + escaped.push('\\'); + } + escaped.push(ch); + } + escaped +} diff --git a/crates/autophagy-store/tests/store.rs b/crates/autophagy-store/tests/store.rs index ba8b461..e23e53d 100644 --- a/crates/autophagy-store/tests/store.rs +++ b/crates/autophagy-store/tests/store.rs @@ -1278,8 +1278,8 @@ fn hit_ids(hits: &[autophagy_store::RetrievalHit]) -> Vec<&str> { hits.iter().map(|hit| hit.event_id.as_str()).collect() } -const BUILD_SIG: &str = "operation/v1|shell|cargo build"; -const TEST_SIG: &str = "operation/v1|shell|npm test"; +const BUILD_SIG: &str = "operation/v2|shell|cargo build"; +const TEST_SIG: &str = "operation/v2|shell|npm test"; fn seed_retrieval_store() -> EventStore { let mut store = EventStore::open_in_memory().expect("store"); @@ -1348,6 +1348,26 @@ fn seed_retrieval_store() -> EventStore { store } +#[test] +fn signatures_below_version_counts_only_superseded_grammar() { + let store = seed_retrieval_store(); + // The seeded index is entirely `operation/v2|…`, so none are stale under v2. + assert_eq!( + store + .signatures_below_version("operation/v2|") + .expect("v2 count"), + 0 + ); + // Against a newer grammar prefix, every indexed row is superseded. The store + // seeds three `cargo build` rows plus one `npm test` row. + assert_eq!( + store + .signatures_below_version("operation/v3|") + .expect("v3 count"), + 4 + ); +} + #[test] fn exact_signature_lookup_orders_by_recency_then_id() { let store = seed_retrieval_store(); @@ -1612,7 +1632,7 @@ fn signature_index_preserves_idempotency_quarantine_and_deletion() { .insert_event( &source, &conflicting, - &retrieval_projection("conflicting", "operation/v1|shell|cargo build --release"), + &retrieval_projection("conflicting", "operation/v2|shell|cargo build --release"), ) .expect("conflict insert"), InsertOutcome::ConflictQuarantined { .. } @@ -1624,7 +1644,7 @@ fn signature_index_preserves_idempotency_quarantine_and_deletion() { assert!( store .retrieve(&RetrievalQuery { - signature: Some("operation/v1|shell|cargo build --release".to_owned()), + signature: Some("operation/v2|shell|cargo build --release".to_owned()), limit: 10, ..RetrievalQuery::default() }) diff --git a/docs/architecture/database-schema.md b/docs/architecture/database-schema.md index afc2d31..ff659e2 100644 --- a/docs/architecture/database-schema.md +++ b/docs/architecture/database-schema.md @@ -279,7 +279,7 @@ are never indexed blindly. `event_signatures` is the exact normalized-signature index behind hybrid retrieval. A row holds one event's normalized operation signature (for example -`operation/v1|shell|cargo build`), supplied through the same redaction-approved +`operation/v2|shell|cargo build`), supplied through the same redaction-approved search projection that gates free-text tool input, and written in the same transaction as the event. Because a signature embeds command text, it is indexed only when the source's tool input is approved for indexing. Rows cascade on event diff --git a/docs/decisions/0014-signature-normalization-v2.md b/docs/decisions/0014-signature-normalization-v2.md new file mode 100644 index 0000000..5fa4282 --- /dev/null +++ b/docs/decisions/0014-signature-normalization-v2.md @@ -0,0 +1,136 @@ +# ADR 0014: Normalize volatile tokens in recurrence signatures (v2) + +- Status: accepted +- Date: 2026-07-17 + +## Context + +Recurrence detection is only as good as the signature that groups operations. A +`v1` signature embedded the full literal command text after two cosmetic passes +(tool-alias folding and the project-prefix `$PROJECT` substitution): +`operation/v1|shell|` and `failure/v1|shell||exit:`. + +Real coding-agent commands are long, one-shot compounds dense with volatile +tokens — absolute scratchpad and session directories, UUIDs, content hashes, +timestamps, ports, and line numbers. Two semantically identical operations almost +never produce a byte-identical `v1` command string, so they never share a +signature and never register as recurring. + +A hands-on audit of the real Claude Code history on the development machine made +the gap concrete. Importing 35,497 events across 151 sessions produced **276 +candidate failure signatures and 0 findings**, and the entire near-miss list was +1-occurrence entries whose only variation was volatile tokens, for example: + +- `until [ -f /private/tmp/claude-501/-Users-…//tasks/.done ]; do sleep 5; done; …` +- `JAR=/private/tmp/claude-501/-Users-…//scratchpad/cookies.txt # 1. create geoset curl -s -b $JAR -X POST http://…` + +The full 69,400-event database yielded exactly 2 findings, both from short, +literally-repeated commands (`cd zuzoto && go build ./... 2>&1`). On its most +common agent the product read as dead. The signature grammar — not the detectors +or thresholds — was the bottleneck. + +Signatures are a versioned public contract (they are minted into evidence +packets, indexed for exact retrieval, and embedded verbatim in mutation trigger +selectors). Per the non-negotiable engineering constraints, changing how they are +minted requires a version bump, a decision record, and an explicit compatibility +story. It must also stay deterministic, inspectable, and model-free. + +## Decision + +Introduce a deterministic signature-normalization pass and bump the signature +grammar from `v1` to `v2`. `autophagy_events::signature::normalize_operation` +(the single implementation both the detectors and the retrieval index build on) +now, after the existing project-prefix and whitespace passes, replaces volatile +tokens with stable placeholders. The rules apply in order, most specific first, +over the whitespace-collapsed command string; command *structure* — binaries, +subcommands, flags, and shell operators — is deliberately preserved. + +| Volatile token | Rule | Placeholder | +| --- | --- | --- | +| URL | `scheme://…` up to the next shell delimiter | `«url»` | +| Absolute path | `/seg/seg…` (≥2 segments) at a shell boundary | `«path»` | +| Home path | `~/…` at a shell boundary (≥1 segment) | `«path»` | +| UUID | RFC 4122 shape `8-4-4-4-12` | `«uuid»` | +| Hex run | ≥8 hex chars containing at least one `a–f` letter | `«hex»` | +| Digit run | ≥4 consecutive digits | `«n»` | + +Design properties: + +- **Structure preserved.** `cargo test -p autophagy-store` and + `cargo test -p autophagy-cli` stay distinct; `cd /a/b && go build` and + `cd /c/d && go build` collapse to one shape. The two-segment guard on absolute + paths avoids collapsing single-slash tokens such as sed flags (`/g`) or a bare + division operator; the `~/` prefix is unambiguous and needs no such guard. +- **Numbers vs hashes.** Hex runs are matched before digit runs, but a run with + no `a–f` letter (a plain long number such as a date `20260717`) is left for the + digit rule, so it reads as `«n»` rather than `«hex»`. +- **Pure and total.** The pass is a function of the command text alone — no + locale, clock, filesystem, network, or model. It is idempotent: normalizing an + already-normalized command is a fixed point (the placeholders cannot re-match + any rule). Distinct structures stay distinct. + +The version bump is applied at every mint site: `operation/v2`, `failure/v2`, +`recovery/v2`, and `correction/v2`. The correction family carries no command text +and its normalization is unchanged, but it bumps too so the whole selector +grammar advances as one coherent version. The single source of truth is +`SIGNATURE_SPEC_VERSION` in `autophagy-events`. + +`DETECTION_SPEC_VERSION` bumps to `detection/v2` in lockstep. It is folded into +the findings-cache key (ADR 0013), so every entry minted under the `v1` grammar +misses and recomputes automatically — no stale report can be served. + +The grammar is documented and fixture-pinned under +[`docs/specs/signature/v2/`](../specs/signature/v2/README.md); the `v2` mint +functions are validated against that schema in `autophagy-events`. + +## Real-data verification + +Re-importing the same Claude Code history (35,682 events, 151 sessions) into a +throwaway database confirms the mechanism and its honest limits: + +- **Operation-level recurrence lifts sharply.** Distinct operation shapes that + recur at the default gate (≥3 occurrences across ≥2 sessions) rose from **2 + under the `v1` literal grammar to 44 under `v2`** — a 22× gain. Volatile + one-shots now share sensible shapes: `operation/v2|shell|«path»` (72×), + `cat «path»` (28×), `reins screenshot --tab «n»` (14×). The distribution is + healthy, not a degenerate mega-bucket (the largest is 0.4% of indexed rows). +- **Failure findings remain data-dependent.** This corpus holds 390 tool + failures forming 276 distinct failing shells, and — verified directly — **not + one repeats even twice**, under either grammar. The failure detector therefore + still reports 276 candidates / 0 findings here: the user's failing commands are + genuinely one-shot exploratory invocations, so there is no recurrence to + surface. `v2` removes the signature bottleneck; it does not manufacture + recurrence that the history does not contain. + +The end-to-end detector gain is proven deterministically in +`autophagy-patterns` (`volatile_paths_recur_as_one_failure_under_v2`): three +byte-distinct failing commands differing only by scratchpad path now share one +`failure/v2` signature and qualify as a single finding across three sessions — +exactly the class of failure the `v1` grammar discarded. + +## Compatibility + +- **No historical rewrite.** Canonical events are untouched; nothing migrates + stored evidence. Only derived, re-derivable projections change. +- **Registered mutations stay valid.** The mutation registry is append-only and + audit-logged; already-registered candidates keep their immutable `v1` trigger + selectors as valid records. A `v1` selector simply no longer matches a + freshly minted `v2` signature. Re-proposing a finding generates a fresh `v2` + candidate (a distinct equivalence key), which is the intended path forward; the + deterministic generator refuses to parse a `v1` selector into a `v2` candidate. +- **Stored index re-mints via `reindex`.** The exact-signature index is a derived + projection. `autophagy reindex --index-tool-input` re-mints every row under + `v2` from the untouched canonical events. Because a database indexed before this + change carries `operation/v1|…` rows that no longer match new signatures, + `autophagy status` detects the mismatch (`EventStore::signatures_below_version`) + and prints a one-line hint to reindex. + +## Privacy + +Normalization is strictly subtractive on derived text: it replaces volatile +tokens with fixed placeholders, so a `v2` signature contains *less* literal +command text than the `v1` signature it supersedes — absolute paths, URLs, +UUIDs, hashes, and long numbers no longer appear in the derived signature at all. +Raw events are unchanged, no new text is persisted, and the signature is still +indexed only from the redaction-approved projection. The change reduces, and +never increases, the sensitive surface of derived data. diff --git a/docs/guides/retrieval.md b/docs/guides/retrieval.md index 8ed7cd3..c316bcc 100644 --- a/docs/guides/retrieval.md +++ b/docs/guides/retrieval.md @@ -14,7 +14,7 @@ The design rationale and privacy stance are in ## Exact normalized signatures Every tool event with an inspectable command is indexed under a normalized -operation signature such as `operation/v1|shell|cargo build`. The same +operation signature such as `operation/v2|shell|cargo build`. The same model-free normalizer that powers the deterministic detectors collapses tool aliases (`bash`, `exec_command`, `shell`), whitespace, and the exact project prefix (`$PROJECT`), so incidental variation does not fragment identical @@ -23,7 +23,7 @@ operations. Look one up exactly: ```sh -autophagy search --signature 'operation/v1|shell|cargo build' +autophagy search --signature 'operation/v2|shell|cargo build' ``` Exact-signature lookup does not require a text query. Combine it with a text @@ -55,7 +55,7 @@ Each filter narrows both the exact and full-text match sources identically: ```sh # Repository filter (exact project path). -autophagy search --signature 'operation/v1|shell|cargo build' --project /repo/alpha +autophagy search --signature 'operation/v2|shell|cargo build' --project /repo/alpha # Recency filter (events within the last 14 days). autophagy search 'linker error' --since-days 14 @@ -64,7 +64,7 @@ autophagy search 'linker error' --since-days 14 autophagy search 'pytest' --event-kind tool.failed --event-kind test.failed # Outcome filter (success or failure polarity). -autophagy search --signature 'operation/v1|shell|cargo build' --outcome failure +autophagy search --signature 'operation/v2|shell|cargo build' --outcome failure ``` Applied filters are echoed into every hit's explanation, so a result set is @@ -77,7 +77,7 @@ self-describing. `explanation` object: ```sh -autophagy --output json search 'succeeded' --signature 'operation/v1|shell|cargo build' +autophagy --output json search 'succeeded' --signature 'operation/v2|shell|cargo build' ``` ## Privacy diff --git a/docs/specs/retrieval/0.1/valid/exact_signature.json b/docs/specs/retrieval/0.1/valid/exact_signature.json index 43e795a..38369d3 100644 --- a/docs/specs/retrieval/0.1/valid/exact_signature.json +++ b/docs/specs/retrieval/0.1/valid/exact_signature.json @@ -6,7 +6,7 @@ { "kind": "exact_signature", "contribution_bps": 10000, - "detail": "exact match on signature operation/v1|shell|cargo build" + "detail": "exact match on signature operation/v2|shell|cargo build" }, { "kind": "recency", diff --git a/docs/specs/retrieval/0.1/valid/hybrid_with_filters.json b/docs/specs/retrieval/0.1/valid/hybrid_with_filters.json index 3d45203..42db56d 100644 --- a/docs/specs/retrieval/0.1/valid/hybrid_with_filters.json +++ b/docs/specs/retrieval/0.1/valid/hybrid_with_filters.json @@ -6,7 +6,7 @@ { "kind": "exact_signature", "contribution_bps": 10000, - "detail": "exact match on signature operation/v1|shell|cargo build" + "detail": "exact match on signature operation/v2|shell|cargo build" }, { "kind": "full_text", diff --git a/docs/specs/signature/v2/README.md b/docs/specs/signature/v2/README.md new file mode 100644 index 0000000..0eaf8bd --- /dev/null +++ b/docs/specs/signature/v2/README.md @@ -0,0 +1,60 @@ +# Signature grammar v2 + +A signature is the deterministic, model-free identity that groups tool +operations for recurrence detection and exact retrieval. It is a versioned +public contract: signatures are minted into evidence packets, indexed for exact +recall, and embedded verbatim in mutation trigger selectors. + +- [`schema.json`](schema.json) is normative for the selector *envelope* grammar. +- [`valid/`](valid) holds selectors the schema accepts. +- [`invalid/`](invalid) holds selectors the schema rejects. + +The canonical implementation is +`autophagy_events::signature::normalize_operation`; `SIGNATURE_SPEC_VERSION` +(`v2`) is its single source of truth. See +[ADR 0014](../../../decisions/0014-signature-normalization-v2.md) for the +rationale, the real-data yield numbers, and the `v1 → v2` compatibility story. + +## Selector families + +Each selector is `family/v2|…`: + +| Family | Shape | +| --- | --- | +| operation | `operation/v2\|\|` | +| failure | `failure/v2\|\|\|exit:` | +| recovery | `recovery/v2\|\|\|exit:\|via\|\|` | +| correction | `correction/v2\|` | + +`` is the alias-folded tool name (`bash`/`exec`/`shell`/`terminal` → +`shell`). `` is normalized command text and may itself contain `|` +(shell pipes). `` is a decimal exit code. `` is a lowercased, +whitespace-collapsed user-authored correction rule. + +## Command normalization (v2) + +`` is produced by a pure, total, idempotent pass. After folding the +tool alias, replacing the concrete project prefix with `$PROJECT`, and collapsing +whitespace runs to single spaces, volatile tokens are replaced with stable +placeholders, most specific first: + +1. URLs (`scheme://…`) → `«url»` +2. Absolute POSIX paths with ≥2 segments (`/a/b…`) → `«path»` +3. Home-relative paths (`~/…`) → `«path»` +4. UUIDs (RFC 4122 `8-4-4-4-12`) → `«uuid»` +5. Hex runs of ≥8 characters containing at least one `a–f` letter → `«hex»` +6. Digit runs of ≥4 → `«n»` + +Command *structure* — binaries, subcommands, flags, and shell operators — is +preserved, so `cargo test -p a` and `cargo test -p b` stay distinct while +`cd /x && go build` and `cd /y && go build` collapse to one shape. The pass +never consults a model, the clock, the locale, the filesystem, or the network, +and normalizing an already-normalized command is a fixed point. + +## Compatibility with v1 + +`v1` selectors embedded literal command text. The grammars are intentionally +non-interoperable: a `v1` selector never matches a freshly minted `v2` signature. +Already-registered mutations keep their immutable `v1` selectors as valid audit +records; stored and indexed signatures are re-minted under `v2` by +`autophagy reindex --index-tool-input`. No historical evidence is rewritten. diff --git a/docs/specs/signature/v2/invalid/failure_missing_exit.json b/docs/specs/signature/v2/invalid/failure_missing_exit.json new file mode 100644 index 0000000..1dd4ffb --- /dev/null +++ b/docs/specs/signature/v2/invalid/failure_missing_exit.json @@ -0,0 +1 @@ +"failure/v2|shell|cargo build" \ No newline at end of file diff --git a/docs/specs/signature/v2/invalid/failure_noninteger_exit.json b/docs/specs/signature/v2/invalid/failure_noninteger_exit.json new file mode 100644 index 0000000..158b94f --- /dev/null +++ b/docs/specs/signature/v2/invalid/failure_noninteger_exit.json @@ -0,0 +1 @@ +"failure/v2|shell|cargo build|exit:abc" \ No newline at end of file diff --git a/docs/specs/signature/v2/invalid/operation_empty_tool.json b/docs/specs/signature/v2/invalid/operation_empty_tool.json new file mode 100644 index 0000000..954d0cd --- /dev/null +++ b/docs/specs/signature/v2/invalid/operation_empty_tool.json @@ -0,0 +1 @@ +"operation/v2||cargo build" \ No newline at end of file diff --git a/docs/specs/signature/v2/invalid/superseded_v1_version.json b/docs/specs/signature/v2/invalid/superseded_v1_version.json new file mode 100644 index 0000000..3e32c7c --- /dev/null +++ b/docs/specs/signature/v2/invalid/superseded_v1_version.json @@ -0,0 +1 @@ +"operation/v1|shell|cargo build" \ No newline at end of file diff --git a/docs/specs/signature/v2/schema.json b/docs/specs/signature/v2/schema.json new file mode 100644 index 0000000..d0d73fb --- /dev/null +++ b/docs/specs/signature/v2/schema.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://autophagy.sh/specs/signature/v2/schema.json", + "title": "Autophagy Signature Selector v2", + "description": "Normative envelope grammar for a v2 recurrence signature / mutation trigger selector. The command-text normalization rules (paths, urls, uuids, hex, digits) are documented in README.md and enforced by the minting code; this schema pins the versioned family envelope so a v1 selector or a malformed v2 selector is rejected.", + "type": "string", + "anyOf": [ + { "pattern": "^operation/v2\\|[^|]+\\|.+$" }, + { "pattern": "^failure/v2\\|[^|]+\\|.+\\|exit:-?[0-9]+$" }, + { "pattern": "^recovery/v2\\|[^|]+\\|.+\\|exit:-?[0-9]+\\|via\\|[^|]+\\|.+$" }, + { "pattern": "^correction/v2\\|.+$" } + ] +} diff --git a/docs/specs/signature/v2/valid/correction.json b/docs/specs/signature/v2/valid/correction.json new file mode 100644 index 0000000..6d7f826 --- /dev/null +++ b/docs/specs/signature/v2/valid/correction.json @@ -0,0 +1 @@ +"correction/v2|run codegen before tests" \ No newline at end of file diff --git a/docs/specs/signature/v2/valid/failure.json b/docs/specs/signature/v2/valid/failure.json new file mode 100644 index 0000000..112c2e0 --- /dev/null +++ b/docs/specs/signature/v2/valid/failure.json @@ -0,0 +1 @@ +"failure/v2|shell|cd «path» && go build ./...|exit:1" \ No newline at end of file diff --git a/docs/specs/signature/v2/valid/operation.json b/docs/specs/signature/v2/valid/operation.json new file mode 100644 index 0000000..bdc91ef --- /dev/null +++ b/docs/specs/signature/v2/valid/operation.json @@ -0,0 +1 @@ +"operation/v2|shell|cat «path»" \ No newline at end of file diff --git a/docs/specs/signature/v2/valid/operation_with_pipe.json b/docs/specs/signature/v2/valid/operation_with_pipe.json new file mode 100644 index 0000000..7b5f8e5 --- /dev/null +++ b/docs/specs/signature/v2/valid/operation_with_pipe.json @@ -0,0 +1 @@ +"operation/v2|shell|reins screenshot --tab «n» 2>&1 | tail -1" \ No newline at end of file diff --git a/docs/specs/signature/v2/valid/recovery.json b/docs/specs/signature/v2/valid/recovery.json new file mode 100644 index 0000000..283e380 --- /dev/null +++ b/docs/specs/signature/v2/valid/recovery.json @@ -0,0 +1 @@ +"recovery/v2|shell|mise run check|exit:1|via|shell|mise run codegen" \ No newline at end of file diff --git a/evals/fixtures/replay/command-preflight-draft.json b/evals/fixtures/replay/command-preflight-draft.json index 3d134e1..16ba022 100644 --- a/evals/fixtures/replay/command-preflight-draft.json +++ b/evals/fixtures/replay/command-preflight-draft.json @@ -1,19 +1,10 @@ { "spec_version": "replay-suite/0.1", - "mutation_id": "mut_d6b7a340eb2fb6f18bee4a20932b9c954adb4975f3ea8136bf0bd264b3ec431c", + "mutation_id": "mut_6b51ef819f54c0275db19b15907b0b23c39598241c912828bb64cd5bf824a0ee", "scenarios": [ { "spec_version": "replay-scenario/0.1", - "scenario_id": "rps_0ac300c6bb4fa5dc4fec94d19870e98f521e11e3b1e5b9e384b645ff0b1ad822", - "source_event_ids": ["evt_failure_3"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], - "expected_action": "intervene", - "counterfactual_outcome": "unknown", - "note": "Review draft from 1 exact evidence event(s) and 0 nearby event(s) in session ses_failure_3. Context: tool.failed. Set counterfactual_outcome to expected_result or contradiction before evaluation." - }, - { - "spec_version": "replay-scenario/0.1", - "scenario_id": "rps_1fdacab5d36a425d2aad793b72ee1a3ac915ed6830bf3c472cb7e403ca44e2ef", + "scenario_id": "rps_77c276dfd99cb70fb4f6f7a2d468f27bc3aefb33957b7f924f1a5a89b848a23b", "source_event_ids": ["evt_success_1"], "observed_trigger_selectors": ["event/v1|type:tool.completed|tool:exec|exit:0"], "expected_action": "no_op", @@ -21,21 +12,30 @@ }, { "spec_version": "replay-scenario/0.1", - "scenario_id": "rps_4f3f45d4f56951fbc777f967d3aee40ed2025e6ffce310f3ae18f9a8e6c573d0", + "scenario_id": "rps_7eb2bffe7596d0ccd929870449e9be9109929baf0799cad5fc59f5da1a1da3c7", "source_event_ids": ["evt_failure_1"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "expected_action": "intervene", "counterfactual_outcome": "unknown", "note": "Review draft from 1 exact evidence event(s) and 0 nearby event(s) in session ses_failure_1. Context: tool.failed. Set counterfactual_outcome to expected_result or contradiction before evaluation." }, { "spec_version": "replay-scenario/0.1", - "scenario_id": "rps_6937dd64cf27b57b419af8590276d64ff2c413d954e3713d77e59bbe7af01df0", + "scenario_id": "rps_ac2ee7748c40173964cda21d78049b4e5e5f6ce65408e78faf39652ac61d8787", "source_event_ids": ["evt_failure_2"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "expected_action": "intervene", "counterfactual_outcome": "unknown", "note": "Review draft from 1 exact evidence event(s) and 0 nearby event(s) in session ses_failure_2. Context: tool.failed. Set counterfactual_outcome to expected_result or contradiction before evaluation." + }, + { + "spec_version": "replay-scenario/0.1", + "scenario_id": "rps_bbbe12ebb281d1e8ddd654a08f876baee63051b04b24ee08fc9ba38b7feba10a", + "source_event_ids": ["evt_failure_3"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], + "expected_action": "intervene", + "counterfactual_outcome": "unknown", + "note": "Review draft from 1 exact evidence event(s) and 0 nearby event(s) in session ses_failure_3. Context: tool.failed. Set counterfactual_outcome to expected_result or contradiction before evaluation." } ] } diff --git a/evals/fixtures/replay/command-preflight-fail.json b/evals/fixtures/replay/command-preflight-fail.json index 226e322..57d3667 100644 --- a/evals/fixtures/replay/command-preflight-fail.json +++ b/evals/fixtures/replay/command-preflight-fail.json @@ -1,12 +1,12 @@ { "spec_version": "replay-suite/0.1", - "mutation_id": "mut_d6b7a340eb2fb6f18bee4a20932b9c954adb4975f3ea8136bf0bd264b3ec431c", + "mutation_id": "mut_6b51ef819f54c0275db19b15907b0b23c39598241c912828bb64cd5bf824a0ee", "scenarios": [ { "spec_version": "replay-scenario/0.1", "scenario_id": "rps_failure_helped_a", "source_event_ids": ["evt_failure_1"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "expected_action": "intervene", "counterfactual_outcome": "expected_result" }, @@ -14,7 +14,7 @@ "spec_version": "replay-scenario/0.1", "scenario_id": "rps_failure_helped_b", "source_event_ids": ["evt_failure_2"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "expected_action": "intervene", "counterfactual_outcome": "expected_result" }, @@ -22,7 +22,7 @@ "spec_version": "replay-scenario/0.1", "scenario_id": "rps_failure_contradicted", "source_event_ids": ["evt_failure_3"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "expected_action": "intervene", "counterfactual_outcome": "contradiction" }, @@ -30,14 +30,14 @@ "spec_version": "replay-scenario/0.1", "scenario_id": "rps_noop_correct", "source_event_ids": ["evt_success_1"], - "observed_trigger_selectors": ["failure/v1|shell|cargo test|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|cargo test|exit:1"], "expected_action": "no_op" }, { "spec_version": "replay-scenario/0.1", "scenario_id": "rps_noop_false_intervention", "source_event_ids": ["evt_correction_counterexample"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "expected_action": "no_op" } ] diff --git a/evals/fixtures/replay/command-preflight-pass.json b/evals/fixtures/replay/command-preflight-pass.json index 1234e83..01f305c 100644 --- a/evals/fixtures/replay/command-preflight-pass.json +++ b/evals/fixtures/replay/command-preflight-pass.json @@ -1,12 +1,12 @@ { "spec_version": "replay-suite/0.1", - "mutation_id": "mut_d6b7a340eb2fb6f18bee4a20932b9c954adb4975f3ea8136bf0bd264b3ec431c", + "mutation_id": "mut_6b51ef819f54c0275db19b15907b0b23c39598241c912828bb64cd5bf824a0ee", "scenarios": [ { "spec_version": "replay-scenario/0.1", "scenario_id": "rps_failure_preconditions", "source_event_ids": ["evt_failure_1"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "expected_action": "intervene", "counterfactual_outcome": "expected_result" }, @@ -14,7 +14,7 @@ "spec_version": "replay-scenario/0.1", "scenario_id": "rps_failure_unchanged_retry", "source_event_ids": ["evt_failure_2"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "expected_action": "intervene", "counterfactual_outcome": "expected_result" }, @@ -22,7 +22,7 @@ "spec_version": "replay-scenario/0.1", "scenario_id": "rps_failure_changed_hypothesis", "source_event_ids": ["evt_failure_3"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "expected_action": "intervene", "counterfactual_outcome": "expected_result" }, @@ -30,14 +30,14 @@ "spec_version": "replay-scenario/0.1", "scenario_id": "rps_noop_different_command", "source_event_ids": ["evt_success_1"], - "observed_trigger_selectors": ["failure/v1|shell|mise run test|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run test|exit:1"], "expected_action": "no_op" }, { "spec_version": "replay-scenario/0.1", "scenario_id": "rps_noop_successful_command", "source_event_ids": ["evt_correction_counterexample"], - "observed_trigger_selectors": ["correction/v1|run codegen before tests"], + "observed_trigger_selectors": ["correction/v2|run codegen before tests"], "expected_action": "no_op" } ] diff --git a/evals/fixtures/shadow/command-preflight-fail.json b/evals/fixtures/shadow/command-preflight-fail.json index 9321a51..324499e 100644 --- a/evals/fixtures/shadow/command-preflight-fail.json +++ b/evals/fixtures/shadow/command-preflight-fail.json @@ -1,40 +1,40 @@ { "spec_version": "shadow-suite/0.1", - "mutation_id": "mut_d6b7a340eb2fb6f18bee4a20932b9c954adb4975f3ea8136bf0bd264b3ec431c", + "mutation_id": "mut_6b51ef819f54c0275db19b15907b0b23c39598241c912828bb64cd5bf824a0ee", "observations": [ { "spec_version": "shadow-observation/0.1", "observation_id": "shd_failure_one", "source_event_ids": ["evt_failure_1"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "intervention_would_help": true }, { "spec_version": "shadow-observation/0.1", "observation_id": "shd_failure_two", "source_event_ids": ["evt_failure_2"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "intervention_would_help": true }, { "spec_version": "shadow-observation/0.1", "observation_id": "shd_failure_three", "source_event_ids": ["evt_failure_3"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "intervention_would_help": true }, { "spec_version": "shadow-observation/0.1", "observation_id": "shd_false_positive", "source_event_ids": ["evt_success_1"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "intervention_would_help": false }, { "spec_version": "shadow-observation/0.1", "observation_id": "shd_correction_noop", "source_event_ids": ["evt_correction_counterexample"], - "observed_trigger_selectors": ["correction/v1|run codegen before tests"], + "observed_trigger_selectors": ["correction/v2|run codegen before tests"], "intervention_would_help": false } ] diff --git a/evals/fixtures/shadow/command-preflight-pass.json b/evals/fixtures/shadow/command-preflight-pass.json index ed02998..07e03e4 100644 --- a/evals/fixtures/shadow/command-preflight-pass.json +++ b/evals/fixtures/shadow/command-preflight-pass.json @@ -1,26 +1,26 @@ { "spec_version": "shadow-suite/0.1", - "mutation_id": "mut_d6b7a340eb2fb6f18bee4a20932b9c954adb4975f3ea8136bf0bd264b3ec431c", + "mutation_id": "mut_6b51ef819f54c0275db19b15907b0b23c39598241c912828bb64cd5bf824a0ee", "observations": [ { "spec_version": "shadow-observation/0.1", "observation_id": "shd_failure_one", "source_event_ids": ["evt_failure_1"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "intervention_would_help": true }, { "spec_version": "shadow-observation/0.1", "observation_id": "shd_failure_two", "source_event_ids": ["evt_failure_2"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "intervention_would_help": true }, { "spec_version": "shadow-observation/0.1", "observation_id": "shd_failure_three", "source_event_ids": ["evt_failure_3"], - "observed_trigger_selectors": ["failure/v1|shell|mise run check|exit:1"], + "observed_trigger_selectors": ["failure/v2|shell|mise run check|exit:1"], "intervention_would_help": true }, { @@ -34,7 +34,7 @@ "spec_version": "shadow-observation/0.1", "observation_id": "shd_correction_noop", "source_event_ids": ["evt_correction_counterexample"], - "observed_trigger_selectors": ["correction/v1|run codegen before tests"], + "observed_trigger_selectors": ["correction/v2|run codegen before tests"], "intervention_would_help": false } ] diff --git a/scripts/demo-milestone-1.sh b/scripts/demo-milestone-1.sh index bd86bac..15afcc8 100755 --- a/scripts/demo-milestone-1.sh +++ b/scripts/demo-milestone-1.sh @@ -3,7 +3,7 @@ set -eu database="${TMPDIR:-/tmp}/autophagy-milestone-1-$$.db" install_repository="${TMPDIR:-/tmp}/autophagy-install-target-$$" -failure_mutation="mut_d6b7a340eb2fb6f18bee4a20932b9c954adb4975f3ea8136bf0bd264b3ec431c" +failure_mutation="mut_6b51ef819f54c0275db19b15907b0b23c39598241c912828bb64cd5bf824a0ee" replay_draft="${TMPDIR:-/tmp}/autophagy-replay-draft-$$.json" trap 'rm -f "$database" "$database-shm" "$database-wal" "$replay_draft"; rm -rf "$install_repository"' EXIT HUP INT TERM mkdir -p "$install_repository"