From 3a8fa41db60166173162c5c1631880651d1f4b71 Mon Sep 17 00:00:00 2001 From: Karn Date: Fri, 17 Jul 2026 19:17:20 +0530 Subject: [PATCH] feat: add shadow-draft exporter and smooth the mutation pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the propose → challenge → replay-draft → replay → shadow-draft → shadow → install lifecycle passable without hand-authoring contract JSON. - Add `mutations shadow-draft --suite [--context-events N] [--force]`, mirroring `replay-draft`. Extraction lives in autophagy-shadow's new `extract_observation_draft`, deriving a schema-valid `shadow-suite/0.1` from the mutation's own supporting evidence (drafted `intervention_would_help: true`) and counterexamples (drafted `false`). Observation ids (`shd_…`) are deterministic SHA-256 of inputs — no clock, no randomness — and source event ids are unique across the suite. Read-only: it writes one user-named file and performs no state transition. - Harmonize replay's flag: `--suite` is canonical; `--scenarios` stays as a hidden alias. Shadow gains the same `--suite`/`--observations` pair. - Manifest-free deterministic synthesis: the built-in `deterministic` provider falls back to a built-in reference manifest when no `--manifest`/config path is given. Explicit manifests still win and validate strictly; other providers still require one, and that error now points at a complete example manifest. - Actionable replay error: the unreviewed-scenarios failure now truncates the id list (first 3 + "and N more") and appends the exact remedy plus the suite path to edit. - One-line next-step hints to stderr (text mode only, never JSON stdout) after propose, challenge, replay-draft, replay, shadow-draft, and shadow, using real ids and paths. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 1 + crates/autophagy-cli/src/main.rs | 276 ++++++++++++++++-- crates/autophagy-cli/tests/cli.rs | 247 +++++++++++++++- crates/autophagy-shadow/Cargo.toml | 1 + crates/autophagy-shadow/src/extract.rs | 357 ++++++++++++++++++++++++ crates/autophagy-shadow/src/lib.rs | 2 + crates/autophagy-shadow/tests/shadow.rs | 92 +++++- docs/guides/replay.md | 16 +- docs/guides/shadow-and-installation.md | 33 ++- docs/guides/synthesis.md | 6 +- 10 files changed, 985 insertions(+), 46 deletions(-) create mode 100644 crates/autophagy-shadow/src/extract.rs diff --git a/Cargo.lock b/Cargo.lock index 8ff02f9..9aac4c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,6 +290,7 @@ name = "autophagy-shadow" version = "0.1.0" dependencies = [ "autophagy-core", + "autophagy-events", "autophagy-mutations", "autophagy-patterns", "autophagy-store", diff --git a/crates/autophagy-cli/src/main.rs b/crates/autophagy-cli/src/main.rs index b1f1aa9..a38eadf 100644 --- a/crates/autophagy-cli/src/main.rs +++ b/crates/autophagy-cli/src/main.rs @@ -46,7 +46,8 @@ use autophagy_replay::{ ReplaySuite, evaluate, extract_review_draft, }; use autophagy_shadow::{ - ShadowEvaluationError, ShadowReport, ShadowSuite, evaluate as evaluate_shadow, + ShadowDraftError, ShadowEvaluationError, ShadowReport, ShadowSuite, + evaluate as evaluate_shadow, extract_observation_draft, }; use autophagy_store::{ DeleteAllSummary, DeleteSummary, EventStore, InstallationRegistration, @@ -56,9 +57,10 @@ use autophagy_store::{ ShadowRegisterOutcome, ShadowRegistration, StoreError, }; use autophagy_synthesis::{ - AgentCliProvider, DeterministicReferenceProvider, EndpointLocality, ManifestError, ModelFormat, - ModelManifest, OllamaProvider, OpenAiCompatibleProvider, SynthesisOutcome, SynthesisProvider, - TokenUsage, classify_endpoint, synthesize_candidates, + AgentCliProvider, Capability, DeterministicReferenceProvider, EndpointLocality, ManifestError, + ManifestSpecVersion, ModelFormat, ModelManifest, OllamaProvider, OpenAiCompatibleProvider, + ResourceHints, SynthesisOutcome, SynthesisProvider, TokenUsage, classify_endpoint, + synthesize_candidates, }; use clap::{ArgMatches, CommandFactory, FromArgMatches, Parser, Subcommand, ValueEnum}; use directories::ProjectDirs; @@ -566,9 +568,10 @@ enum MutationAction { /// Stable mutation identity. mutation_id: String, - /// Replay Suite v0.1 JSON file. - #[arg(long, value_name = "PATH")] - scenarios: PathBuf, + /// Annotated Replay Suite v0.1 JSON file (as written by `replay-draft`). + /// The former `--scenarios` name stays accepted as a hidden alias. + #[arg(long, alias = "scenarios", value_name = "PATH")] + suite: PathBuf, }, /// Export an evidence-linked Replay Suite draft for human annotation. ReplayDraft { @@ -587,14 +590,31 @@ enum MutationAction { #[arg(long)] force: bool, }, + /// Export an evidence-linked Shadow Suite draft for human annotation. + ShadowDraft { + /// Stable mutation identity. + mutation_id: String, + + /// Destination for the Shadow Suite v0.1 JSON draft. + #[arg(long, value_name = "PATH")] + suite: PathBuf, + + /// Nearby events retained on either side of each exact evidence event. + #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u8).range(0..=20), value_name = "COUNT")] + context_events: u8, + + /// Replace an existing destination file. + #[arg(long)] + force: bool, + }, /// Measure would-be trigger precision without applying the mutation. Shadow { /// Stable mutation identity. mutation_id: String, - /// Shadow Suite v0.1 JSON file. - #[arg(long, value_name = "PATH")] - observations: PathBuf, + /// Annotated Shadow Suite v0.1 JSON file (as written by `shadow-draft`). + #[arg(long, alias = "observations", value_name = "PATH")] + suite: PathBuf, }, /// Install one shadow-passed mutation as a repo-scoped coding-agent skill. Install { @@ -735,6 +755,8 @@ enum CommandReport { MutationReplay(MutationReplayReport), #[serde(rename = "mutations_replay_draft")] MutationReplayDraft(MutationReplayDraftReport), + #[serde(rename = "mutations_shadow_draft")] + MutationShadowDraft(MutationShadowDraftReport), #[serde(rename = "mutations_shadow")] MutationShadow(MutationShadowReport), #[serde(rename = "mutations_install")] @@ -876,6 +898,16 @@ struct MutationReplayDraftReport { draft: ReplaySuite, } +#[derive(Debug, Serialize)] +struct MutationShadowDraftReport { + path: String, + context_events: u8, + observations: usize, + intervention_observations: usize, + no_op_observations: usize, + draft: ShadowSuite, +} + #[derive(Debug, Serialize)] struct MutationShadowReport { evaluation: ShadowReport, @@ -935,6 +967,7 @@ impl CommandReport { | Self::MutationShow(_) | Self::MutationTransition(_) | Self::MutationReplayDraft(_) + | Self::MutationShadowDraft(_) | Self::MutationInstall(_) | Self::MutationUninstall(_) | Self::Export(_) @@ -948,6 +981,57 @@ impl CommandReport { | Self::Config(_) => false, } } + + /// A single copy-pasteable next command for the mutation lifecycle, printed + /// to stderr in text mode only. Uses real ids and paths from the outcome so + /// the user never has to guess the next impassable step. Returns `None` when + /// there is no unambiguous next action (dry runs, rejections, failed gates). + fn next_step_hint(&self) -> Option { + match self { + Self::MutationProposal(report) => { + let mutation_id = register_outcome_id(report.registrations.first()?); + Some(format!( + "next: autophagy mutations challenge {mutation_id} \ + --check coincidence-considered --check sessions-comparable \ + --check trigger-observable --check legitimate-uses-bounded \ + --check equivalent-searched --check counterexamples-reviewed" + )) + } + Self::MutationTransition(outcome) if outcome.to_state == "challenged" => Some(format!( + "next: autophagy mutations replay-draft {} --suite replay-suite.json", + outcome.mutation_id + )), + Self::MutationReplayDraft(report) => Some(format!( + "next: autophagy mutations replay {} --suite {} \ + (after setting counterfactual_outcome for each intervention scenario)", + report.draft.mutation_id, report.path + )), + Self::MutationReplay(report) if report.evaluation.passed => Some(format!( + "next: autophagy mutations shadow-draft {} --suite shadow-suite.json", + report.evaluation.mutation_id + )), + Self::MutationShadowDraft(report) => Some(format!( + "next: autophagy mutations shadow {} --suite {} \ + (after confirming intervention_would_help for each observation)", + report.draft.mutation_id, report.path + )), + Self::MutationShadow(report) if report.evaluation.passed => Some(format!( + "next: autophagy mutations install {} --repository \ + --target claude-code --confirm-permissions repo-skill-write", + report.evaluation.mutation_id + )), + _ => None, + } + } +} + +/// The stored mutation identity behind any registration outcome. +fn register_outcome_id(outcome: &MutationRegisterOutcome) -> &str { + match outcome { + MutationRegisterOutcome::Inserted { mutation_id } + | MutationRegisterOutcome::Duplicate { mutation_id } + | MutationRegisterOutcome::EquivalentExisting { mutation_id, .. } => mutation_id, + } } #[derive(Debug, thiserror::Error)] @@ -989,6 +1073,8 @@ enum CliError { #[error(transparent)] ReplayDraft(#[from] ReplayDraftError), #[error(transparent)] + ShadowDraft(#[from] ShadowDraftError), + #[error(transparent)] Shadow(#[from] ShadowEvaluationError), #[error(transparent)] Install(#[from] InstallError), @@ -1044,9 +1130,62 @@ enum CliError { #[error("configuration error: {0}")] Config(String), #[error( - "mutations synthesize needs a manifest; pass --manifest or set synthesis.manifest_path in config" + "provider '{provider}' needs a manifest; pass --manifest or set synthesis.manifest_path in config. \ + For a complete example see docs/specs/synthesis/0.3/manifest/valid/claude_cli.json. \ + (The built-in `deterministic` provider needs no manifest.)" )] - MissingManifest, + MissingManifest { provider: &'static str }, + #[error( + "replay suite has {count} unreviewed scenario(s) ({ids}); \ + edit {suite_path} and set counterfactual_outcome to \"expected_result\" or \"contradiction\" for each listed scenario" + )] + UnreviewedReplayScenarios { + count: usize, + ids: String, + suite_path: String, + }, +} + +/// The built-in reference manifest for the offline `deterministic` provider. +/// +/// It mirrors `docs/specs/synthesis/0.1/manifest/valid/deterministic.json` so +/// the model-free reference path needs no hand-written manifest: the provider +/// loads no model and performs no I/O, so these fields are pure descriptive +/// metadata. An explicit `--manifest` still overrides this and is validated +/// strictly. +fn builtin_deterministic_manifest() -> ModelManifest { + ModelManifest { + spec_version: ManifestSpecVersion::V0_1, + name: "deterministic-reference".to_owned(), + format: ModelFormat::Deterministic, + path: "builtin://deterministic".to_owned(), + revision: "v1".to_owned(), + digest: None, + capabilities: vec![Capability::MutationSynthesis], + resource_hints: ResourceHints { + min_memory_mb: 1, + recommended_memory_mb: None, + context_window_tokens: None, + }, + timeouts: None, + api_key_env: None, + model: None, + } +} + +/// Render at most the first three IDs, then summarize the remainder, keeping a +/// long unreviewed list actionable without flooding the terminal. +fn truncate_ids(ids: &[String]) -> String { + const SHOWN: usize = 3; + if ids.len() <= SHOWN { + ids.join(", ") + } else { + format!( + "{}, and {} more", + ids[..SHOWN].join(", "), + ids.len() - SHOWN + ) + } } fn main() -> ExitCode { @@ -1061,6 +1200,13 @@ fn main() -> ExitCode { match dispatch(cli, &matches).and_then(|report| { let has_issues = report.has_issues(); write_report(io::stdout().lock(), output, &report)?; + // Next-step hints guide the user through the mutation lifecycle. They go + // to stderr and only in text mode, so JSON stdout stays a clean report. + if output == OutputFormat::Text { + if let Some(hint) = report.next_step_hint() { + eprintln!("{hint}"); + } + } Ok(has_issues) }) { Ok(true) => ExitCode::from(2), @@ -1587,16 +1733,29 @@ fn execute_mutation_action( _ => provider, } }; + // Resolve the manifest under flag < config. An explicit manifest + // always wins and is validated strictly. When none is given, the + // built-in `deterministic` provider falls back to its reference + // manifest; every other provider still requires one. let manifest = match manifest { - Some(path) => path, - None => config - .synthesis_manifest_path - .as_ref() - .map(PathBuf::from) - .ok_or(CliError::MissingManifest)?, + Some(path) => Some(path), + None => config.synthesis_manifest_path.as_ref().map(PathBuf::from), + }; + let (manifest_path, model_manifest) = match manifest { + Some(path) => { + let manifest_path = path.display().to_string(); + (manifest_path, ModelManifest::from_path(&path)?) + } + None if provider == SynthesisProviderChoice::Deterministic => { + let model_manifest = builtin_deterministic_manifest(); + (model_manifest.path.clone(), model_manifest) + } + None => { + return Err(CliError::MissingManifest { + provider: provider.as_str(), + }); + } }; - let manifest_path = manifest.display().to_string(); - let model_manifest = ModelManifest::from_path(&manifest)?; // The provider choice must match what the manifest declares. if model_manifest.format != provider.required_format() { return Err(CliError::ProviderFormatMismatch { @@ -1744,14 +1903,30 @@ fn execute_mutation_action( } => Ok(CommandReport::MutationTransition( store.reject_mutation(&mutation_id, &reason)?, )), - MutationAction::Replay { - mutation_id, - scenarios, - } => { - let suite: ReplaySuite = serde_json::from_slice(&fs::read(scenarios)?)?; + MutationAction::Replay { mutation_id, suite } => { + let suite_path = suite.to_string_lossy().into_owned(); + let suite: ReplaySuite = serde_json::from_slice(&fs::read(&suite)?)?; let details = store.get_mutation(&mutation_id)?; let package = serde_json::from_value(details.mutation.package)?; - let evaluation = evaluate(&package, &suite)?; + let evaluation = match evaluate(&package, &suite) { + Ok(evaluation) => evaluation, + Err(ReplayEvaluationError::UnreviewedScenarios { .. }) => { + let ids = suite + .scenarios + .iter() + .filter(|scenario| { + scenario.counterfactual_outcome == Some(CounterfactualOutcome::Unknown) + }) + .map(|scenario| scenario.scenario_id.clone()) + .collect::>(); + return Err(CliError::UnreviewedReplayScenarios { + count: ids.len(), + ids: truncate_ids(&ids), + suite_path, + }); + } + Err(other) => return Err(other.into()), + }; for event_id in suite .scenarios .iter() @@ -1823,11 +1998,45 @@ fn execute_mutation_action( }, )) } - MutationAction::Shadow { + MutationAction::ShadowDraft { mutation_id, - observations, + suite, + context_events, + force, } => { - let suite: ShadowSuite = serde_json::from_slice(&fs::read(observations)?)?; + let details = store.get_mutation(&mutation_id)?; + let package = serde_json::from_value(details.mutation.package)?; + let events = store.list_events_for_detection(None)?; + let draft = extract_observation_draft(&package, &events, usize::from(context_events))?; + let intervention_observations = draft + .observations + .iter() + .filter(|observation| observation.intervention_would_help) + .count(); + let no_op_observations = draft.observations.len() - intervention_observations; + let mut options = fs::OpenOptions::new(); + options.write(true); + if force { + options.create(true).truncate(true); + } else { + options.create_new(true); + } + let mut destination = options.open(&suite)?; + serde_json::to_writer_pretty(&mut destination, &draft)?; + writeln!(destination)?; + Ok(CommandReport::MutationShadowDraft( + MutationShadowDraftReport { + path: suite.to_string_lossy().into_owned(), + context_events, + observations: draft.observations.len(), + intervention_observations, + no_op_observations, + draft, + }, + )) + } + MutationAction::Shadow { mutation_id, suite } => { + let suite: ShadowSuite = serde_json::from_slice(&fs::read(&suite)?)?; let details = store.get_mutation(&mutation_id)?; let package = serde_json::from_value(details.mutation.package)?; let evaluation = evaluate_shadow(&package, &suite)?; @@ -2201,6 +2410,15 @@ fn write_report( report.unreviewed_scenarios, report.path )?, + CommandReport::MutationShadowDraft(report) => writeln!( + writer, + "{}\t{} observations · {} intervention · {} no-op\t{}", + report.draft.mutation_id, + report.observations, + report.intervention_observations, + report.no_op_observations, + report.path + )?, CommandReport::MutationShadow(report) => writeln!( writer, "{}\tpassed={}\tprecision={} bps · recall={} bps · {} false positives\tmutation_applied=false", diff --git a/crates/autophagy-cli/tests/cli.rs b/crates/autophagy-cli/tests/cli.rs index fb44b51..0360fd3 100644 --- a/crates/autophagy-cli/tests/cli.rs +++ b/crates/autophagy-cli/tests/cli.rs @@ -327,6 +327,7 @@ fn milestone_demo_digests_exports_deletes_and_prunes_offline() { .count(), 3 ); + // `--scenarios` still works as a hidden alias for the canonical `--suite`. let unreviewed = command(&database) .args([ "mutations", @@ -338,8 +339,25 @@ fn milestone_demo_digests_exports_deletes_and_prunes_offline() { .output() .expect("unreviewed replay"); assert!(!unreviewed.status.success()); + let unreviewed_stderr = String::from_utf8_lossy(&unreviewed.stderr); + // The error names the unreviewed scenarios and the exact remedy, pointing at + // the suite file the user must edit. assert!( - String::from_utf8_lossy(&unreviewed.stderr).contains("unreviewed counterfactual outcomes") + unreviewed_stderr.contains("unreviewed scenario"), + "missing scenario count: {unreviewed_stderr}" + ); + assert!( + unreviewed_stderr.contains("rps_"), + "missing scenario ids: {unreviewed_stderr}" + ); + assert!( + unreviewed_stderr + .contains("set counterfactual_outcome to \"expected_result\" or \"contradiction\""), + "missing remedy: {unreviewed_stderr}" + ); + assert!( + unreviewed_stderr.contains(review_draft.to_str().expect("UTF-8 path")), + "missing suite path: {unreviewed_stderr}" ); let incomplete = command(&database) @@ -1969,6 +1987,233 @@ fn setup_omits_global_notice_when_config_dir_is_isolated() { ); } +#[test] +fn deterministic_synthesis_needs_no_manifest() { + let directory = tempfile::tempdir().expect("temporary directory"); + let database = directory.path().join("autophagy.db"); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../evals/fixtures/findings/deterministic.jsonl"); + run_json(&database, ["import", fixture.to_str().expect("UTF-8 path")]); + + // The built-in `deterministic` provider synthesizes with no --manifest and + // no config manifest path: it falls back to its reference manifest. + let synthesized = run_json(&database, ["mutations", "synthesize"]); + assert_eq!(synthesized["command"], "mutations_synthesize"); + assert_eq!(synthesized["result"]["provider"], "deterministic"); + assert_eq!(synthesized["result"]["model_used"], false); + assert_eq!(synthesized["result"]["network_used"], false); + assert_eq!( + synthesized["result"]["manifest_path"], + "builtin://deterministic" + ); + assert!( + !synthesized["result"]["registrations"] + .as_array() + .expect("registrations") + .is_empty() + ); + + // A non-deterministic provider still requires a manifest, and the error now + // points at a complete example manifest. + let missing = command(&database) + .args(["mutations", "synthesize", "--provider", "ollama"]) + .output() + .expect("missing manifest"); + assert!(!missing.status.success()); + let missing_stderr = String::from_utf8_lossy(&missing.stderr); + assert!( + missing_stderr.contains("needs a manifest"), + "missing manifest hint: {missing_stderr}" + ); + assert!( + missing_stderr.contains("docs/specs/synthesis/0.3/manifest/valid/claude_cli.json"), + "missing example pointer: {missing_stderr}" + ); +} + +#[test] +#[allow(clippy::too_many_lines)] +fn mutation_pipeline_ergonomics_are_smooth_end_to_end() { + let directory = tempfile::tempdir().expect("temporary directory"); + let database = directory.path().join("autophagy.db"); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../evals/fixtures/findings/deterministic.jsonl"); + run_json(&database, ["import", fixture.to_str().expect("UTF-8 path")]); + + // propose (text mode) prints a next-step challenge hint to stderr. + let proposed = command(&database) + .args(["mutations", "propose"]) + .output() + .expect("propose"); + assert!(proposed.status.success()); + let proposed_stderr = String::from_utf8_lossy(&proposed.stderr); + assert!( + proposed_stderr.contains("next: autophagy mutations challenge"), + "missing propose hint: {proposed_stderr}" + ); + + // The same propose under JSON output keeps stdout a clean report and prints + // no hint at all. + let proposed_json = command(&database) + .args(["--output", "json", "mutations", "propose"]) + .output() + .expect("propose json"); + assert!(proposed_json.status.success()); + assert!( + !String::from_utf8_lossy(&proposed_json.stderr).contains("next:"), + "JSON stdout must not carry a next-step hint on stderr" + ); + serde_json::from_slice::(&proposed_json.stdout).expect("clean JSON stdout"); + + let registry = run_json(&database, ["mutations", "list"]); + let failure_id = registry["result"] + .as_array() + .expect("registry") + .iter() + .find(|mutation| mutation["source_detector"] == "repeated_command_failure") + .expect("failure mutation")["mutation_id"] + .as_str() + .expect("mutation ID") + .to_owned(); + + // challenge (text mode) prints a next-step replay-draft hint. + let challenged = command(&database) + .args([ + "mutations", + "challenge", + &failure_id, + "--check", + "coincidence-considered", + "--check", + "sessions-comparable", + "--check", + "trigger-observable", + "--check", + "legitimate-uses-bounded", + "--check", + "equivalent-searched", + "--check", + "counterexamples-reviewed", + ]) + .output() + .expect("challenge"); + assert!(challenged.status.success()); + assert!( + String::from_utf8_lossy(&challenged.stderr) + .contains("next: autophagy mutations replay-draft"), + "missing challenge hint" + ); + + // replay accepts the canonical --suite name; on pass it hints shadow-draft. + let passing_replay = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../evals/fixtures/replay/command-preflight-pass.json"); + let replayed = command(&database) + .args([ + "mutations", + "replay", + &failure_id, + "--suite", + passing_replay.to_str().expect("UTF-8 path"), + ]) + .output() + .expect("replay --suite"); + assert!(replayed.status.success(), "replay --suite must succeed"); + assert!( + String::from_utf8_lossy(&replayed.stderr) + .contains("next: autophagy mutations shadow-draft"), + "missing replay hint" + ); + + // shadow-draft exports a schema-valid, deterministic Shadow Suite draft. + let shadow_suite = directory.path().join("shadow-suite.json"); + let drafted = run_json( + &database, + [ + "mutations", + "shadow-draft", + &failure_id, + "--suite", + shadow_suite.to_str().expect("UTF-8 path"), + "--context-events", + "1", + ], + ); + assert_eq!(drafted["command"], "mutations_shadow_draft"); + assert!( + drafted["result"]["observations"] + .as_u64() + .expect("observation count") + >= 1 + ); + let written_draft: Value = + serde_json::from_slice(&fs::read(&shadow_suite).expect("read shadow draft")) + .expect("shadow draft JSON"); + assert_eq!(written_draft, drafted["result"]["draft"]); + assert_eq!(written_draft["spec_version"], "shadow-suite/0.1"); + assert_eq!(written_draft["mutation_id"], failure_id.as_str()); + assert!( + written_draft["observations"] + .as_array() + .expect("observations") + .iter() + .all(|observation| observation["observation_id"] + .as_str() + .expect("observation id") + .starts_with("shd_")) + ); + + // Re-drafting without --force refuses to clobber the existing file. + let refused = command(&database) + .args([ + "mutations", + "shadow-draft", + &failure_id, + "--suite", + shadow_suite.to_str().expect("UTF-8 path"), + ]) + .output() + .expect("shadow-draft no force"); + assert!( + !refused.status.success(), + "shadow-draft must not overwrite without --force" + ); + + // With --force it overwrites and the content is byte-for-byte deterministic. + let redrafted = run_json( + &database, + [ + "mutations", + "shadow-draft", + &failure_id, + "--suite", + shadow_suite.to_str().expect("UTF-8 path"), + "--context-events", + "1", + "--force", + ], + ); + assert_eq!(redrafted["result"]["draft"], drafted["result"]["draft"]); + + // shadow accepts the canonical --suite name; on pass it hints install. + let passing_shadow = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../evals/fixtures/shadow/command-preflight-pass.json"); + let shadowed = command(&database) + .args([ + "mutations", + "shadow", + &failure_id, + "--suite", + passing_shadow.to_str().expect("UTF-8 path"), + ]) + .output() + .expect("shadow --suite"); + assert!(shadowed.status.success(), "shadow --suite must pass"); + assert!( + String::from_utf8_lossy(&shadowed.stderr).contains("next: autophagy mutations install"), + "missing shadow install hint" + ); +} + fn run_json(database: &Path, args: [&str; N]) -> Value { let output = command(database) .args(["--output", "json"]) diff --git a/crates/autophagy-shadow/Cargo.toml b/crates/autophagy-shadow/Cargo.toml index 88b7aec..b6647c8 100644 --- a/crates/autophagy-shadow/Cargo.toml +++ b/crates/autophagy-shadow/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true publish = false [dependencies] +autophagy-events = { path = "../autophagy-events" } autophagy-mutations = { path = "../autophagy-mutations" } serde.workspace = true serde_json.workspace = true diff --git a/crates/autophagy-shadow/src/extract.rs b/crates/autophagy-shadow/src/extract.rs new file mode 100644 index 0000000..9cfb4a8 --- /dev/null +++ b/crates/autophagy-shadow/src/extract.rs @@ -0,0 +1,357 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt::Write as _, +}; + +use autophagy_events::Event; +use autophagy_mutations::MutationPackage; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::{ShadowObservation, ShadowObservationSpecVersion, ShadowSuite, ShadowSuiteSpecVersion}; + +#[derive(Default)] +struct EvidenceGroup<'a> { + supporting: Vec<&'a Event>, + counterexamples: Vec<&'a Event>, +} + +/// Derive a reviewable Shadow Suite v0.1 draft from exact mutation evidence. +/// +/// Supporting evidence becomes an observation drafted `intervention_would_help: +/// true`; counterexample evidence becomes one drafted `false`. A bounded event +/// window adds nearby structural context, but every drafted annotation is a +/// starting point a reviewer must confirm against the real outcome before the +/// suite is used to measure trigger precision. +/// +/// Each observation's `source_event_ids` are globally unique across the suite: +/// evidence sets are disjoint, sessions partition the events, and a session that +/// carries both classifications draws no nearby context (mirroring the replay +/// review-draft rule), so no event is ever cited by two observations. +/// +/// # Errors +/// Returns an error when evidence is missing, one event has contradictory +/// classifications, or extraction cannot form a valid shadow suite. +pub fn extract_observation_draft( + package: &MutationPackage, + events: &[Event], + context_events: usize, +) -> Result { + let supporting_ids = package + .hypothesis + .supporting_event_ids + .iter() + .map(String::as_str) + .collect::>(); + let counterexample_ids = package + .hypothesis + .counterexample_event_ids + .iter() + .map(String::as_str) + .collect::>(); + let overlapping_ids = supporting_ids + .intersection(&counterexample_ids) + .copied() + .collect::>(); + if !overlapping_ids.is_empty() { + return Err(ShadowDraftError::ConflictingEvidenceIds( + overlapping_ids.join(", "), + )); + } + + let by_id = events + .iter() + .map(|event| (event.event_id.as_str(), event)) + .collect::>(); + let missing = supporting_ids + .iter() + .chain(&counterexample_ids) + .filter(|event_id| !by_id.contains_key(**event_id)) + .copied() + .collect::>(); + if !missing.is_empty() { + return Err(ShadowDraftError::MissingEvidence(missing.join(", "))); + } + + let mut groups: BTreeMap<&str, EvidenceGroup<'_>> = BTreeMap::new(); + for event_id in &supporting_ids { + let event = by_id[event_id]; + groups + .entry(event.session_id.as_str()) + .or_default() + .supporting + .push(event); + } + for event_id in &counterexample_ids { + let event = by_id[event_id]; + groups + .entry(event.session_id.as_str()) + .or_default() + .counterexamples + .push(event); + } + if groups.is_empty() { + return Err(ShadowDraftError::NoEvidence); + } + + let mut session_events: BTreeMap<&str, Vec<&Event>> = BTreeMap::new(); + for event in events { + session_events + .entry(event.session_id.as_str()) + .or_default() + .push(event); + } + for session in session_events.values_mut() { + session.sort_by(|left, right| event_order(left, right)); + } + + let trigger_selectors = package + .triggers + .iter() + .map(|trigger| trigger.selector.clone()) + .collect::>(); + let mut observations = Vec::with_capacity(groups.len()); + for (session_id, group) in groups { + let session = session_events + .get(session_id) + .ok_or_else(|| ShadowDraftError::MissingEvidence(session_id.to_owned()))?; + append_group_observations( + &mut observations, + package, + session_id, + group, + session, + context_events, + &trigger_selectors, + ); + } + observations.sort_by(|left, right| left.observation_id.cmp(&right.observation_id)); + let suite = ShadowSuite { + spec_version: ShadowSuiteSpecVersion::V0_1, + mutation_id: package.mutation_id.clone(), + observations, + }; + suite.validate().map_err(ShadowDraftError::InvalidDraft)?; + Ok(suite) +} + +fn append_group_observations( + observations: &mut Vec, + package: &MutationPackage, + session_id: &str, + group: EvidenceGroup<'_>, + session: &[&Event], + context_events: usize, + trigger_selectors: &BTreeSet, +) { + let mixed_classification = !group.supporting.is_empty() && !group.counterexamples.is_empty(); + let effective_context = if mixed_classification { + 0 + } else { + context_events + }; + if !group.supporting.is_empty() { + observations.push(build_observation( + package, + session_id, + EvidenceGroup { + supporting: group.supporting, + counterexamples: Vec::new(), + }, + session, + effective_context, + trigger_selectors, + )); + } + if !group.counterexamples.is_empty() { + observations.push(build_observation( + package, + session_id, + EvidenceGroup { + supporting: Vec::new(), + counterexamples: group.counterexamples, + }, + session, + effective_context, + trigger_selectors, + )); + } +} + +fn build_observation( + package: &MutationPackage, + session_id: &str, + mut group: EvidenceGroup<'_>, + session: &[&Event], + context_events: usize, + trigger_selectors: &BTreeSet, +) -> ShadowObservation { + group + .supporting + .sort_by(|left, right| event_order(left, right)); + group + .counterexamples + .sort_by(|left, right| event_order(left, right)); + let (evidence, intervention_would_help) = if group.supporting.is_empty() { + (&group.counterexamples, false) + } else { + (&group.supporting, true) + }; + let evidence_ids = evidence + .iter() + .map(|event| event.event_id.as_str()) + .collect::>(); + let selected = context_window(session, &evidence_ids, context_events); + let source_event_ids = selected + .iter() + .map(|event| event.event_id.as_str().to_owned()) + .collect::>(); + let observed_trigger_selectors = if intervention_would_help { + trigger_selectors.iter().cloned().collect() + } else { + evidence + .iter() + .map(|event| event_selector(event)) + .collect::>() + .into_iter() + .collect() + }; + let context_kinds = selected + .iter() + .map(|event| event.kind.as_str()) + .collect::>() + .join(" -> "); + let nearby_count = selected.len().saturating_sub(evidence.len()); + ShadowObservation { + spec_version: ShadowObservationSpecVersion::V0_1, + observation_id: observation_id( + &package.mutation_id, + session_id, + intervention_would_help, + &source_event_ids, + ), + source_event_ids, + observed_trigger_selectors, + intervention_would_help, + note: Some(observation_note( + intervention_would_help, + evidence.len(), + nearby_count, + session_id, + &context_kinds, + )), + } +} + +fn observation_note( + intervention_would_help: bool, + evidence_count: usize, + nearby_count: usize, + session_id: &str, + context_kinds: &str, +) -> String { + if intervention_would_help { + format!( + "Draft observation from {evidence_count} exact supporting event(s) and {nearby_count} nearby event(s) in session {session_id}. Context: {context_kinds}. Confirm the real outcome shows intervention would have helped, or set intervention_would_help to false, before shadow evaluation." + ) + } else { + format!( + "Draft observation from {evidence_count} exact counterexample event(s) and {nearby_count} nearby event(s) in session {session_id}. Context: {context_kinds}. Confirm the real outcome was a legitimate no-op, or set intervention_would_help to true, before shadow evaluation." + ) + } +} + +fn event_order(left: &Event, right: &Event) -> std::cmp::Ordering { + left.timestamp + .cmp(&right.timestamp) + .then_with(|| left.sequence.cmp(&right.sequence)) + .then_with(|| left.event_id.as_str().cmp(right.event_id.as_str())) +} + +fn context_window<'a>( + session: &[&'a Event], + evidence_ids: &BTreeSet<&str>, + radius: usize, +) -> Vec<&'a Event> { + let mut selected = BTreeSet::new(); + for (index, event) in session.iter().enumerate() { + if evidence_ids.contains(event.event_id.as_str()) { + let start = index.saturating_sub(radius); + let end = index.saturating_add(radius).min(session.len() - 1); + selected.extend(start..=end); + } + } + selected.into_iter().map(|index| session[index]).collect() +} + +fn event_selector(event: &Event) -> String { + let mut selector = format!("event/v1|type:{}", event.kind.as_str()); + if let Some(tool) = &event.tool { + write!(selector, "|tool:{}", normalize(&tool.name)).expect("writing to String cannot fail"); + if let Some(exit_code) = tool.exit_code { + write!(selector, "|exit:{exit_code}").expect("writing to String cannot fail"); + } + } + if let Some(classification) = [ + "autophagy.signature", + "correction_signature", + "correction_key", + ] + .iter() + .find_map(|key| event.metadata.get(*key).and_then(Value::as_str)) + { + write!(selector, "|classification:{}", normalize(classification)) + .expect("writing to String cannot fail"); + } + selector +} + +fn normalize(value: &str) -> String { + value + .split_whitespace() + .map(str::to_ascii_lowercase) + .collect::>() + .join(" ") +} + +fn observation_id( + mutation_id: &str, + session_id: &str, + intervention_would_help: bool, + source_event_ids: &[String], +) -> String { + let annotation = if intervention_would_help { + "would_help" + } else { + "no_op" + }; + let digest = Sha256::digest( + format!( + "shadow-draft/v1\0{mutation_id}\0{session_id}\0{annotation}\0{}", + source_event_ids.join("\0") + ) + .as_bytes(), + ); + let mut encoded = String::with_capacity(digest.len() * 2); + for byte in digest { + write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); + } + format!("shd_{encoded}") +} + +/// Error produced while deriving an evidence-linked shadow review draft. +#[derive(Debug, thiserror::Error)] +pub enum ShadowDraftError { + /// The package did not retain any supporting or counterexample evidence. + #[error("mutation package has no evidence to extract")] + NoEvidence, + /// One or more package evidence IDs were absent from the supplied event set. + #[error("mutation evidence is missing from the supplied events: {0}")] + MissingEvidence(String), + /// The same event was classified as both supporting and contradictory. + #[error("mutation evidence IDs have conflicting classifications: {0}")] + ConflictingEvidenceIds(String), + /// The derived suite violated Shadow Suite v0.1 invariants. + #[error("derived shadow draft is invalid: {0}")] + InvalidDraft(crate::ShadowValidationErrors), +} diff --git a/crates/autophagy-shadow/src/lib.rs b/crates/autophagy-shadow/src/lib.rs index 5f45f29..6ea36d3 100644 --- a/crates/autophagy-shadow/src/lib.rs +++ b/crates/autophagy-shadow/src/lib.rs @@ -5,10 +5,12 @@ //! model. mod evaluate; +mod extract; mod model; mod validate; pub use evaluate::{ShadowEvaluationError, evaluate}; +pub use extract::{ShadowDraftError, extract_observation_draft}; pub use model::{ ShadowDisposition, ShadowObservation, ShadowObservationSpecVersion, ShadowPolicy, ShadowReport, ShadowResult, ShadowResultSpecVersion, ShadowSuite, ShadowSuiteSpecVersion, ShadowSummary, diff --git a/crates/autophagy-shadow/tests/shadow.rs b/crates/autophagy-shadow/tests/shadow.rs index 56f4e59..a3da221 100644 --- a/crates/autophagy-shadow/tests/shadow.rs +++ b/crates/autophagy-shadow/tests/shadow.rs @@ -1,11 +1,14 @@ //! Contract and deterministic metric tests for shadow v0.1. -use std::io::Cursor; +use std::{collections::BTreeSet, io::Cursor}; use autophagy_core::{ImportOptions, import_jsonl}; +use autophagy_events::Event; use autophagy_mutations::{GenerationOutcome, MutationPackage, generate_candidates}; use autophagy_patterns::{DetectorConfig, detect}; -use autophagy_shadow::{ShadowDisposition, ShadowSuite, ShadowThresholdFailure, evaluate}; +use autophagy_shadow::{ + ShadowDisposition, ShadowSuite, ShadowThresholdFailure, evaluate, extract_observation_draft, +}; use autophagy_store::EventStore; const CORPUS: &str = include_str!("../../../evals/fixtures/findings/deterministic.jsonl"); @@ -71,6 +74,76 @@ fn suite_schema_and_semantic_independence_are_enforced() { assert!(errors.iter().any(|error| error.code == "duplicate")); } +#[test] +fn evidence_extraction_produces_stable_schema_valid_shadow_draft() { + let (package, events) = command_failure_package_and_events(); + let draft = extract_observation_draft(&package, &events, 1).expect("shadow draft"); + draft.validate().expect("structurally valid draft"); + + // Deterministic across repeated runs and independent of event input order. + assert_eq!( + draft, + extract_observation_draft(&package, &events, 1).expect("stable draft") + ); + let mut reversed = events.clone(); + reversed.reverse(); + assert_eq!( + draft, + extract_observation_draft(&package, &reversed, 1).expect("stable reversed draft") + ); + + // The draft covers both would-help and legitimate no-op annotations. + assert!( + draft + .observations + .iter() + .any(|observation| observation.intervention_would_help) + ); + assert!( + draft + .observations + .iter() + .any(|observation| !observation.intervention_would_help) + ); + + // Every observation carries a stable shd_ identity and a review note. + assert!( + draft + .observations + .iter() + .all(|observation| observation.observation_id.starts_with("shd_")) + ); + assert!( + draft + .observations + .iter() + .all(|observation| observation.note.is_some()) + ); + + // Source event ids are globally unique across observations, as the suite + // contract requires. + let total: usize = draft + .observations + .iter() + .map(|observation| observation.source_event_ids.len()) + .sum(); + let unique = draft + .observations + .iter() + .flat_map(|observation| observation.source_event_ids.iter()) + .collect::>(); + assert_eq!(total, unique.len()); + + // The draft is a legal input to shadow evaluation. + evaluate(&package, &draft).expect("draft evaluates"); + + validate_schema( + "../../../docs/specs/shadow/0.1/suite.schema.json", + include_str!("../../../docs/specs/shadow/0.1/suite.schema.json"), + &serde_json::to_value(&draft).expect("draft JSON"), + ); +} + fn validate_schema(_path: &str, schema: &str, instance: &serde_json::Value) { let schema: serde_json::Value = serde_json::from_str(schema).expect("schema JSON"); let validator = jsonschema::validator_for(&schema).expect("compile schema"); @@ -78,6 +151,10 @@ fn validate_schema(_path: &str, schema: &str, instance: &serde_json::Value) { } fn command_failure_package() -> MutationPackage { + command_failure_package_and_events().0 +} + +fn command_failure_package_and_events() -> (MutationPackage, Vec) { let mut store = EventStore::open_in_memory().expect("store"); import_jsonl( Cursor::new(CORPUS), @@ -85,11 +162,9 @@ fn command_failure_package() -> MutationPackage { &ImportOptions::new("fixture:shadow"), ) .expect("import"); - let findings = detect( - &store.list_events_for_detection(None).expect("events"), - DetectorConfig::default(), - ); - generate_candidates(&findings) + let events = store.list_events_for_detection(None).expect("events"); + let findings = detect(&events, DetectorConfig::default()); + let package = generate_candidates(&findings) .into_iter() .find_map(|outcome| match outcome { GenerationOutcome::Candidate { package } @@ -100,5 +175,6 @@ fn command_failure_package() -> MutationPackage { } _ => None, }) - .expect("command failure package") + .expect("command failure package"); + (package, events) } diff --git a/docs/guides/replay.md b/docs/guides/replay.md index a7f742c..34096e7 100644 --- a/docs/guides/replay.md +++ b/docs/guides/replay.md @@ -25,13 +25,19 @@ omits nearby context that would duplicate event provenance between them. ```sh autophagy mutations replay mut_example \ - --scenarios evals/fixtures/replay/example.json + --suite replay-review.json ``` -The command evaluates the suite, persists the immutable report, and returns -exit code `2` when thresholds do not pass. A failed report leaves the mutation -`challenged`. A passing report creates one audited `challenged -> replay_passed` -transition. Repeating the identical replay is a storage no-op. +The canonical flag is `--suite` — the same name `replay-draft` writes to — so the +draft and its evaluation refer to one file by one name. The former `--scenarios` +name still works as a hidden alias. The command evaluates the suite, persists the +immutable report, and returns exit code `2` when thresholds do not pass. If any +scenario still carries `counterfactual_outcome: unknown`, replay refuses to run +and names the unreviewed scenarios plus the exact remedy: edit the suite file and +set each one to `expected_result` or `contradiction`. A failed report leaves the +mutation `challenged`. A passing report creates one audited +`challenged -> replay_passed` transition. Repeating the identical replay is a +storage no-op. ## Scenario annotations diff --git a/docs/guides/shadow-and-installation.md b/docs/guides/shadow-and-installation.md index 6f76b41..2c09d6d 100644 --- a/docs/guides/shadow-and-installation.md +++ b/docs/guides/shadow-and-installation.md @@ -4,12 +4,41 @@ Shadow is the final measurement gate before a user may install an instruction mutation. It observes where the immutable trigger would fire but never changes agent context or behavior. +## Drafting a Shadow Suite + +You do not have to hand-author the Shadow Suite JSON. `mutations shadow-draft` +exports an evidence-linked draft from the mutation's own supporting evidence and +counterexamples, mirroring `replay-draft`: + +```sh +autophagy mutations shadow-draft mut_example \ + --suite shadow-review.json \ + --context-events 1 +``` + +Supporting evidence becomes an observation drafted `intervention_would_help: +true`; counterexample evidence becomes one drafted `false`. Each observation +carries a human-readable `note` describing what to confirm against the real +outcome, the exact source event IDs, and the observed trigger selectors. +Observation IDs (`shd_…`) are derived deterministically from their inputs — no +clock, no randomness — so re-running produces byte-for-byte identical output. +Every observation's source event IDs are unique across the suite, as the +contract requires. Drafting only reads the database and writes the one file you +name; it performs no state transition. `--force` overwrites an existing file. + +Review each drafted `intervention_would_help` against the real session outcome +before evaluating; the draft is a starting point, not a verdict. + +## Evaluating a Shadow Suite + ```sh autophagy mutations shadow mut_example \ - --observations evals/fixtures/shadow/example.json + --suite shadow-review.json ``` -Each Shadow Suite v0.1 observation cites independent local AEP event IDs, +The canonical flag is `--suite`; the former `--observations` name still works as +a hidden alias. Each Shadow Suite v0.1 observation cites independent local AEP +event IDs, records selectors visible before the outcome, and annotates whether intervention would have helped. Exact selector matching produces true positives, true negatives, false positives, and false negatives. Passing requires five diff --git a/docs/guides/synthesis.md b/docs/guides/synthesis.md index b5ee1c2..69e1f49 100644 --- a/docs/guides/synthesis.md +++ b/docs/guides/synthesis.md @@ -14,7 +14,11 @@ authenticated coding-agent CLI — against `claude` or `codex` with no API key a no server to run. ```sh -# Offline, no model — the default path. +# Offline, no model — the default path. The built-in deterministic provider +# needs no manifest; it falls back to a built-in reference manifest. +autophagy mutations synthesize + +# Passing an explicit --manifest still works and is validated strictly. autophagy mutations synthesize --provider deterministic \ --manifest docs/specs/synthesis/0.1/manifest/valid/deterministic.json