Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 114 additions & 7 deletions crates/autophagy-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ mod status;
mod watch;

use std::{
collections::BTreeSet,
collections::{BTreeMap, BTreeSet},
fmt::Write as _,
fs::{self, File},
io::{self, BufRead, BufReader, Write},
Expand Down Expand Up @@ -870,6 +870,20 @@ struct MutationProposalReport {
dry_run: bool,
generated: Vec<GenerationOutcome>,
registrations: Vec<MutationRegisterOutcome>,
/// Non-fatal registration notes, currently only immutable-package template
/// conflicts (see [`template_conflict_warning`]). Skipped from JSON when
/// empty so the machine surface is unchanged for conflict-free runs.
#[serde(skip_serializing_if = "Vec::is_empty")]
warnings: Vec<String>,
/// Each generated candidate's CURRENT registry lifecycle state, keyed by
/// mutation ID, fetched from the store after the registration attempt
/// above. Deterministic generation re-derives the same mutation ID for
/// evidence that was already registered, so a candidate can already be
/// `shadow_passed`, `retired`, etc.; this map is the source of truth the
/// text and JSON renderers use instead of assuming every row is still
/// `candidate`. Absent entries mean the mutation has never been
/// registered (dry-run preview of a brand-new candidate).
current_states: BTreeMap<String, String>,
}

#[derive(Debug, Serialize)]
Expand All @@ -890,6 +904,10 @@ struct MutationSynthesisReport {
total_completion_tokens: Option<u64>,
synthesized: Vec<SynthesisOutcome>,
registrations: Vec<MutationRegisterOutcome>,
/// See [`MutationProposalReport::current_states`]: each synthesized
/// candidate's CURRENT registry lifecycle state, fetched after the
/// registration attempt above, keyed by mutation ID.
current_states: BTreeMap<String, String>,
}

#[derive(Debug, Serialize)]
Expand Down Expand Up @@ -1961,6 +1979,7 @@ fn execute_mutation_action(
.findings;
let generated = generate_candidates(&findings);
let mut registrations = Vec::new();
let mut warnings = Vec::new();
if !dry_run {
for outcome in &generated {
let GenerationOutcome::Candidate { package } = outcome else {
Expand All @@ -1980,13 +1999,28 @@ fn execute_mutation_action(
.counterexample_event_ids
.clone(),
};
registrations.push(store.register_mutation(&registration)?);
match store.register_mutation(&registration) {
Ok(outcome) => registrations.push(outcome),
Err(StoreError::MutationContentConflict { mutation_id }) => {
warnings.push(template_conflict_warning(&mutation_id));
}
Err(error) => return Err(error.into()),
}
}
}
let current_states = mutation_states_by_id(
&store,
generated.iter().filter_map(|outcome| match outcome {
GenerationOutcome::Candidate { package } => Some(package.mutation_id.as_str()),
GenerationOutcome::InsufficientEvidence { .. } => None,
}),
)?;
Ok(CommandReport::MutationProposal(MutationProposalReport {
dry_run,
generated,
registrations,
warnings,
current_states,
}))
}
MutationAction::Synthesize {
Expand Down Expand Up @@ -2118,9 +2152,24 @@ fn execute_mutation_action(
.counterexample_event_ids
.clone(),
};
registrations.push(store.register_mutation(&registration)?);
match store.register_mutation(&registration) {
Ok(outcome) => registrations.push(outcome),
Err(StoreError::MutationContentConflict { mutation_id }) => {
warnings.push(template_conflict_warning(&mutation_id));
}
Err(error) => return Err(error.into()),
}
}
}
let current_states = mutation_states_by_id(
&store,
synthesized.iter().filter_map(|outcome| match outcome {
SynthesisOutcome::Candidate { package, .. } => {
Some(package.mutation_id.as_str())
}
_ => None,
}),
)?;
Ok(CommandReport::MutationSynthesis(MutationSynthesisReport {
dry_run,
provider: synthesis_provider.name().to_owned(),
Expand All @@ -2134,6 +2183,7 @@ fn execute_mutation_action(
total_completion_tokens,
synthesized,
registrations,
current_states,
}))
}
MutationAction::List => Ok(CommandReport::MutationList(store.list_mutations()?)),
Expand Down Expand Up @@ -3121,21 +3171,50 @@ fn write_mutation_lesson(writer: &mut impl Write, details: &MutationDetails) ->
Ok(())
}

/// Look up a mutation's current display state, falling back to `candidate`
/// only when the ID has no stored state yet (never registered — a dry-run
/// preview of brand-new evidence, which will in fact become `candidate` the
/// moment it is registered).
fn display_state<'a>(
current_states: &'a BTreeMap<String, String>,
mutation_id: &'a str,
) -> &'a str {
current_states
.get(mutation_id)
.map_or("candidate", String::as_str)
}

/// Warning for a candidate whose evidence re-derived an already-registered
/// mutation ID with different package content. Registered packages are
/// immutable, so the stored package stays authoritative; this happens when the
/// deterministic template evolved between the original registration and this
/// run, and it must not abort the rest of the batch.
fn template_conflict_warning(mutation_id: &str) -> String {
format!(
"mutation '{mutation_id}' is already registered with earlier-template content; \
the stored immutable package is kept"
)
}

fn write_mutation_proposal(
writer: &mut impl Write,
report: &MutationProposalReport,
) -> io::Result<()> {
for warning in &report.warnings {
writeln!(writer, "warning: {warning}")?;
}
if report.generated.is_empty() {
writeln!(writer, "no mutation candidates above evidence threshold")?;
}
for outcome in &report.generated {
match outcome {
GenerationOutcome::Candidate { package } => writeln!(
writer,
"{}\t{}\t{} evidence\tzero permissions\tcandidate",
"{}\t{}\t{} evidence\tzero permissions\t{}",
package.mutation_id,
package.title,
package.hypothesis.supporting_event_ids.len()
package.hypothesis.supporting_event_ids.len(),
display_state(&report.current_states, &package.mutation_id),
)?,
GenerationOutcome::InsufficientEvidence { finding_id, reason } => {
writeln!(writer, "{finding_id}\tinsufficient evidence\t{reason}")?;
Expand All @@ -3150,6 +3229,33 @@ fn write_mutation_proposal(
Ok(())
}

/// Fetch each mutation's CURRENT registry lifecycle state, keyed by mutation
/// ID, for every ID a `propose`/`synthesize` pass generated a candidate for.
///
/// Generation is deterministic: the same finding always re-derives the same
/// mutation ID, so re-running `propose`/`synthesize` over evidence that was
/// registered in an earlier pass reproduces an identical `GenerationOutcome::
/// Candidate`/`SynthesisOutcome::Candidate` even after that mutation has been
/// challenged, replayed, shadow-evaluated, promoted, or retired. Looking the
/// state up fresh from the store — rather than assuming every generated row
/// is still `candidate` — is what keeps the displayed state honest. IDs never
/// registered (a dry-run preview of brand-new evidence) are simply absent.
///
/// # Errors
/// Returns [`CliError`] when the store cannot be queried.
fn mutation_states_by_id<'a>(
store: &EventStore,
mutation_ids: impl Iterator<Item = &'a str>,
) -> Result<BTreeMap<String, String>, CliError> {
let mut states = BTreeMap::new();
for mutation_id in mutation_ids {
if let Some(state) = store.mutation_state(mutation_id)? {
states.insert(mutation_id.to_owned(), state);
}
}
Ok(states)
}

fn aggregate_usage(outcomes: &[SynthesisOutcome]) -> (Option<u64>, Option<u64>) {
let mut prompt: Option<u64> = None;
let mut completion: Option<u64> = None;
Expand Down Expand Up @@ -3207,10 +3313,11 @@ fn write_mutation_synthesis(
match outcome {
SynthesisOutcome::Candidate { package, .. } => writeln!(
writer,
"{}\t{}\t{} evidence\tzero permissions\tcandidate",
"{}\t{}\t{} evidence\tzero permissions\t{}",
package.mutation_id,
package.title,
package.hypothesis.supporting_event_ids.len()
package.hypothesis.supporting_event_ids.len(),
display_state(&report.current_states, &package.mutation_id),
)?,
SynthesisOutcome::InsufficientEvidence { finding_id, reason } => {
writeln!(writer, "{finding_id}\tinsufficient evidence\t{reason}")?;
Expand Down
175 changes: 175 additions & 0 deletions crates/autophagy-cli/tests/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -726,6 +726,98 @@ fn milestone_demo_digests_exports_deletes_and_prunes_offline() {
);
}

/// `propose`/`synthesize` regenerate the exact same deterministic candidate
/// for evidence that was already registered in an earlier pass. Once that
/// mutation moves past `candidate` (rejected, shadow-evaluated, retired, ...)
/// re-running `propose`/`synthesize` must display its ACTUAL current state,
/// never the stale, generation-time `candidate` label.
#[test]
fn propose_and_synthesize_display_current_mutation_state_not_stale_candidate() {
let directory = tempfile::tempdir().expect("temporary directory");
let database = directory.path().join("autophagy.db");
let fixture = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../evals/fixtures/findings/deterministic.jsonl");
run_json(&database, ["import", fixture.to_str().expect("UTF-8 path")]);

let proposed = run_json(&database, ["mutations", "propose"]);
let generated = proposed["result"]["generated"]
.as_array()
.expect("generated");
assert_eq!(generated.len(), 2);
let mutation_id = generated[0]["package"]["mutation_id"]
.as_str()
.expect("mutation id")
.to_owned();
// Freshly generated and just registered: both fixture rows are still
// literally `candidate`.
assert_eq!(
proposed["result"]["current_states"][mutation_id.as_str()],
"candidate"
);

// Move the mutation past `candidate` with an auditable rejection.
let rejected = command(&database)
.args([
"mutations",
"reject",
&mutation_id,
"--reason",
"superseded by manual review",
])
.output()
.expect("reject");
assert!(
rejected.status.success(),
"reject failed: {}",
String::from_utf8_lossy(&rejected.stderr)
);

let registry = run_json(&database, ["mutations", "list"]);
let stored_state = registry["result"]
.as_array()
.expect("registry")
.iter()
.find(|mutation| mutation["mutation_id"] == mutation_id)
.expect("mutation still registered")["state"]
.clone();
assert_eq!(stored_state, "rejected");

// Re-running `propose` deterministically re-derives the identical
// candidate for the same evidence (same mutation ID, same content), so
// registration collapses to a no-op `duplicate` outcome. The JSON
// `current_states` map must reflect the mutation's real state, not the
// package's generation-time classification.
let reproposed = run_json(&database, ["mutations", "propose"]);
assert_eq!(
reproposed["result"]["current_states"][mutation_id.as_str()],
"rejected"
);

// `mutations synthesize` (deterministic provider) must show the same fix.
let synthesized = run_json(&database, ["mutations", "synthesize"]);
assert_eq!(
synthesized["result"]["current_states"][mutation_id.as_str()],
"rejected"
);

// The human-readable text row must print the real state too — never the
// hardcoded `candidate` literal the audit found.
let text_output = command(&database)
.args(["mutations", "propose"])
.output()
.expect("run propose text");
assert!(text_output.status.success());
let stdout = String::from_utf8_lossy(&text_output.stdout);
let row = stdout
.lines()
.find(|line| line.starts_with(&mutation_id))
.unwrap_or_else(|| panic!("no row for {mutation_id} in:\n{stdout}"));
assert!(
row.ends_with("\trejected"),
"row must show the actual current state, not a stale 'candidate': {row}"
);
}

#[test]
fn recovery_motif_is_detected_and_registered_end_to_end() {
let directory = tempfile::tempdir().expect("temporary directory");
Expand Down Expand Up @@ -2542,3 +2634,86 @@ fn command(database: &Path) -> Command {
}
command
}

#[test]
fn propose_and_synthesize_survive_immutable_template_conflicts() {
let directory = tempfile::tempdir().expect("temporary directory");
let database = directory.path().join("autophagy.db");
let fixture = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../evals/fixtures/findings/deterministic.jsonl");
run_json(&database, ["import", fixture.to_str().expect("UTF-8 path")]);

// Preview the exact candidate generation would register today.
let preview = run_json(&database, ["mutations", "propose", "--dry-run"]);
let mut package: autophagy_mutations::MutationPackage =
serde_json::from_value(preview["result"]["generated"][0]["package"].clone())
.expect("generated package");
let mutation_id = package.mutation_id.clone();

// Register it as an earlier template revision would have: identical
// identity and evidence, different exclusion phrasing. The registry is
// immutable, so a later propose regenerating today's phrasing cannot
// overwrite it — and must not abort the batch over it either.
package.exclusions[0] = autophagy_mutations::LEGACY_ADVISORY_UNTIL_REPLAY_EXCLUSION.to_owned();
let mut store = autophagy_store::EventStore::open(&database).expect("store");
store
.register_mutation(&autophagy_store::MutationRegistration {
mutation_id: package.mutation_id.clone(),
source_finding_id: package.source_finding_id.clone(),
source_detector: package.source_detector.as_str().to_owned(),
equivalence_key: autophagy_mutations::equivalence_key(&package),
spec_version: package.spec_version.as_str().to_owned(),
semantic_version: package.version.clone(),
package: serde_json::to_value(&package).expect("package JSON"),
supporting_event_ids: package.hypothesis.supporting_event_ids.clone(),
counterexample_event_ids: package.hypothesis.counterexample_event_ids.clone(),
})
.expect("register earlier-template package");
drop(store);

// Text mode: the run succeeds, warns about the one conflict, and still
// lists every candidate with its real stored state.
let output = command(&database)
.args(["mutations", "propose"])
.output()
.expect("propose");
assert!(
output.status.success(),
"propose failed: {}",
String::from_utf8_lossy(&output.stderr)
);
let stdout = String::from_utf8_lossy(&output.stdout);
assert!(
stdout.contains("earlier-template content"),
"missing conflict warning: {stdout}"
);

// JSON mode: the warning is machine-visible and the conflicted mutation
// still reports its current stored state.
let proposed = run_json(&database, ["mutations", "propose"]);
let warnings = proposed["result"]["warnings"].as_array().expect("warnings");
assert!(
warnings.iter().any(|warning| warning
.as_str()
.expect("warning text")
.contains(&mutation_id)),
"warnings missing conflicted id: {warnings:?}"
);
assert_eq!(
proposed["result"]["current_states"][mutation_id.as_str()],
"candidate"
);

// The deterministic synthesize path takes the same non-fatal branch.
let synthesized = run_json(&database, ["mutations", "synthesize"]);
assert!(
synthesized["result"]["warnings"]
.as_array()
.expect("synthesis warnings")
.iter()
.any(|warning| warning
.as_str()
.expect("warning text")
.contains(&mutation_id))
);
}
Loading
Loading