diff --git a/artifacts/requirements.yaml b/artifacts/requirements.yaml index 05240e3..2862cfe 100644 --- a/artifacts/requirements.yaml +++ b/artifacts/requirements.yaml @@ -7568,7 +7568,7 @@ artifacts: - id: REQ-238 type: requirement title: graphical req->test-result trace view - status: proposed + status: implemented description: "A graphical/visual way to trace a requirement to its test results. #547. v0.23." provenance: created-by: ai-assisted diff --git a/rivet-cli/src/main.rs b/rivet-cli/src/main.rs index ba4cef0..72960c2 100644 --- a/rivet-cli/src/main.rs +++ b/rivet-cli/src/main.rs @@ -454,6 +454,25 @@ enum Command { id: String, }, + /// Trace a requirement FORWARD to the test results that cover it (#547). + /// Walks backlinks (what verifies it, possibly multi-hop through the ASPICE + /// V) and reports each reached artifact with its latest test-result status, + /// plus a roll-up verdict. Text or JSON — the JSON is what the `rivet serve` + /// graphical trace view consumes. + #[command(name = "trace-results")] + TraceResults { + /// Requirement (or any artifact) ID to trace forward from. + id: String, + + /// Maximum hops to walk (default: 4 — the ASPICE V depth). + #[arg(long, default_value = "4")] + depth: usize, + + /// Output format: "text" (default) or "json". + #[arg(short, long, default_value = "text")] + format: String, + }, + /// Bundle an artifact and its link-graph closure as a single pasteable document Bundle { /// Root artifact ID @@ -2370,6 +2389,7 @@ fn run(cli: Cli) -> Result { // per-artifact traceability view; it renders the same as // `validate --explain ` (which stays as an alias). Command::Trace { id } => cmd_explain(&cli, id), + Command::TraceResults { id, depth, format } => cmd_trace_results(&cli, id, *depth, format), Command::Bundle { id, depth, @@ -7709,6 +7729,69 @@ fn cmd_check_verification_evidence( Ok(missing.is_empty()) } +/// #547 (REQ-238): trace a requirement FORWARD to the test results that cover +/// it — the reverse of the authored `verifies` direction. Text tree or JSON +/// (the JSON is what the `rivet serve` graphical trace view consumes). Exits +/// non-zero only when a covering test recorded a FAILING result, so it is +/// usable as a per-requirement gate. +fn cmd_trace_results(cli: &Cli, id: &str, depth: usize, format: &str) -> Result { + use rivet_core::result_trace::{TraceVerdict, trace_test_results, verdict}; + use rivet_core::results::{ResultStore, TestStatus}; + validate_format(format, &["text", "json"])?; + let ctx = ProjectContext::load_full(cli)?; + ctx.warn_parse_error_skips(cli); + if !ctx.store.contains(id) { + anyhow::bail!("artifact '{id}' not found"); + } + let empty = ResultStore::new(); + let results = ctx.result_store.as_ref().unwrap_or(&empty); + let nodes = trace_test_results(id, &ctx.graph, results, depth); + let v = verdict(&nodes); + let ok = !matches!(v, TraceVerdict::Failing); + + let badge = |s: Option<&TestStatus>| match s { + Some(TestStatus::Pass) => "pass", + Some(TestStatus::Fail) => "FAIL", + Some(TestStatus::Error) => "ERROR", + Some(TestStatus::Skip) => "skip", + Some(TestStatus::Blocked) => "blocked", + None => "·", + }; + + if format == "json" { + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "command": "trace-results", + "root": id, + "verdict": v, + "nodes": nodes, + }))? + ); + } else { + let verdict_str = match v { + TraceVerdict::Passing => "\u{2713} passing", + TraceVerdict::Failing => "\u{2717} failing", + TraceVerdict::NoEvidence => "\u{2014} no test evidence", + }; + println!("Test-result trace for {id}: {verdict_str}"); + if nodes.is_empty() { + println!(" (nothing traces to {id})"); + } + for n in &nodes { + let indent = " ".repeat(n.distance); + println!( + "{indent}{} --{}--> {} [{}]", + n.artifact_id, + n.link_type, + n.via_target, + badge(n.status.as_ref()) + ); + } + } + Ok(ok) +} + /// #559: advance an artifact to `verified` when it has verifying evidence — /// an incoming `verifies` link, OR a `// rivet: verifies ` source marker. /// Opt-in and auditable (no auto-advance); the artifact must be `implemented`. @@ -15271,7 +15354,6 @@ impl ProjectContext { } /// Load project with artifacts, schema, link graph, documents, and test results. - #[allow(dead_code)] fn load_full(cli: &Cli) -> Result { let mut ctx = Self::load_with_docs(cli)?; diff --git a/rivet-cli/tests/cli_commands.rs b/rivet-cli/tests/cli_commands.rs index cf322df..082bd22 100644 --- a/rivet-cli/tests/cli_commands.rs +++ b/rivet-cli/tests/cli_commands.rs @@ -786,6 +786,75 @@ fn check_verification_evidence_flags_missing_named_test() { assert_eq!(missing, vec!["renamed_or_typod_test"]); } +/// #547 (REQ-238): `rivet trace-results ` walks FORWARD from a requirement +/// to the test results that cover it (the reverse of the authored `verifies` +/// direction) and rolls up a pass/fail verdict — the data behind the graphical +/// dashboard trace view. Exits non-zero when a covering test failed. +/// +/// rivet: verifies REQ-238 +#[test] +fn trace_results_forward_from_requirement_to_test_outcome() { + let tmp = tempfile::tempdir().expect("temp dir"); + let dir = tmp.path(); + let dirs = dir.to_str().unwrap(); + std::fs::create_dir_all(dir.join("artifacts")).unwrap(); + std::fs::create_dir_all(dir.join("results")).unwrap(); + std::fs::write( + dir.join("rivet.yaml"), + "project:\n name: p\n schemas: [common, dev]\n\ + sources:\n - path: artifacts\n format: generic-yaml\nresults: results\n", + ) + .unwrap(); + std::fs::write( + dir.join("artifacts/a.yaml"), + "artifacts:\n \ + - id: REQ-1\n type: requirement\n title: r\n status: approved\n \ + - id: TEST-1\n type: test\n title: t\n status: approved\n \ + links:\n - type: verifies\n target: REQ-1\n", + ) + .unwrap(); + let write_result = |status: &str| { + std::fs::write( + dir.join("results/run1.yaml"), + format!( + "run:\n id: run-1\n timestamp: \"2026-07-01T00:00:00Z\"\n\ + results:\n - artifact: TEST-1\n status: {status}\n" + ), + ) + .unwrap(); + }; + + // Passing result → verdict passing, exit 0, TEST-1 reached with status. + write_result("pass"); + let out = Command::new(rivet_bin()) + .args([ + "--project", + dirs, + "trace-results", + "REQ-1", + "--format", + "json", + ]) + .output() + .expect("trace-results"); + assert!(out.status.success()); + let v: serde_json::Value = serde_json::from_slice(&out.stdout).expect("json"); + assert_eq!(v["verdict"], "passing"); + assert_eq!(v["nodes"][0]["artifact_id"], "TEST-1"); + assert_eq!(v["nodes"][0]["status"], "pass"); + + // Failing result → exit non-zero (gate-usable). + write_result("fail"); + let out2 = Command::new(rivet_bin()) + .args(["--project", dirs, "trace-results", "REQ-1"]) + .output() + .expect("trace-results"); + assert!( + !out2.status.success(), + "a failing covering test must make trace-results exit non-zero" + ); +} + /// #620 (REQ-241): `rivet validate` (default salsa path) and /// `rivet validate --direct` (library path) must produce IDENTICAL results /// on the same project. A user reported them disagreeing — one flagging diff --git a/rivet-core/src/lib.rs b/rivet-core/src/lib.rs index b103dad..3e85bc9 100644 --- a/rivet-core/src/lib.rs +++ b/rivet-core/src/lib.rs @@ -74,6 +74,7 @@ pub mod ownership; pub mod query; pub mod remediation; pub mod reqif; +pub mod result_trace; pub mod results; pub mod rivet_version; pub mod runs; diff --git a/rivet-core/src/result_trace.rs b/rivet-core/src/result_trace.rs new file mode 100644 index 0000000..515b989 --- /dev/null +++ b/rivet-core/src/result_trace.rs @@ -0,0 +1,275 @@ +//! Forward req → test-result trace (#547 / REQ-238). +//! +//! The trace graph is authored "upstream": a verification/test artifact carries +//! an outgoing `verifies` link to the requirement it covers, and its pass/fail +//! outcome lives in the [`ResultStore`] keyed by that verification's id. So +//! "given a requirement, what are its test results?" means walking **backlinks** +//! (incoming links) from the requirement — possibly several hops, e.g. the +//! ASPICE chain `sw-req <- sw-detail-design <- unit-verification` — and reading +//! the latest result for each reached artifact. +//! +//! This module computes that trace as plain data; the CLI and the `rivet serve` +//! dashboard render it (the dashboard adds the graphical view). + +// SAFETY-REVIEW (SCRC Phase 1, DD-058): file-scope allow, see sibling modules. +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::arithmetic_side_effects +)] + +use std::collections::{BTreeSet, VecDeque}; + +use crate::links::LinkGraph; +use crate::results::{ResultStore, TestStatus}; + +/// One artifact reached while tracing backwards from a requirement toward its +/// test evidence. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct ResultTraceNode { + /// The reached artifact's id. + pub artifact_id: String, + /// The id one hop closer to the root that this node links to (its parent in + /// the trace tree) — so a consumer can draw edges. `None` for depth-1 nodes + /// whose parent is the root itself is still the root's id. + pub via_target: String, + /// The link type connecting this node to `via_target` (e.g. `verifies`, + /// `satisfies`). + pub link_type: String, + /// Hops from the root requirement (1 = directly links to it). + pub distance: usize, + /// Latest test-result status for this artifact, if the result store has one. + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, +} + +impl ResultTraceNode { + /// True when this node carries an actual test outcome (i.e. it is a leaf + /// verification with a recorded result), as opposed to an intermediate + /// design/architecture artifact on the path. + pub fn has_result(&self) -> bool { + self.status.is_some() + } +} + +/// Trace from a requirement to the test results that cover it. +/// +/// Breadth-first over backlinks from `root`, up to `max_depth` hops (a sane +/// default is 4 — long enough for the ASPICE V, short enough to stay cheap). +/// Every reached artifact is returned once (nearest distance wins), annotated +/// with its latest test-result status if any. The result is sorted by +/// `(distance, artifact_id)` for stable rendering. `root` itself is not +/// included. +pub fn trace_test_results( + root: &str, + graph: &LinkGraph, + results: &ResultStore, + max_depth: usize, +) -> Vec { + let mut seen: BTreeSet = BTreeSet::new(); + seen.insert(root.to_string()); + let mut out: Vec = Vec::new(); + let mut queue: VecDeque<(String, usize)> = VecDeque::new(); + queue.push_back((root.to_string(), 0)); + + while let Some((current, depth)) = queue.pop_front() { + if depth >= max_depth { + continue; + } + // Deterministic order: sort backlinks by (source, link_type). + let mut backlinks: Vec<&crate::links::Backlink> = + graph.backlinks_to(¤t).iter().collect(); + backlinks.sort_by(|a, b| a.source.cmp(&b.source).then(a.link_type.cmp(&b.link_type))); + for bl in backlinks { + if !seen.insert(bl.source.clone()) { + continue; + } + let status = results + .latest_for(&bl.source) + .map(|(_, r)| r.status.clone()); + out.push(ResultTraceNode { + artifact_id: bl.source.clone(), + via_target: current.clone(), + link_type: bl.link_type.clone(), + distance: depth + 1, + status, + }); + queue.push_back((bl.source.clone(), depth + 1)); + } + } + + out.sort_by(|a, b| { + a.distance + .cmp(&b.distance) + .then(a.artifact_id.cmp(&b.artifact_id)) + }); + out +} + +/// Roll up a trace into a one-line verdict for the root requirement: does it +/// have any test evidence, and did all recorded results pass? +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum TraceVerdict { + /// At least one reached verification recorded a result, and every recorded + /// result passed. + Passing, + /// At least one reached verification recorded a failing/erroring result. + Failing, + /// No reached artifact has any recorded test result. + NoEvidence, +} + +/// Summarise a trace (as produced by [`trace_test_results`]). +pub fn verdict(nodes: &[ResultTraceNode]) -> TraceVerdict { + let mut any = false; + let mut all_pass = true; + for n in nodes { + match n.status.as_ref() { + Some(TestStatus::Pass) => any = true, + Some(_) => { + any = true; + all_pass = false; + } + None => {} + } + } + match (any, all_pass) { + (false, _) => TraceVerdict::NoEvidence, + (true, true) => TraceVerdict::Passing, + (true, false) => TraceVerdict::Failing, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{Artifact, Link}; + use crate::results::{RunMetadata, TestResult, TestRun}; + use crate::schema::Schema; + use crate::store::Store; + + fn artifact(id: &str, ty: &str, links: &[(&str, &str)]) -> Artifact { + Artifact { + id: id.to_string(), + artifact_type: ty.to_string(), + links: links.iter().map(|(lt, tgt)| Link::new(*lt, *tgt)).collect(), + ..Default::default() + } + } + + fn run(results: &[(&str, TestStatus)]) -> TestRun { + TestRun { + run: RunMetadata { + id: "r1".into(), + timestamp: "2026-07-01".into(), + source: None, + environment: None, + commit: None, + }, + results: results + .iter() + .map(|(id, st)| TestResult { + artifact: id.to_string(), + status: st.clone(), + duration: None, + message: None, + }) + .collect(), + source_file: None, + } + } + + #[test] + fn traces_multi_hop_aspice_chain_to_results() { + // sw-req <- (satisfies) sw-detail-design <- (verifies) unit-verification + let mut store = Store::new(); + store.insert(artifact("REQ-1", "sw-req", &[])).unwrap(); + store + .insert(artifact( + "DD-1", + "sw-detail-design", + &[("satisfies", "REQ-1")], + )) + .unwrap(); + store + .insert(artifact( + "UV-1", + "unit-verification", + &[("verifies", "DD-1")], + )) + .unwrap(); + let graph = LinkGraph::build(&store, &Schema::merge(&[])); + + let mut results = ResultStore::new(); + results.insert(run(&[("UV-1", TestStatus::Pass)])); + + let trace = trace_test_results("REQ-1", &graph, &results, 4); + // Both DD-1 (depth 1, no result) and UV-1 (depth 2, pass) are reached. + assert_eq!(trace.len(), 2); + assert_eq!(trace[0].artifact_id, "DD-1"); + assert_eq!(trace[0].distance, 1); + assert!(!trace[0].has_result()); + assert_eq!(trace[1].artifact_id, "UV-1"); + assert_eq!(trace[1].distance, 2); + assert_eq!(trace[1].status, Some(TestStatus::Pass)); + assert_eq!(verdict(&trace), TraceVerdict::Passing); + } + + #[test] + fn a_failing_result_makes_the_verdict_failing() { + let mut store = Store::new(); + store.insert(artifact("REQ-1", "requirement", &[])).unwrap(); + store + .insert(artifact("T-1", "test", &[("verifies", "REQ-1")])) + .unwrap(); + store + .insert(artifact("T-2", "test", &[("verifies", "REQ-1")])) + .unwrap(); + let graph = LinkGraph::build(&store, &Schema::merge(&[])); + let mut results = ResultStore::new(); + results.insert(run(&[("T-1", TestStatus::Pass), ("T-2", TestStatus::Fail)])); + + let trace = trace_test_results("REQ-1", &graph, &results, 4); + assert_eq!(trace.len(), 2); + assert_eq!(verdict(&trace), TraceVerdict::Failing); + } + + #[test] + fn no_backlinks_or_no_results_is_no_evidence() { + let mut store = Store::new(); + store.insert(artifact("REQ-1", "requirement", &[])).unwrap(); + store + .insert(artifact( + "DD-1", + "design-decision", + &[("satisfies", "REQ-1")], + )) + .unwrap(); + let graph = LinkGraph::build(&store, &Schema::merge(&[])); + let results = ResultStore::new(); + + let trace = trace_test_results("REQ-1", &graph, &results, 4); + assert_eq!(trace.len(), 1); // DD-1 reached, but no result + assert_eq!(verdict(&trace), TraceVerdict::NoEvidence); + } + + #[test] + fn depth_limit_is_respected() { + let mut store = Store::new(); + store.insert(artifact("REQ-1", "requirement", &[])).unwrap(); + store + .insert(artifact("A", "x", &[("satisfies", "REQ-1")])) + .unwrap(); + store + .insert(artifact("B", "x", &[("satisfies", "A")])) + .unwrap(); + let graph = LinkGraph::build(&store, &Schema::merge(&[])); + let results = ResultStore::new(); + + let trace = trace_test_results("REQ-1", &graph, &results, 1); + assert_eq!(trace.len(), 1); // only A (depth 1); B is at depth 2 + assert_eq!(trace[0].artifact_id, "A"); + } +}