feat(check): rivet check verification-evidence — named-test-exists oracle (REQ-236, #556)

artifacts/requirements.yaml

@@ -7546,7 +7546,7 @@ artifacts:
   - id: REQ-236
     type: requirement
     title: cited-source on verification types + native named-test-exists check
-    status: proposed
+    status: verified
     description: "Declare cited-source on sw/unit/sys-verification types and add a native named-test-exists check so requirement->test evidence is drift-checked, not just asserted. #556. v0.23 centerpiece."
     provenance:
       created-by: ai-assisted

rivet-cli/src/main.rs

@@ -1996,6 +1996,23 @@ enum CheckAction {
         format: String,
     },
 
+    /// Check that a verification artifact's named-test steps reference tests
+    /// that actually exist (#556 / REQ-236). `cargo test <filter>` exits 0 with
+    /// "0 passed" when the filter matches nothing, so a renamed/typo'd test name
+    /// silently keeps a requirement `verified`. For each `fields.steps[].run`
+    /// that names a cargo test filter, this asserts a matching test exists in
+    /// the scanned Rust sources. Exits non-zero on any missing test.
+    VerificationEvidence {
+        /// Directories to scan for Rust test sources (default: workspace-aware
+        /// src/ + tests/, same as `rivet verify`).
+        #[arg(long = "scan")]
+        scan: Vec<std::path::PathBuf>,
+
+        /// Output format: "text" (default) or "json".
+        #[arg(short, long, default_value = "text")]
+        format: String,
+    },
+
     /// List artifacts with `cited-source` and the current hash status
     /// (match / drift / missing-hash / read-error / skipped-remote / stale).
     /// Phase 1 only handles `kind: file` — see
@@ -2686,6 +2703,9 @@ fn run(cli: Cli) -> Result<bool> {
             CheckAction::GapsJson { baseline, format } => {
                 cmd_check_gaps_json(&cli, baseline.as_deref(), format)
             }
+            CheckAction::VerificationEvidence { scan, format } => {
+                cmd_check_verification_evidence(&cli, scan, format)
+            }
             CheckAction::Sources {
                 update,
                 apply,
@@ -7565,6 +7585,130 @@ fn default_marker_scan_paths(project: &std::path::Path) -> Vec<std::path::PathBu
     paths
 }
 
+/// Recursively collect Rust `fn` names from every `.rs` file under `dir`,
+/// skipping `target/`, `node_modules/`, and dot-dirs (same exclusions as the
+/// marker scanner). Used by `rivet check verification-evidence`.
+fn collect_rust_fn_names(dir: &std::path::Path, out: &mut std::collections::BTreeSet<String>) {
+    let entries = match std::fs::read_dir(dir) {
+        Ok(e) => e,
+        Err(_) => return,
+    };
+    for entry in entries.flatten() {
+        let path = entry.path();
+        if path.is_dir() {
+            if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
+                if name.starts_with('.') || name == "target" || name == "node_modules" {
+                    continue;
+                }
+            }
+            collect_rust_fn_names(&path, out);
+        } else if path.extension().and_then(|e| e.to_str()) == Some("rs") {
+            if let Ok(src) = std::fs::read_to_string(&path) {
+                out.extend(rivet_core::verification_evidence::extract_rust_fn_names(
+                    &src,
+                ));
+            }
+        }
+    }
+}
+
+/// #556 (REQ-236 pt2): assert that a verification artifact's named-test steps
+/// (`fields.steps[].run: "cargo test … <filter>"`) reference tests that
+/// actually exist in the scanned Rust sources — catching the silent-drift case
+/// where `cargo test <typo>` exits 0 with "0 passed" and keeps the requirement
+/// falsely `verified`.
+fn cmd_check_verification_evidence(
+    cli: &Cli,
+    scan: &[std::path::PathBuf],
+    format: &str,
+) -> Result<bool> {
+    use rivet_core::verification_evidence as ve;
+    validate_format(format, &["text", "json"])?;
+    let ctx = ProjectContext::load(cli)?;
+    ctx.warn_parse_error_skips(cli);
+
+    let scan_paths: Vec<std::path::PathBuf> = if scan.is_empty() {
+        default_marker_scan_paths(&cli.project)
+    } else {
+        scan.iter()
+            .map(|p| {
+                if p.is_absolute() {
+                    p.clone()
+                } else {
+                    cli.project.join(p)
+                }
+            })
+            .collect()
+    };
+    let mut fn_names: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
+    for p in &scan_paths {
+        collect_rust_fn_names(p, &mut fn_names);
+    }
+
+    // Walk every artifact's `steps` for `run` commands naming a cargo filter.
+    #[derive(serde::Serialize)]
+    struct Missing {
+        artifact: String,
+        filter: String,
+        command: String,
+    }
+    let mut missing: Vec<Missing> = Vec::new();
+    let mut checked = 0usize;
+    let mut sorted: Vec<&rivet_core::model::Artifact> = ctx.store.iter().collect();
+    sorted.sort_by(|a, b| a.id.cmp(&b.id));
+    for a in sorted {
+        let Some(steps) = a.fields.get("steps").and_then(|v| v.as_sequence()) else {
+            continue;
+        };
+        for step in steps {
+            let Some(run) = step.get("run").and_then(|v| v.as_str()) else {
+                continue;
+            };
+            let Some(filter) = ve::parse_cargo_test_filter(run) else {
+                continue;
+            };
+            checked += 1;
+            if !ve::filter_matches_any(&filter, &fn_names) {
+                missing.push(Missing {
+                    artifact: a.id.clone(),
+                    filter,
+                    command: run.to_string(),
+                });
+            }
+        }
+    }
+
+    if format == "json" {
+        let obj = serde_json::json!({
+            "command": "check verification-evidence",
+            "named_test_steps_checked": checked,
+            "missing": missing,
+            "ok": missing.is_empty(),
+        });
+        println!("{}", serde_json::to_string_pretty(&obj)?);
+    } else if missing.is_empty() {
+        println!(
+            "\u{2713} verification-evidence: {checked} named-test step(s) all reference an existing test."
+        );
+    } else {
+        println!(
+            "\u{2717} verification-evidence: {} named-test step(s) reference a test that does not exist:",
+            missing.len()
+        );
+        for m in &missing {
+            println!(
+                "  {} — no test matching `{}` found (from `{}`)",
+                m.artifact, m.filter, m.command
+            );
+        }
+        println!(
+            "\n  A `cargo test <filter>` that matches nothing exits 0 with \"0 passed\", so this\n  \
+             would otherwise keep the requirement silently `verified`. Fix the filter or the test name."
+        );
+    }
+    Ok(missing.is_empty())
+}
+
 /// #559: advance an artifact to `verified` when it has verifying evidence —
 /// an incoming `verifies` link, OR a `// rivet: verifies <ID>` source marker.
 /// Opt-in and auditable (no auto-advance); the artifact must be `implemented`.

rivet-cli/tests/cli_commands.rs

@@ -723,6 +723,69 @@ fn lifecycle_gap_names_the_aspice_verification_chain() {
     );
 }
 
+/// #556 (REQ-236): `rivet check verification-evidence` flags a verification
+/// step whose `cargo test <filter>` names a test that does not exist — the
+/// silent-drift case (`cargo test typo` exits 0 with "0 passed", keeping the
+/// requirement falsely `verified`). A step naming a real test passes; a
+/// non-cargo step is ignored.
+///
+/// rivet: verifies REQ-236
+#[test]
+fn check_verification_evidence_flags_missing_named_test() {
+    let tmp = tempfile::tempdir().expect("temp dir");
+    let dir = tmp.path();
+    let dirs = dir.to_str().unwrap();
+    std::fs::create_dir_all(dir.join("artifacts")).unwrap();
+    std::fs::create_dir_all(dir.join("src")).unwrap();
+    std::fs::write(
+        dir.join("rivet.yaml"),
+        "project:\n  name: p\n  schemas: [common, dev]\n\
+         sources:\n  - path: artifacts\n    format: generic-yaml\n",
+    )
+    .unwrap();
+    std::fs::write(
+        dir.join("src/lib.rs"),
+        "#[test]\nfn real_relocation_test() { assert!(true); }\n",
+    )
+    .unwrap();
+    std::fs::write(
+        dir.join("artifacts/a.yaml"),
+        "artifacts:\n  \
+         - id: FV-001\n    type: requirement\n    title: v\n    status: implemented\n    \
+             fields:\n      steps:\n        \
+             - run: \"cargo test -p p real_relocation_test\"\n        \
+             - run: \"cargo test -p p renamed_or_typod_test\"\n        \
+             - run: \"make lint\"\n",
+    )
+    .unwrap();
+
+    let out = Command::new(rivet_bin())
+        .args([
+            "--project",
+            dirs,
+            "check",
+            "verification-evidence",
+            "--format",
+            "json",
+        ])
+        .output()
+        .expect("check");
+    assert!(
+        !out.status.success(),
+        "must exit non-zero when a named test is missing"
+    );
+    let v: serde_json::Value = serde_json::from_slice(&out.stdout).expect("json");
+    // 2 cargo-test steps checked (the `make lint` step is ignored).
+    assert_eq!(v["named_test_steps_checked"], 2);
+    let missing: Vec<&str> = v["missing"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|m| m["filter"].as_str().unwrap())
+        .collect();
+    assert_eq!(missing, vec!["renamed_or_typod_test"]);
+}
+
 /// #620 (REQ-241): `rivet validate` (default salsa path) and
 /// `rivet validate --direct` (library path) must produce IDENTICAL results
 /// on the same project. A user reported them disagreeing — one flagging

rivet-core/src/lib.rs

@@ -89,6 +89,7 @@ pub mod templates;
 pub mod test_scanner;
 pub mod validate;
 pub mod variant_emit;
+pub mod verification_evidence;
 pub mod yaml_cst;
 pub mod yaml_edit;
 pub mod yaml_hir;