diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index b3a9384b..cc414084 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -385,3 +385,8 @@ jobs: - name: Harness unit tests run: python3 tests/smoke/test_harness.py -v + + - name: Hero-drift tool exit-code tests + run: | + pip install numpy==2.2.6 Pillow==11.2.1 + python3 tests/test_measure_hero_drift.py -v diff --git a/scripts/measure_hero_drift.py b/scripts/measure_hero_drift.py index 785516a7..7554f037 100644 --- a/scripts/measure_hero_drift.py +++ b/scripts/measure_hero_drift.py @@ -26,8 +26,9 @@ python scripts/measure_hero_drift.py --blender PATH [--only NAME ...] [--out DIR] [--json FILE] -Exit codes: 0 measured (drift is reported, not judged), 2 usage, 3 a render -failed for at least one entry. +Exit codes: 0 measured (drift is reported, not judged), 2 usage (unknown +--only name, nothing measured, unreadable Blender version), 3 a render failed or +a fresh render's size differs from the committed hero (a size change is drift). """ from __future__ import annotations @@ -76,6 +77,8 @@ def render_args(entry: dict, png: Path) -> list[str] | None: continue args = shlex.split(m.group(1)) i = args.index("--output") + if i + 1 >= len(args): + return None # bare trailing --output: no path to retarget args[i + 1] = str(png) return args return None @@ -124,17 +127,34 @@ def main(argv=None) -> int: p.add_argument("--json", default=None, help="also write the results as JSON here") args = p.parse_args(argv) + all_entries = entries() + unknown = sorted(set(args.only or []) - {e["name"] for e in all_entries}) + if unknown: + print(f"error: --only names match no example or showcase piece: {', '.join(unknown)}", + file=sys.stderr) + return 2 + blender = str(Path(args.blender).resolve()) out_dir = Path(args.out) out_dir.mkdir(parents=True, exist_ok=True) - version = subprocess.run([blender, "--version"], capture_output=True, text=True, - encoding="utf-8", errors="replace").stdout.splitlines()[0] + try: + probe = subprocess.run([blender, "--version"], capture_output=True, text=True, + encoding="utf-8", errors="replace") + except OSError as exc: + print(f"error: cannot run Blender at {blender}: {exc}", file=sys.stderr) + return 2 + lines = probe.stdout.splitlines() + if probe.returncode != 0 or not lines: + print(f"error: `{blender} --version` failed (exit {probe.returncode}) or printed nothing", + file=sys.stderr) + return 2 + version = lines[0] print(f"# renderer: {blender}\n# version: {version}", flush=True) print(f"{'name':32} {'mean_abs':>9} {'gt2pct':>8} {'vs_q90':>8} {'luma c/f':>17} {'secs':>6} verdict", flush=True) results, failed = [], [] - for e in entries(): + for e in all_entries: name = e["name"] if args.only and name not in args.only: continue @@ -162,6 +182,7 @@ def main(argv=None) -> int: results.append(row) if "mean_abs" not in row: print(f"{name:32} size mismatch {row}", flush=True) + failed.append(name) continue verdict = "matches" if row["vs_q90"] <= DRIFT_THRESHOLD else "DRIFTED" row["verdict"] = verdict @@ -175,6 +196,9 @@ def main(argv=None) -> int: encoding="utf-8") drifted = [r["name"] for r in results if r.get("verdict") == "DRIFTED"] print(f"\n# {len(results)} measured, {len(drifted)} drifted, {len(failed)} failed") + if not results and not failed: + print("error: no heroes were measured", file=sys.stderr) + return 2 return 3 if failed else 0 diff --git a/tests/test_measure_hero_drift.py b/tests/test_measure_hero_drift.py new file mode 100644 index 00000000..d3b7bc21 --- /dev/null +++ b/tests/test_measure_hero_drift.py @@ -0,0 +1,101 @@ +"""Exit-code tests for scripts/measure_hero_drift.py. No Blender required. + +Needs numpy and Pillow (the script's own dependencies). Run: + python tests/test_measure_hero_drift.py -v +""" +from __future__ import annotations + +import contextlib +import io +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "scripts")) + +import measure_hero_drift as m # noqa: E402 + +ENTRY = {"name": "demo", "dir": "examples/demo", "hero": "docs/gallery/assets/demo-hero.webp"} + + +def run(argv, *, entries=(ENTRY,), version="Blender 5.2.1 LTS", version_exit=0, size_match=True): + """Call m.main with every external effect faked; return (exit code, stderr).""" + td = Path(tempfile.mkdtemp()) + (td / "examples" / "demo").mkdir(parents=True) + (td / "examples" / "demo" / "demo.py").write_text("", encoding="utf-8") + + def fake_run(command, **kw): + if "--version" in command: + return mock.Mock(stdout=(version + "\n") if version else "", returncode=version_exit) + Path(command[command.index("--output") + 1]).write_bytes(b"png") + return mock.Mock(returncode=0, stdout="", stderr="") + + def fake_compare(committed, fresh): + if not size_match: + return {"shape_committed": (16, 16), "shape_fresh": (8, 8)} + return {"mean_abs": 0.0, "gt2pct": 0.0, "vs_q90": 0.0, + "luma_committed": 0.1, "luma_fresh": 0.1, "bytes_committed": 1} + + err = io.StringIO() + with mock.patch.object(m, "REPO", td), \ + mock.patch.object(m, "entries", lambda: list(entries)), \ + mock.patch.object(m, "render_args", lambda e, png: ["--output", str(png)]), \ + mock.patch.object(m.subprocess, "run", fake_run), \ + mock.patch.object(m, "compare", fake_compare), \ + contextlib.redirect_stderr(err), contextlib.redirect_stdout(io.StringIO()): + code = m.main(["--blender", "blender", "--out", str(td / "out"), *argv]) + return code, err.getvalue() + + +class ExitCodes(unittest.TestCase): + def test_matching_sizes_exit_0(self): + self.assertEqual(run([])[0], 0) + + def test_size_mismatch_is_a_failure(self): + code, _ = run([], size_match=False) + self.assertEqual(code, 3) + + def test_unknown_only_name_is_usage_error(self): + code, err = run(["--only", "nope"]) + self.assertEqual(code, 2) + self.assertIn("nope", err) + + def test_nothing_measured_is_usage_error(self): + self.assertEqual(run([], entries=())[0], 2) + + def test_blender_version_failure_is_clear_error(self): + code, err = run([], version="", version_exit=1) + self.assertEqual(code, 2) + self.assertIn("--version", err) + + +class RenderArgs(unittest.TestCase): + def test_bare_trailing_output_returns_none(self): + with tempfile.TemporaryDirectory() as td: + d = Path(td) / "examples" / "demo" + d.mkdir(parents=True) + (d / "README.md").write_text( + "blender --background --python demo.py -- --engine cycles --output\n", + encoding="utf-8", + ) + with mock.patch.object(m, "REPO", Path(td)): + self.assertIsNone(m.render_args({"dir": "examples/demo"}, Path("p.png"))) + + def test_output_path_is_retargeted(self): + with tempfile.TemporaryDirectory() as td: + d = Path(td) / "examples" / "demo" + d.mkdir(parents=True) + (d / "README.md").write_text( + "blender --background --python demo.py -- --output out.png --engine cycles\n", + encoding="utf-8", + ) + with mock.patch.object(m, "REPO", Path(td)): + args = m.render_args({"dir": "examples/demo"}, Path("fresh.png")) + self.assertEqual(args[args.index("--output") + 1], "fresh.png") + + +if __name__ == "__main__": + unittest.main()