diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 9d10db40..e6ced3af 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -136,6 +136,12 @@ jobs: - name: Check product exit codes are in README tables run: python3 tests/check_exit_code_readme.py + - name: Check EEVEE engine-id mappings + run: python3 tests/check_engine_id.py + + - name: Check falsifiers declare the budget they target + run: python3 tests/check_falsifier_targets.py + - name: Validate template Python syntax run: | echo "Checking template Python syntax..." diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a21af97..4ce2d6e0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -223,6 +223,39 @@ Three checks that appear on PRs are deliberately **excluded**: The two `Socket Security` checks come from a third-party GitHub App. An outage or an uninstall would deadlock merges, so they are advisory. +## Falsifiers must fail the budget they target + +Every falsifier declares the budget it aims at, and it must fail **that** +budget. A falsifier that exits non-zero on some earlier check proves +nothing about its target: the run is red for the wrong reason, and the +budget it was built for was never reached. + +Fix a collision by changing the model or the falsifier. **Never widen a +band so an ill-aimed falsifier lands.** + +Worked example. `showcase/stone-archway`'s arch falsifier began as +`--flat-arch`, laying the voussoirs as a flat lintel. A lintel is 0.62 m +shorter than the arch, so it tripped the bounding-box budget (exit 8) and +never reached the intrados circle fit (exit 19) it existed to break. +Widening the bbox tolerance to let it through would have destroyed a real +budget to rescue a bad falsifier. It was replaced by `--off-circle`, which +keeps the angles, joints, materials, triangle count and envelope identical +and wanders only the intrados radius. Four other falsifiers in the same run +needed the same treatment: a stray vertex moved inside the silhouette, a +`--short-skids` that floats one runner of three instead of all of them, a +`--same-seed` split into design and placement RNG streams, and a +`--sink-keystone` that no longer changes the Y envelope. + +`tests/check_falsifier_targets.py` enforces the declaration. Its default +static mode reads each showcase piece's falsifier table and asserts the +flags are real argparse flags, the declared exit codes appear in that +piece's exit-code table, and every falsifier names a target budget. Its +`--run BLENDER` mode executes each falsifier and asserts the **observed** +exit equals the declared one — the mode that catches an ill-aimed +falsifier. Runtime costs one Blender launch per falsifier, measured at +276 s for the whole showcase tree on one version, so it is an authoring +and cron tool rather than a per-PR smoke step. + ## Exit codes Three roles, not one global table. Do not copy a code from one script into diff --git a/examples/gn-sdf-remesh/gn_sdf_remesh.py b/examples/gn-sdf-remesh/gn_sdf_remesh.py index d811da54..76451fe3 100644 --- a/examples/gn-sdf-remesh/gn_sdf_remesh.py +++ b/examples/gn-sdf-remesh/gn_sdf_remesh.py @@ -116,7 +116,7 @@ def main(): # EEVEE-id inversion witnessed for real: the OTHER era's id must be # rejected by this build, the helper's accepted eid = get_eevee_engine_id() - wrong = 'BLENDER_EEVEE_NEXT' if bpy.app.version >= (5, 0, 0) else 'BLENDER_EEVEE' + wrong = 'BLENDER_EEVEE_NEXT' if bpy.app.version >= (5, 0, 0) else 'BLENDER_EEVEE' # engine-id-exempt: the wrong-era id this example asserts is rejected try: bpy.context.scene.render.engine = wrong print(f"ERROR: wrong-era EEVEE id '{wrong}' was accepted", file=sys.stderr); return 5 diff --git a/examples/swatch-grid/swatch_grid.py b/examples/swatch-grid/swatch_grid.py index 80e65d6c..d3937977 100644 --- a/examples/swatch-grid/swatch_grid.py +++ b/examples/swatch-grid/swatch_grid.py @@ -265,7 +265,7 @@ def main(): # Cycles. Witness the inversion for real: the OTHER era's id must be rejected # by this build, and the helper's id must be accepted. eid = get_eevee_engine_id() - wrong = 'BLENDER_EEVEE_NEXT' if bpy.app.version >= (5, 0, 0) else 'BLENDER_EEVEE' + wrong = 'BLENDER_EEVEE_NEXT' if bpy.app.version >= (5, 0, 0) else 'BLENDER_EEVEE' # engine-id-exempt: the wrong-era id this example asserts is rejected try: sc.render.engine = wrong print(f"ERROR: wrong-era EEVEE id '{wrong}' was accepted by this build — " diff --git a/examples/turntable/turntable.py b/examples/turntable/turntable.py index 85924bb8..788e4f59 100644 --- a/examples/turntable/turntable.py +++ b/examples/turntable/turntable.py @@ -135,7 +135,7 @@ def main(): # the EEVEE-id mapping is asserted regardless of whether we render: the # OTHER era's id must be rejected by this build, the helper's accepted eid = get_eevee_engine_id() - wrong = 'BLENDER_EEVEE_NEXT' if bpy.app.version >= (5, 0, 0) else 'BLENDER_EEVEE' + wrong = 'BLENDER_EEVEE_NEXT' if bpy.app.version >= (5, 0, 0) else 'BLENDER_EEVEE' # engine-id-exempt: the wrong-era id this example asserts is rejected try: bpy.context.scene.render.engine = wrong print(f"ERROR: wrong-era EEVEE id '{wrong}' was accepted", file=sys.stderr); return 5 diff --git a/showcase/README.md b/showcase/README.md index e8e8b5e1..0706b253 100644 --- a/showcase/README.md +++ b/showcase/README.md @@ -45,6 +45,18 @@ entry in `showcase/gallery.json`, and a rendered still. A budget with no falsifier witnesses nothing: prove each one fails once, and check the exit code, not just non-zero. +- **A falsifier must fail the budget it targets.** Declare the target in + the piece README's falsifier table — flag, target budget, exit code — + and make the run exit *that* code. A falsifier that trips an earlier + check is red for the wrong reason and proves nothing about its target. + Fix a collision by changing the model or the falsifier; never widen a + band so an ill-aimed falsifier lands. `stone-archway`'s `--flat-arch` + laid the voussoirs as a lintel, which is 0.62 m shorter and so failed + the bounding box (exit 8) instead of the intrados circle fit (exit 19); + `--off-circle` replaced it by keeping the whole envelope and wandering + only the radius. `tests/check_falsifier_targets.py` checks the + declaration statically, and with `--run BLENDER` executes each falsifier + and asserts the observed exit matches. - **Hygiene budgets.** Copied combinatorics from `examples/mesh-hygiene-audit` (do not import the example). Every piece asserts on the generated mesh: non-manifold edges 0, loose verts 0, @@ -164,6 +176,46 @@ entry in `showcase/gallery.json`, and a rendered still. `measure_framing_deviation` and assert at the call site. Do not move or modify `gallery_framing.py` — import it by resolving the repo root (see the shipping-crate script). +- **Contact sheet (required).** Composite the candidate hero beside the + pinned calibration set — canonical membership is in `CLAUDE.md` + § Quality Gates — commit it as + `docs/gallery/contact-sheets/-contact-sheet.webp`, link it in the + PR body, and report a per-criterion verdict: stage darkness, wedge + warmth, subject fill, saturation, thumbnail legibility, plus mean + luminance against the calibration band. A claim without the committed + composite is not evidence. + + Required because it has caught a real defect in showcase work. The + first sheets for `crate-stack` and `stone-archway` showed both wedge + pools reading as cool grey bands rather than the warm pool the house + style calls for; both were relit as a result. Nothing else in the + pipeline looks at the still beside its peers, so nothing else could + have seen it. + +- **Asset sheet (required).** Render the hero alone — neutral + three-quarter view, plain studio lighting, no staging tricks, no + labels, no comparison props — composite it beside the pinned + asset-quality reference set rendered the same way, commit under + `docs/gallery/asset-sheets/`, and report a verdict. The piece ships + only if it is not identifiable as the least-designed object in that + lineup. + + Required because showcase is *entirely* game props, which is exactly + the scope `docs/VISUAL-STYLE.md` § Asset quality names, and because it + covers something no other gate here does. Budgets measure geometry + conformance; the contact sheet measures staged presentation. Neither + removes the scene, and a strong scene carries a weak model. The + recorded evidence is `socket-attach-points`: it passed every + measurable floor on its first draft — `edge90` 0.000, ten materials, + no default datablock names — and was then judged bad by eye and + rebuilt from scratch. The floors scored the bevels, not the design. + + `examples/gallery_asset_quality.check_asset_quality` returns **11** on + violation, the same call pattern as `gallery_framing`. Showcase + numbering already spends 11 on the collider-triangle ceiling, so remap + the return at the call site rather than letting two budgets share a + code. + - **Composition.** The README names which shipped skills and snippets the piece composes. Duplicated helpers stay inlined or copied; showcase scripts do not import snippets as a package. diff --git a/showcase/iron-cauldron/iron_cauldron.py b/showcase/iron-cauldron/iron_cauldron.py index a8e28fc9..6e1a5a83 100644 --- a/showcase/iron-cauldron/iron_cauldron.py +++ b/showcase/iron-cauldron/iron_cauldron.py @@ -115,7 +115,14 @@ def eevee_engine_id(): - return "BLENDER_EEVEE_NEXT" if bpy.app.version >= (4, 2, 0) else "BLENDER_EEVEE" + """EEVEE id: 'BLENDER_EEVEE' on 5.0+, 'BLENDER_EEVEE_NEXT' on 4.2-4.5. + + Mapping matches examples/swatch-grid.get_eevee_engine_id, which asserts it + against the running build (copied, not imported). Inherited the inverted + form from showcase/shipping-crate; it returned 'BLENDER_EEVEE_NEXT' on 5.x, + where that id does not exist. + """ + return "BLENDER_EEVEE" if bpy.app.version >= (5, 0, 0) else "BLENDER_EEVEE_NEXT" def fail(msg, code): diff --git a/showcase/shipping-crate/shipping_crate.py b/showcase/shipping-crate/shipping_crate.py index 89dc0bf2..d48c2e49 100644 --- a/showcase/shipping-crate/shipping_crate.py +++ b/showcase/shipping-crate/shipping_crate.py @@ -101,7 +101,14 @@ def eevee_engine_id(): - return "BLENDER_EEVEE_NEXT" if bpy.app.version >= (4, 2, 0) else "BLENDER_EEVEE" + """EEVEE id: 'BLENDER_EEVEE' on 5.0+, 'BLENDER_EEVEE_NEXT' on 4.2-4.5. + + Mapping matches examples/swatch-grid.get_eevee_engine_id, which asserts it + against the running build (copied, not imported). The old form here keyed + on >= (4, 2, 0) and so returned 'BLENDER_EEVEE_NEXT' on 5.x, where that id + does not exist — the render path raised TypeError on 5.1 and 5.2. + """ + return "BLENDER_EEVEE" if bpy.app.version >= (5, 0, 0) else "BLENDER_EEVEE_NEXT" def fail(msg, code): diff --git a/tests/check_engine_id.py b/tests/check_engine_id.py new file mode 100644 index 00000000..db59952f --- /dev/null +++ b/tests/check_engine_id.py @@ -0,0 +1,274 @@ +"""Every EEVEE engine-id mapping in the tree must match the canonical one. + +Legacy EEVEE was removed in Blender 4.2. EEVEE Next used the id +``BLENDER_EEVEE_NEXT`` on 4.2 through 4.5 LTS, then reclaimed the plain +``BLENDER_EEVEE`` id in 5.0. So: + + BLENDER_EEVEE on 5.0 and above + BLENDER_EEVEE_NEXT on 4.2 through 4.5 + +Why this file exists +-------------------- + +`tests/smoke/run_smoke.py` already asserts the mapping against the running +build, which catches a wrong *canonical* mapping. It cannot catch a wrong +*copy*: showcase and example scripts are standalone by convention, so each +one carries its own `eevee_engine_id()`. Eighty-five files in this tree +mention the id. + +That is how an inverted ternary keyed on ``>= (4, 2, 0)`` — which returns +``BLENDER_EEVEE_NEXT`` on 5.x, where that id does not exist — survived in +`showcase/shipping-crate` and propagated to `showcase/iron-cauldron`. Both +raised ``TypeError`` on the ``--output`` path on 5.1 and 5.2. Smoke never +passes ``--output``, so nothing went red. + +A render canary would not have caught it either. Blender's EEVEE aborts on +GPU-less runners without EGL, which is why every render in +`blender-smoke.yml` uses Cycles — and Cycles never touches the EEVEE id. + +Method +------ + +AST, not regex. Every function that returns an EEVEE id is compiled and +*called* twice, against a stubbed ``bpy.app.version`` of (4, 5, 11) and +(5, 2, 1); every conditional expression that yields one is evaluated the +same way. Both forms — ternary and ``if``/``return`` — are handled by the +same code path, because a regex over either one misreads the other. + +A deliberately inverted mapping (``examples/swatch-grid`` witnesses the +inversion by asserting the wrong-era id is rejected) must carry +``# engine-id-exempt: `` on its own line. + +Exit codes: 0 clean, 1 a mapping disagrees, 2 usage. +""" +from __future__ import annotations + +import ast +import io +import os +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SCOPES = ("examples", "showcase", "templates", "snippets", "scripts", "tests") +SKIP_DIRS = {".git", "__pycache__", ".scratch", "node_modules", "docs"} +# The checker states the canonical mapping itself; judging it would be circular. +SKIP_FILES = {"tests/check_engine_id.py"} + +EEVEE_IDS = {"BLENDER_EEVEE", "BLENDER_EEVEE_NEXT"} +PROBES = ((4, 5, 11), (5, 2, 1)) +EXEMPT = "engine-id-exempt:" + + +def canonical(version): + return "BLENDER_EEVEE" if version >= (5, 0, 0) else "BLENDER_EEVEE_NEXT" + + +class _App: + def __init__(self, version): + self.version = version + + +class _Bpy: + def __init__(self, version): + self.app = _App(version) + + +def _mentions_id(node): + for sub in ast.walk(node): + if isinstance(sub, ast.Constant) and sub.value in EEVEE_IDS: + return True + return False + + +def _yields_id(node): + """True when *node* evaluates to an EEVEE id constant (possibly branching).""" + if isinstance(node, ast.Constant): + return node.value in EEVEE_IDS + if isinstance(node, ast.IfExp): + return _yields_id(node.body) and _yields_id(node.orelse) + return False + + +def _is_resolver(fn): + """True when *fn* does nothing but resolve an id from the version. + + Deliberately narrow. A ``main()`` that happens to mention the id + somewhere is not a resolver, and compiling and calling it would need + the whole module's imports. Only a body of returns and version + branches qualifies, which is the shape every copy in the tree uses. + """ + returns = [] + + def walk(stmts): + for st in stmts: + if isinstance(st, ast.Return): + if st.value is None or not _yields_id(st.value): + return False + returns.append(st) + elif isinstance(st, ast.If): + if not walk(st.body) or not walk(st.orelse): + return False + elif isinstance(st, ast.Expr) and isinstance(st.value, ast.Constant): + continue # docstring + elif isinstance(st, ast.Pass): + continue + else: + return False + return True + + return walk(fn.body) and bool(returns) and not fn.args.args + + +def _namespace(version): + return { + "bpy": _Bpy(version), + "IS_5X": version >= (5, 0, 0), + "IS_5_X": version >= (5, 0, 0), + "__builtins__": __builtins__, + } + + +def _evaluate(node, version): + """Return the id this node yields on *version*.""" + ns = _namespace(version) + if isinstance(node, ast.FunctionDef): + mod = ast.Module(body=[node], type_ignores=[]) + ast.fix_missing_locations(mod) + exec(compile(mod, "", "exec"), ns) + return ns[node.name]() + expr = ast.Expression(body=node) + ast.fix_missing_locations(expr) + return eval(compile(expr, "", "eval"), ns) + + +def _exempt_reason(lines, lineno): + line = lines[lineno - 1] if 0 < lineno <= len(lines) else "" + if EXEMPT in line: + return line.split(EXEMPT, 1)[1].strip() or "(no reason given)" + return None + + +def check_file(path): + """(checked, exempted, failures) for one file.""" + src = io.open(path, encoding="utf-8", errors="ignore").read() + if not any(i in src for i in EEVEE_IDS): + return 0, 0, [] + lines = src.split("\n") + try: + tree = ast.parse(src) + except SyntaxError as exc: + return 0, 0, [(path, getattr(exc, "lineno", 0), f"unparseable: {exc}")] + + rel = os.path.relpath(path, ROOT).replace("\\", "/") + checked = exempted = 0 + failures = [] + seen = set() + + # Functions first, so a ternary inside one is not also judged alone. + covered_lines = set() + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and _mentions_id(node) and _is_resolver(node): + for sub in ast.walk(node): + if hasattr(sub, "lineno"): + covered_lines.add(sub.lineno) + reason = _exempt_reason(lines, node.lineno) + if reason: + exempted += 1 + print(f" exempt {rel}:{node.lineno} {node.name}() - {reason}") + continue + try: + got = {v: _evaluate(node, v) for v in PROBES} + except Exception as exc: + failures.append( + (rel, node.lineno, f"{node.name}() not resolvable: {exc}") + ) + continue + checked += 1 + bad = {v: got[v] for v in PROBES if got[v] != canonical(v)} + if bad: + detail = " ".join( + f"{'.'.join(map(str, v))}->{got[v]} (want {canonical(v)})" + for v in bad + ) + failures.append((rel, node.lineno, f"{node.name}() {detail}")) + seen.add(node.lineno) + + for node in ast.walk(tree): + if not isinstance(node, ast.IfExp) or not _mentions_id(node): + continue + if node.lineno in covered_lines: + continue + reason = _exempt_reason(lines, node.lineno) + if reason: + exempted += 1 + print(f" exempt {rel}:{node.lineno} conditional - {reason}") + continue + try: + got = {v: _evaluate(node, v) for v in PROBES} + except Exception as exc: + failures.append((rel, node.lineno, f"conditional not resolvable: {exc}")) + continue + checked += 1 + bad = {v: got[v] for v in PROBES if got[v] != canonical(v)} + if bad: + detail = " ".join( + f"{'.'.join(map(str, v))}->{got[v]} (want {canonical(v)})" + for v in bad + ) + failures.append((rel, node.lineno, f"conditional {detail}")) + return checked, exempted, failures + + +def main(argv=None): + argv = list(sys.argv[1:] if argv is None else argv) + if argv: + print(__doc__.strip().split("\n")[0], file=sys.stderr) + print("usage: python tests/check_engine_id.py", file=sys.stderr) + return 2 + + total = exempt_total = 0 + failures = [] + files = 0 + for scope in SCOPES: + base = os.path.join(ROOT, scope) + if not os.path.isdir(base): + continue + for root, dirs, names in os.walk(base): + dirs[:] = [d for d in dirs if d not in SKIP_DIRS] + for name in sorted(names): + if not name.endswith(".py"): + continue + path = os.path.join(root, name) + if os.path.relpath(path, ROOT).replace("\\", "/") in SKIP_FILES: + continue + checked, exempted, fails = check_file(path) + if checked or exempted or fails: + files += 1 + total += checked + exempt_total += exempted + failures.extend(fails) + + if failures: + print( + f"\n{len(failures)} engine-id mapping(s) disagree with the canonical one " + "(BLENDER_EEVEE on 5.0+, BLENDER_EEVEE_NEXT on 4.2-4.5):", + file=sys.stderr, + ) + for rel, line, detail in failures: + print(f" ERROR: {rel}:{line} {detail}", file=sys.stderr) + print( + "\nA deliberately inverted mapping must carry " + f"'# {EXEMPT} ' on its own line.", + file=sys.stderr, + ) + return 1 + + print( + f"engine-id checks passed: {total} mapping(s) across {files} file(s), " + f"{exempt_total} exempt." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/check_falsifier_targets.py b/tests/check_falsifier_targets.py new file mode 100644 index 00000000..5930bd06 --- /dev/null +++ b/tests/check_falsifier_targets.py @@ -0,0 +1,285 @@ +"""A falsifier must fail the budget it targets, not an earlier check. + +A falsifier that exits non-zero on some *earlier* budget proves nothing +about the budget it was built for. The exit code is right there to catch +it, so this makes the flag -> budget -> exit-code mapping machine-checked +instead of prose. + +Worked example, from the run that motivated this file: `stone-archway`'s +arch falsifier began as ``--flat-arch``, laying the voussoirs as a flat +lintel. A lintel is 0.62 m shorter than the arch, so it tripped the +bounding-box budget (exit 8) and never reached the intrados circle fit +(exit 19) it existed to break. It was replaced by ``--off-circle``, which +keeps the angles, joints, materials, triangle count and envelope +identical and wanders only the intrados radius. Four other falsifiers +needed the same treatment in the same run. + +The fix is always to change the model or the falsifier. Never widen a +band so an ill-aimed falsifier lands. + +Two modes +--------- + +**Static (default, no Blender).** Reads each piece's README and script and +asserts they agree: + +1. every falsifier flag in the README table is a real argparse flag; +2. every declared exit code appears in that README's exit-code table; +3. every falsifier row names the budget it targets, in its own column; +4. every falsifier-shaped flag in the script is documented. + +**Runtime (``--run BLENDER``).** Executes each falsifier and asserts the +*observed* exit equals the declared one. This is the mode that catches a +falsifier tripping an earlier check. It costs one Blender launch per +falsifier: measured at 276 s for the whole showcase tree on one version +(160 falsifiers, 25 pieces, Blender 5.2.1). That is an authoring and cron +tool, not a per-PR smoke step. + +Exit codes: 0 clean, 1 a mapping disagrees, 2 usage. +""" +from __future__ import annotations + +import argparse +import ast +import io +import os +import re +import subprocess +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SHOWCASE = os.path.join(ROOT, "showcase") + +# Pieces that predate the falsifier-table convention. Tracked for backfill; +# a NEW piece without a table is an error, not an entry here. +KNOWN_UNDOCUMENTED = {"cart", "hay-bale", "stone-well"} + +# Flags that select a code path rather than break a contract. Per +# CONTRIBUTING.md these are explicitly not falsifiers. +NOT_FALSIFIERS = { + "--output", "--engine", "--api", "--check-pixels", "--obj", + "--samples", "--width", "--height", "--force-run", "--expect-sidecar", + "--seed", "--quality", "--verbose", "--no-render", +} + +FLAG_RE = re.compile(r"`?(--[a-z0-9][a-z0-9-]*)`?") + + +def _tables(text): + """Every markdown table as (headers, rows-of-cells).""" + out = [] + lines = text.split("\n") + i = 0 + while i < len(lines): + if lines[i].strip().startswith("|") and i + 1 < len(lines) and re.match( + r"^\|[\s:|-]+\|$", lines[i + 1].strip() + ): + headers = [c.strip() for c in lines[i].strip().strip("|").split("|")] + rows = [] + j = i + 2 + while j < len(lines) and lines[j].strip().startswith("|"): + rows.append([c.strip() for c in lines[j].strip().strip("|").split("|")]) + j += 1 + out.append((headers, rows)) + i = j + else: + i += 1 + return out + + +def _column(headers, *patterns): + for idx, h in enumerate(headers): + low = h.lower() + if any(re.search(p, low) for p in patterns): + return idx + return None + + +def parse_readme(path): + """(falsifiers {flag: (exit, target)}, exit_table {code: meaning}).""" + text = io.open(path, encoding="utf-8").read() + falsifiers, exits = {}, {} + for headers, rows in _tables(text): + code_col = _column(headers, r"^exit$", r"exit code", r"^code$") + if code_col is None: + continue + flag_col = _column(headers, r"falsifier", r"^flag$") + if flag_col is not None: + # The remaining column is the declared target budget. Every + # falsifier has to say what it aims at; that is the whole point. + target_col = next( + (i for i in range(len(headers)) if i not in (flag_col, code_col)), + None, + ) + for r in rows: + if len(r) <= max(flag_col, code_col): + continue + m = FLAG_RE.search(r[flag_col]) + if not m or not r[code_col].strip().strip("`").isdigit(): + continue + target = "" + if target_col is not None and len(r) > target_col: + target = r[target_col].strip() + falsifiers[m.group(1)] = ( + int(r[code_col].strip().strip("`")), target + ) + continue + # exit-code table: code column plus a meaning column + mean_col = _column(headers, r"meaning", r"description") + if mean_col is None: + mean_col = 1 if len(headers) > 1 else None + if mean_col is None: + continue + for r in rows: + if len(r) <= max(code_col, mean_col): + continue + c = r[code_col].strip().strip("`") + if c.isdigit(): + exits[int(c)] = r[mean_col] + return falsifiers, exits + + +def script_flags(path): + """store_true argparse flags declared in the script.""" + src = io.open(path, encoding="utf-8").read() + flags = set() + try: + tree = ast.parse(src) + except SyntaxError: + return flags + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + fn = node.func + if not (isinstance(fn, ast.Attribute) and fn.attr == "add_argument"): + continue + store_true = any( + kw.arg == "action" + and isinstance(kw.value, ast.Constant) + and kw.value.value == "store_true" + for kw in node.keywords + ) + for arg in node.args: + if isinstance(arg, ast.Constant) and isinstance(arg.value, str): + if arg.value.startswith("--") and store_true: + flags.add(arg.value) + return flags + + +def piece_paths(): + for name in sorted(os.listdir(SHOWCASE)): + d = os.path.join(SHOWCASE, name) + if not os.path.isdir(d): + continue + readme = os.path.join(d, "README.md") + script = os.path.join(d, name.replace("-", "_") + ".py") + if os.path.isfile(readme) and os.path.isfile(script): + yield name, readme, script + + +def check_static(): + failures, checked, pieces = [], 0, 0 + undocumented_only = [] + undocumented_pieces = [] + for name, readme, script in piece_paths(): + falsifiers, exits = parse_readme(readme) + if not falsifiers: + if name in KNOWN_UNDOCUMENTED: + undocumented_pieces.append(name) + continue + failures.append( + (name, "no falsifier table mapping flag to target budget to " + "exit code") + ) + continue + pieces += 1 + flags = script_flags(script) + for flag, (code, target) in sorted(falsifiers.items()): + checked += 1 + if flag not in flags: + failures.append( + (name, f"{flag} documented but not an argparse flag") + ) + continue + if code not in exits: + failures.append( + (name, f"{flag} declares exit {code}, absent from the " + "exit-code table") + ) + continue + if not target: + failures.append( + (name, f"{flag} names no target budget in its row") + ) + for flag in sorted(flags - set(falsifiers) - NOT_FALSIFIERS): + undocumented_only.append((name, flag)) + return failures, undocumented_only, checked, pieces, undocumented_pieces + + +def check_runtime(blender, only=None): + failures, checked = [], 0 + for name, readme, script in piece_paths(): + if only and name not in only: + continue + falsifiers, _exits = parse_readme(readme) + for flag, (want, _target) in sorted(falsifiers.items()): + checked += 1 + proc = subprocess.run( + [blender, "--background", "--python", script, "--", flag], + capture_output=True, + ) + got = proc.returncode + status = "ok" if got == want else "MISMATCH" + print(f" {status:9} {name} {flag}: want {want}, got {got}") + if got != want: + failures.append( + (name, f"{flag} declared exit {want} but exited {got}") + ) + return failures, checked + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--run", metavar="BLENDER", default=None, + help="also execute each falsifier with this Blender binary") + p.add_argument("--only", action="append", default=None, + help="limit the runtime mode to these piece names") + args = p.parse_args(argv) + + failures, undocumented, checked, pieces, skipped = check_static() + for name, flag in undocumented: + failures.append((name, f"{flag} looks like a falsifier but is not in " + "the falsifier table")) + + if skipped: + print( + f"note: {len(skipped)} piece(s) predate the falsifier table and are " + f"tracked for backfill: {', '.join(sorted(skipped))}" + ) + + if args.run: + print(f"runtime falsifier sweep with {args.run}") + rt_failures, rt_checked = check_runtime(args.run, set(args.only or []) or None) + failures.extend(rt_failures) + checked += rt_checked + + if failures: + print(f"\n{len(failures)} falsifier mapping problem(s):", file=sys.stderr) + for name, detail in failures: + print(f" ERROR: showcase/{name}: {detail}", file=sys.stderr) + print( + "\nA falsifier must fail the budget it targets. Fix a collision by " + "changing the model or the falsifier, never by widening a band.", + file=sys.stderr, + ) + return 1 + + print( + f"falsifier-target checks passed: {checked} falsifier(s) across " + f"{pieces} showcase piece(s)." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())