ci: smoke the shipped templates, enforce catalog completeness, parse frontmatter, cache and verify Blender (#314)

- #291: smoke runs templates/headless-batch-script-template/script.py in place (copy
  deleted); extension template gets `extension validate` plus a register/unregister
  lifecycle row in the catalog (operator, panel, Scene pointer, operator behaviour)
- #292: Validate fails when an example/showcase dir has no catalog row or a row points
  at a missing dir; run_catalog runs every row and lists all failures (--fail-fast is
  opt-in); an empty catalog is red; unit tests added
- #293: skill/rule frontmatter parsed as YAML (first block only); rules must carry a
  boolean alwaysApply and non-empty globs
- #294: Blender Smoke skips docs-only PRs, caches the extracted Blender by resolved
  version, verifies the published sha256 before extracting, drops the duplicate
  harness unit-test run

Closes #291, closes #292, closes #293, closes #294

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>

Author: TMHSDigital

.github/workflows/blender-smoke.yml

@@ -33,6 +33,12 @@ on:
   pull_request:
     branches: [main]
     types: [opened, synchronize, reopened, labeled]
+    # Docs-only PRs do not touch anything Blender runs. A PR that also changes
+    # code is not ignored: paths-ignore skips only when every file matches.
+    paths-ignore:
+      - "**.md"
+      - "docs/**"
+      - "assets/**"
 
 permissions:
   contents: read
@@ -87,11 +93,6 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
 
-      - name: Harness protocol unit tests
-        run: |
-          set -euo pipefail
-          python3 tests/smoke/test_harness.py -v
-
       - name: Init smoke status log
         run: |
           set -euo pipefail
@@ -109,7 +110,8 @@ jobs:
             xvfb libgl1 libegl1 libxrender1 libxxf86vm1 libxfixes3 libxi6 \
             libxkbcommon0 libsm6 libice6
 
-      - name: Resolve and download Blender ${{ matrix.series }}
+      - name: Resolve Blender ${{ matrix.series }}
+        id: resolve
         run: |
           set -euo pipefail
           series="${{ matrix.series }}"
@@ -123,14 +125,46 @@ jobs:
             echo "::error::Could not resolve a linux-x64 build for Blender ${series} at $base"
             exit 1
           fi
-          url="${base}${file}"
-          echo "Downloading $url"
+          version="${file#blender-}"
+          version="${version%-linux-x64.tar.xz}"
+          echo "base=$base" >> "$GITHUB_OUTPUT"
+          echo "file=$file" >> "$GITHUB_OUTPUT"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+          echo "Resolved Blender $version"
+
+      - name: Restore Blender ${{ steps.resolve.outputs.version }} from cache
+        id: bl-cache
+        uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
+        with:
+          path: ${{ runner.temp }}/bl
+          key: blender-${{ steps.resolve.outputs.version }}-linux-x64
+
+      - name: Download Blender and verify its published sha256
+        if: steps.bl-cache.outputs.cache-hit != 'true'
+        run: |
+          set -euo pipefail
+          base="${{ steps.resolve.outputs.base }}"
+          file="${{ steps.resolve.outputs.file }}"
+          version="${{ steps.resolve.outputs.version }}"
           mkdir -p "$RUNNER_TEMP/bl"
-          curl -fSL --retry 3 -o "$RUNNER_TEMP/bl.tar.xz" "$url"
+          echo "Downloading ${base}${file}"
+          curl -fSL --retry 3 -o "$RUNNER_TEMP/bl.tar.xz" "${base}${file}"
+          curl -fsSL --retry 3 -o "$RUNNER_TEMP/bl.sha256" "${base}blender-${version}.sha256"
+          expected=$(grep -E "  ${file}$" "$RUNNER_TEMP/bl.sha256" | awk '{print $1}')
+          if [ -z "$expected" ]; then
+            echo "::error::no published sha256 for ${file}"
+            exit 1
+          fi
+          echo "${expected}  $RUNNER_TEMP/bl.tar.xz" | sha256sum -c - \
+            || { echo "::error::sha256 mismatch for ${file}; refusing to extract"; exit 1; }
           tar -xf "$RUNNER_TEMP/bl.tar.xz" -C "$RUNNER_TEMP/bl"
+
+      - name: Locate Blender binary
+        run: |
+          set -euo pipefail
           bl=$(find "$RUNNER_TEMP/bl" -maxdepth 2 -type f -name blender | head -1)
           if [ -z "$bl" ]; then
-            echo "::error::blender binary not found after extraction"
+            echo "::error::blender binary not found"
             exit 1
           fi
           echo "BLENDER=$bl" >> "$GITHUB_ENV"
@@ -158,7 +192,7 @@ jobs:
         run: |
           set -euo pipefail
           xvfb-run -a "$BLENDER" --background "$RUNNER_TEMP/out/input.blend" \
-            --python tests/smoke/tmpl_gltf.py -- \
+            --python templates/headless-batch-script-template/script.py -- \
             --output "$RUNNER_TEMP/out/out.glb" --apply-modifier SUBSURF
           test -s "$RUNNER_TEMP/out/out.glb" || { echo "::error::glTF output missing/empty"; exit 1; }
           head -c4 "$RUNNER_TEMP/out/out.glb" | grep -q "glTF" || { echo "::error::not a glTF binary"; exit 1; }
@@ -167,12 +201,17 @@ jobs:
         run: |
           set +e
           xvfb-run -a "$BLENDER" --background "$RUNNER_TEMP/out/empty.blend" \
-            --python tests/smoke/tmpl_gltf.py -- --output "$RUNNER_TEMP/out/none.glb"
+            --python templates/headless-batch-script-template/script.py -- --output "$RUNNER_TEMP/out/none.glb"
           code=$?
           set -e
           [ "$code" -eq 2 ] || { echo "::error::expected exit 2 for no-mesh input, got $code"; exit 1; }
           echo "no-mesh exit code = $code (correct)"
 
+      - name: Extension template passes Blender's own manifest validator
+        run: |
+          set -euo pipefail
+          "$BLENDER" --command extension validate templates/extension-addon-template
+
       - name: Build pipeline template fixture GLB
         run: |
           set -euo pipefail

.github/workflows/validate.yml

@@ -36,82 +36,10 @@ jobs:
           fi
           echo "VERSION valid: $version"
 
-      - name: Validate skill frontmatter
+      - name: Validate skill and rule frontmatter (parsed YAML, first block only)
         run: |
-          echo "Checking skill YAML frontmatter..."
-          errors=0
-          for skill in skills/*/SKILL.md; do
-            if [ ! -f "$skill" ]; then continue; fi
-            if ! head -1 "$skill" | grep -q "^---$"; then
-              echo "ERROR: $skill missing YAML frontmatter opening ---"
-              errors=$((errors + 1))
-              continue
-            fi
-            frontmatter=$(sed -n '/^---$/,/^---$/p' "$skill" | sed '1d;$d')
-            if ! echo "$frontmatter" | grep -q "^name:"; then
-              echo "ERROR: $skill missing 'name' in frontmatter"
-              errors=$((errors + 1))
-            fi
-            if ! echo "$frontmatter" | grep -q "^description:"; then
-              echo "ERROR: $skill missing 'description' in frontmatter"
-              errors=$((errors + 1))
-            fi
-            if ! echo "$frontmatter" | grep -q "^standards-version:"; then
-              echo "ERROR: $skill missing 'standards-version' in frontmatter"
-              errors=$((errors + 1))
-            fi
-          done
-          if [ "$errors" -gt 0 ]; then
-            echo "$errors frontmatter error(s) found."
-            exit 1
-          fi
-          echo "All skill frontmatter valid."
-
-      - name: Check skill name matches directory
-        run: |
-          echo "Checking skill names match directory names..."
-          errors=0
-          for skill in skills/*/SKILL.md; do
-            if [ ! -f "$skill" ]; then continue; fi
-            dir_name=$(echo "$skill" | sed 's|skills/\(.*\)/SKILL.md|\1|')
-            fm_name=$(sed -n '/^---$/,/^---$/p' "$skill" | sed '1d;$d' | grep "^name:" | sed 's/^name: *//')
-            if [ "$dir_name" != "$fm_name" ]; then
-              echo "ERROR: $skill has name '$fm_name' but directory is '$dir_name'"
-              errors=$((errors + 1))
-            fi
-          done
-          if [ "$errors" -gt 0 ]; then
-            echo "$errors name mismatch(es) found."
-            exit 1
-          fi
-          echo "All skill names match their directories."
-
-      - name: Validate rule frontmatter
-        run: |
-          echo "Checking rule YAML frontmatter..."
-          errors=0
-          for rule in rules/*.mdc; do
-            if [ ! -f "$rule" ]; then continue; fi
-            if ! head -1 "$rule" | grep -q "^---$"; then
-              echo "ERROR: $rule missing YAML frontmatter opening ---"
-              errors=$((errors + 1))
-              continue
-            fi
-            frontmatter=$(sed -n '/^---$/,/^---$/p' "$rule" | sed '1d;$d')
-            if ! echo "$frontmatter" | grep -q "^description:"; then
-              echo "ERROR: $rule missing 'description' in frontmatter"
-              errors=$((errors + 1))
-            fi
-            if ! echo "$frontmatter" | grep -q "^standards-version:"; then
-              echo "ERROR: $rule missing 'standards-version' in frontmatter"
-              errors=$((errors + 1))
-            fi
-          done
-          if [ "$errors" -gt 0 ]; then
-            echo "$errors frontmatter error(s) found."
-            exit 1
-          fi
-          echo "All rule frontmatter valid."
+          pip install PyYAML==6.0.2
+          python3 tests/check_frontmatter.py
 
       - name: Validate snippet Python syntax
         run: |
@@ -133,6 +61,9 @@ jobs:
       - name: Check import-scale and unevaluated-export anti-patterns
         run: python3 tests/check_import_export_rules.py
 
+      - name: Check the smoke catalog covers every example and showcase piece
+        run: python3 tests/check_smoke_catalog.py
+
       - name: Check product exit codes are in README tables
         run: python3 tests/check_exit_code_readme.py
 

tests/check_frontmatter.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Validate skill and rule frontmatter by parsing the leading YAML block.
+
+Only the first ``---`` block is read (a Markdown horizontal rule later in the
+body cannot satisfy a missing key). Skills need name (matching the directory),
+description and standards-version. Rules need description, standards-version, a
+boolean alwaysApply, and non-empty globs given as a string or a list of strings.
+
+    python tests/check_frontmatter.py [--root PATH]
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import yaml
+
+
+def front(path: Path) -> tuple[dict | None, str]:
+    text = path.read_text(encoding="utf-8").replace("\r\n", "\n")
+    if not text.startswith("---\n"):
+        return None, "missing opening ---"
+    parts = text.split("---\n", 2)
+    if len(parts) < 3:
+        return None, "frontmatter is not closed by ---"
+    try:
+        data = yaml.safe_load(parts[1])
+    except yaml.YAMLError as exc:
+        return None, f"invalid YAML: {exc}"
+    if not isinstance(data, dict):
+        return None, "frontmatter is not a mapping"
+    return data, ""
+
+
+def need_str(data: dict, key: str, label: str, errors: list[str]) -> None:
+    if not isinstance(data.get(key), str) or not data[key].strip():
+        errors.append(f"{label}: '{key}' must be a non-empty string")
+
+
+def check(root: Path) -> list[str]:
+    errors: list[str] = []
+    for path in sorted((root / "skills").glob("*/SKILL.md")):
+        label = path.relative_to(root).as_posix()
+        data, why = front(path)
+        if data is None:
+            errors.append(f"{label}: {why}")
+            continue
+        for key in ("name", "description", "standards-version"):
+            need_str(data, key, label, errors)
+        if data.get("name") != path.parent.name:
+            errors.append(f"{label}: name {data.get('name')!r} != directory {path.parent.name!r}")
+    for path in sorted((root / "rules").glob("*.mdc")):
+        label = path.relative_to(root).as_posix()
+        data, why = front(path)
+        if data is None:
+            errors.append(f"{label}: {why}")
+            continue
+        for key in ("description", "standards-version"):
+            need_str(data, key, label, errors)
+        if not isinstance(data.get("alwaysApply"), bool):
+            errors.append(f"{label}: 'alwaysApply' must be a boolean (got {data.get('alwaysApply')!r})")
+        globs = data.get("globs")
+        ok = (isinstance(globs, str) and globs.strip()) or (
+            isinstance(globs, list) and globs and all(isinstance(g, str) and g.strip() for g in globs)
+        )
+        if not ok:
+            errors.append(f"{label}: 'globs' must be a non-empty string or list of strings")
+    return errors
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--root", default=str(Path(__file__).resolve().parent.parent))
+    errors = check(Path(ap.parse_args().root))
+    for e in errors:
+        print(f"::error::{e}", file=sys.stderr)
+    if errors:
+        return 1
+    print("skill and rule frontmatter valid")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/check_smoke_catalog.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Fail when examples/showcase pieces and tests/smoke/catalog.json disagree.
+
+Every examples/<name>/ and showcase/<name>/ directory must be the target of at
+least one catalog row (the row's ``script`` path names the directory), and every
+row that points into examples/ or showcase/ must name a directory that exists.
+An empty catalog fails. Rows for harness code (tests/smoke/...) are allowed.
+
+    python tests/check_smoke_catalog.py [--root PATH]
+
+No Blender needed; runs in Validate.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+# Directories that intentionally have no smoke row. Keep empty unless justified.
+EXCLUDED: set[str] = set()
+
+
+def check(root: Path) -> list[str]:
+    catalog = json.loads((root / "tests/smoke/catalog.json").read_text(encoding="utf-8"))
+    if not catalog:
+        return ["tests/smoke/catalog.json is empty"]
+
+    covered: set[str] = set()
+    errors: list[str] = []
+    for row in catalog:
+        parts = Path(row["script"]).parts
+        if len(parts) >= 2 and parts[0] in ("examples", "showcase"):
+            key = f"{parts[0]}/{parts[1]}"
+            covered.add(key)
+            if not (root / key).is_dir():
+                errors.append(f"catalog row {row['name']!r} points at missing directory {key}")
+
+    for top in ("examples", "showcase"):
+        for d in sorted((root / top).iterdir()):
+            if not d.is_dir() or d.name == "__pycache__":
+                continue
+            key = f"{top}/{d.name}"
+            if key not in covered and key not in EXCLUDED:
+                errors.append(f"{key} has no row in tests/smoke/catalog.json (it would never run)")
+    return errors
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--root", default=str(Path(__file__).resolve().parent.parent))
+    root = Path(ap.parse_args().root)
+    errors = check(root)
+    for e in errors:
+        print(f"::error::{e}", file=sys.stderr)
+    if errors:
+        return 1
+    print("smoke catalog covers every example and showcase directory")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/smoke/catalog.json

@@ -79,6 +79,7 @@
   {"name": "lattice-deform", "script": "examples/lattice-deform/lattice_deform.py"},
   {"name": "solidify-even-thickness", "script": "examples/solidify-even-thickness/solidify_even_thickness.py"},
   {"name": "boolean-exact-volume", "script": "examples/boolean-exact-volume/boolean_exact_volume.py"},
+  {"name": "extension-template-lifecycle", "script": "tests/smoke/check_extension_template.py"},
   {"name": "shipping-crate", "script": "showcase/shipping-crate/shipping_crate.py"},
   {"name": "stone-well", "script": "showcase/stone-well/stone_well.py"},
   {"name": "wooden-barrel", "script": "showcase/wooden-barrel/wooden_barrel.py"},