diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
new file mode 100644
index 0000000..9ae855f
--- /dev/null
+++ b/.github/workflows/release.yaml
@@ -0,0 +1,164 @@
+name: Release to PyPI
+
+# GitHub had v0.8.0 from 28 August while PyPI served v0.7.3 from 14 May, so
+# `pip install aicertify` did not install what this repository represented as
+# the product. Publishing was a manual step and manual steps get skipped.
+#
+# A tag now publishes. The tag is the only trigger, the version in the tag must
+# match the version in pyproject.toml, and the wheel is installed into a clean
+# environment and made to produce real verdicts before anything is uploaded.
+#
+# Authentication is PyPI Trusted Publishing (OIDC). There is no API token in
+# this repository. Configure the publisher once at
+# https://pypi.org/manage/project/aicertify/settings/publishing/ with:
+# owner Principled-Evolution
+# repository aicertify
+# workflow release.yaml
+# environment pypi
+
+on:
+ push:
+ tags:
+ - "v*"
+ workflow_dispatch:
+ inputs:
+ dry_run:
+ description: "Build and smoke-test only; do not publish"
+ type: boolean
+ default: true
+
+permissions:
+ contents: read
+
+jobs:
+ build:
+ name: Build and verify the wheel
+ runs-on: ubuntu-latest
+ outputs:
+ version: ${{ steps.version.outputs.version }}
+ steps:
+ # The policy library is a submodule. Without it the wheel builds and
+ # installs but contains no .rego files, so every evaluation returns
+ # nothing while still exiting successfully.
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - name: Install Poetry
+ run: |
+ curl -sSL https://install.python-poetry.org | python -
+ echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+ - name: Read the version from pyproject.toml
+ id: version
+ run: |
+ version="$(poetry version --short)"
+ echo "version=${version}" >> "$GITHUB_OUTPUT"
+ echo "pyproject version is ${version}"
+
+ # A tag that disagrees with pyproject.toml is how the published version
+ # and the represented version come apart in the first place.
+ - name: Check the tag matches the version
+ if: startsWith(github.ref, 'refs/tags/v')
+ run: |
+ tag="${GITHUB_REF#refs/tags/v}"
+ version="${{ steps.version.outputs.version }}"
+ if [ "${tag}" != "${version}" ]; then
+ echo "::error::tag v${tag} does not match pyproject version ${version}" >&2
+ exit 1
+ fi
+ echo "tag v${tag} matches pyproject version ${version}"
+
+ - name: Build sdist and wheel
+ run: poetry build
+
+ # The policies are data files, not code, so a packaging change can drop
+ # them silently. An empty policy directory is indistinguishable from a
+ # passing run at the CLI, so it is checked here.
+ - name: Check the wheel carries the policy library
+ run: |
+ wheel="$(ls dist/*.whl)"
+ rego="$(unzip -l "${wheel}" | grep -c '\.rego' || true)"
+ echo "${rego} .rego files in ${wheel}"
+ if [ "${rego}" -lt 50 ]; then
+ echo "::error::wheel contains ${rego} .rego files; the submodule is missing or unpackaged" >&2
+ exit 1
+ fi
+ unzip -l "${wheel}" | grep -q 'opa_policies/docs/coverage/coverage.json' || {
+ echo "::error::wheel has no coverage.json; verdict extraction reads it to find each policy's decision rule" >&2
+ exit 1
+ }
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: dist
+ path: dist/
+
+ smoke:
+ name: Clean-install smoke test
+ needs: build
+ runs-on: ubuntu-latest
+ steps:
+ # Deliberately no repository checkout beyond the smoke script: the point
+ # is to exercise what a user gets from `pip install aicertify`, not what
+ # a developer gets from a working tree. A source checkout would mask a
+ # missing package data file, because the file would be on disk anyway.
+ - uses: actions/checkout@v4
+ with:
+ sparse-checkout: |
+ scripts/smoke_test_wheel.py
+ sparse-checkout-cone-mode: false
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - uses: actions/download-artifact@v4
+ with:
+ name: dist
+ path: dist/
+
+ - name: Install OPA
+ env:
+ OPA_VERSION: v1.20.1
+ run: |
+ curl -L -o opa "https://openpolicyagent.org/downloads/${OPA_VERSION}/opa_linux_amd64"
+ chmod 755 opa
+ sudo mv opa /usr/local/bin/
+ opa version
+
+ - name: Install the built wheel into a clean environment
+ run: |
+ python -m venv /tmp/smoke
+ /tmp/smoke/bin/pip install --upgrade pip
+ /tmp/smoke/bin/pip install dist/*.whl
+
+ # Runs a real framework evaluation and asserts on the number of verdicts.
+ # The CLI exits 0 and prints "OPA Evaluation: Successful" whether it
+ # produced 29 verdicts or none, so the exit code cannot detect the
+ # regression this is here to catch.
+ - name: Evaluate real frameworks from the installed wheel
+ run: /tmp/smoke/bin/python scripts/smoke_test_wheel.py
+
+ publish:
+ name: Publish to PyPI
+ needs: [build, smoke]
+ if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push'
+ runs-on: ubuntu-latest
+ environment:
+ name: pypi
+ url: https://pypi.org/project/aicertify/${{ needs.build.outputs.version }}/
+ permissions:
+ id-token: write # Trusted Publishing exchanges this for a PyPI token.
+ steps:
+ - uses: actions/download-artifact@v4
+ with:
+ name: dist
+ path: dist/
+
+ - name: Publish
+ uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/README-pypi.md b/README-pypi.md
index 90d21eb..99b4998 100644
--- a/README-pypi.md
+++ b/README-pypi.md
@@ -11,6 +11,7 @@
+ 
diff --git a/README.hi-IN.md b/README.hi-IN.md
index 7bb15cc..f0b894a 100644
--- a/README.hi-IN.md
+++ b/README.hi-IN.md
@@ -19,6 +19,7 @@
+
diff --git a/README.ja-JP.md b/README.ja-JP.md
index 6eb15c0..c10df56 100644
--- a/README.ja-JP.md
+++ b/README.ja-JP.md
@@ -19,6 +19,7 @@
+
diff --git a/README.ko-KR.md b/README.ko-KR.md
index 1811bd7..0c39bfb 100644
--- a/README.ko-KR.md
+++ b/README.ko-KR.md
@@ -19,6 +19,7 @@
+
diff --git a/README.md b/README.md
index db0042e..8005eac 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,7 @@
+
diff --git a/README.zh-CN.md b/README.zh-CN.md
index 20cfc7e..8ab8358 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -19,6 +19,7 @@
+
diff --git a/aicertify/opa_core/evaluator.py b/aicertify/opa_core/evaluator.py
index 4077efe..8cd51c0 100644
--- a/aicertify/opa_core/evaluator.py
+++ b/aicertify/opa_core/evaluator.py
@@ -128,11 +128,26 @@ def __init__(
)
self.policy_loader = PolicyLoader()
- self.opa_path = (
- None
- if (use_external_server or skip_opa_check)
- else self._verify_opa_installation()
- )
+
+ # Skipping the installation check must not mean discarding the path.
+ #
+ # GitHub Actions sets CI=true, which set skip_opa_check and left
+ # opa_path as None. None then went into argv[0], so every call through
+ # evaluate_policy raised "sequence item 0: expected str instance,
+ # NoneType found" and the folder evaluation reported "No valid results
+ # from any policy evaluation". The path through
+ # _evaluate_with_local_opa is worse: it checks for None and returns a
+ # mock result, so a run in CI reported fabricated verdicts as real ones.
+ #
+ # The flag exists so a missing binary does not abort startup, not so a
+ # present one goes unused. Resolve it either way and let the call site
+ # fail on a real missing executable.
+ if use_external_server:
+ self.opa_path = None
+ elif skip_opa_check:
+ self.opa_path = shutil.which("opa") or os.environ.get("OPA_PATH") or "opa"
+ else:
+ self.opa_path = self._verify_opa_installation()
self.use_external_server = use_external_server
self.server_url = server_url
self.policies_loaded = False
diff --git a/aicertify/opa_policies b/aicertify/opa_policies
index 0936496..e565a60 160000
--- a/aicertify/opa_policies
+++ b/aicertify/opa_policies
@@ -1 +1 @@
-Subproject commit 09364967d494b4137b8c492135c9c7738964e601
+Subproject commit e565a6020cba595db630f8a037859f3a0b81d837
diff --git a/pyproject.toml b/pyproject.toml
index 68c6788..76627e1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -142,6 +142,12 @@ black = ">=26.3.1,<27.0.0"
# broken, that autoload crashes pytest itself, regardless of whether any test
# imports deepeval. Disabling it here is unrelated to test content.
addopts = "-p no:plugins"
+markers = [
+ # Real OPA evaluations over the vendored policy library. Deselect with
+ # -m "not slow" for a fast loop; CI runs them, because they are the only
+ # tests that can catch a framework silently returning no verdicts.
+ "slow: runs a real OPA evaluation against the policy library",
+]
[tool.ruff]
line-length = 88
diff --git a/scripts/smoke_test_wheel.py b/scripts/smoke_test_wheel.py
new file mode 100644
index 0000000..abe3995
--- /dev/null
+++ b/scripts/smoke_test_wheel.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""
+Prove an installed aicertify wheel produces real verdicts before it is published.
+
+Run with the interpreter of a clean virtualenv that has the wheel installed and
+nothing else from this repository:
+
+ python -m venv /tmp/smoke
+ /tmp/smoke/bin/pip install dist/aicertify-*.whl
+ /tmp/smoke/bin/python scripts/smoke_test_wheel.py
+
+Why this is not covered by the unit tests. The unit tests run against a source
+checkout, where the .rego files and coverage.json are on disk whether or not
+packaging includes them. A wheel that ships no policies passes every one of
+them. It also passes at the CLI: `aicertify evaluate` prints "OPA Evaluation:
+Successful" and exits 0 whether the run produced 29 verdicts or none, so no
+exit code distinguishes a working release from an empty one.
+
+Before v0.8.0, an EU AI Act evaluation reported 4 verdicts out of 29 and UK,
+NIST, BFS, legal, healthcare and education evaluations reported none at all,
+while the process exited successfully. This script asserts on the number of
+verdicts, which is the only signal that separates those two outcomes.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from importlib import resources
+from pathlib import Path
+
+# The library logs at INFO through evaluation; the report below is the output.
+logging.disable(logging.WARNING)
+
+# Floors: 80% of the count declared today, never below 1. The same rule and the
+# same values as MINIMUM_EXPECTED in tests/test_framework_golden.py, which
+# carries the reasoning; the two cannot import from each other because this runs
+# against an installed wheel with no repository on the path.
+#
+# This covers the eight frameworks a release most needs to prove, not all
+# fourteen. A packaging failure that drops the policy library fails every
+# framework at once, so eight demonstrates it as well as fourteen and keeps the
+# release gate quick. The golden tests cover all fourteen, where the risk being
+# checked is delivery rather than packaging.
+MINIMUM_VERDICTS = {
+ "eu_ai_act": 23,
+ "uk": 4,
+ "nist": 4,
+ "bfs": 3,
+ "legal": 2,
+ "global": 4,
+ "healthcare": 1,
+ "education": 4,
+}
+
+
+def fail(message: str) -> None:
+ print(f"FAIL: {message}", file=sys.stderr)
+ sys.exit(1)
+
+
+def main() -> int:
+ try:
+ import aicertify
+ from aicertify.opa_core.decision_index import load_index
+ from aicertify.opa_core.evaluator import OpaEvaluator
+ from aicertify.opa_core.extraction import (
+ _package_values_from,
+ extract_results_from_packages,
+ )
+ except Exception as exc: # noqa: BLE001 - any import failure is a failed release
+ fail(f"cannot import the installed package: {exc!r}")
+
+ installed = Path(aicertify.__file__).resolve().parent
+ print(f"aicertify {aicertify.__version__} from {installed}")
+
+ # Importing from a source checkout would defeat the purpose: the files
+ # under test would be the repository's, not the wheel's.
+ if (installed.parent / "pyproject.toml").exists():
+ fail(
+ f"aicertify was imported from a source checkout at {installed.parent}, "
+ "not from an installed wheel. Run this with a clean virtualenv."
+ )
+
+ with resources.as_file(
+ resources.files("aicertify") / "_demo" / "sample_contract.json"
+ ) as p:
+ if not p.exists():
+ fail("the wheel does not carry aicertify/_demo/sample_contract.json")
+ contract = json.loads(p.read_text())
+
+ evaluator = OpaEvaluator()
+ evaluator.load_policies()
+ policy_dir = evaluator.policy_loader.get_policy_dir()
+ print(f"policy library: {policy_dir}")
+
+ rego = list(Path(policy_dir).rglob("*.rego"))
+ if len(rego) < 50:
+ fail(f"only {len(rego)} .rego files in the installed policy directory")
+ print(f"{len(rego)} .rego files present\n")
+
+ index = load_index(policy_dir)
+
+ print(f"{'framework':<12}{'declared':>9}{'delivered':>10}{'floor':>7} result")
+ failures = []
+ for framework, floor in MINIMUM_VERDICTS.items():
+ folders = evaluator.find_matching_policy_folders(framework)
+ if not folders:
+ failures.append(f"{framework}: no policy folder matches this name")
+ print(f"{framework:<12}{'-':>9}{'-':>10}{floor:>7} NO FOLDER")
+ continue
+ prefix = folders[0].split("opa_policies/")[-1].replace("/", ".")
+
+ try:
+ raw = evaluator.evaluate_by_folder_name(framework, contract)
+ except Exception as exc: # noqa: BLE001
+ failures.append(f"{framework}: evaluation raised {exc!r}")
+ print(f"{framework:<12}{'-':>9}{'-':>10}{floor:>7} ERROR {exc!r}")
+ continue
+
+ if isinstance(raw, dict) and "error" in raw:
+ failures.append(f"{framework}: {raw['error']}")
+ print(f"{framework:<12}{'-':>9}{'-':>10}{floor:>7} ERROR {raw['error']}")
+ continue
+
+ # What the library says should arrive, read from coverage.json rather
+ # than from the packages that came back. A package that fails to
+ # evaluate is absent from the results, so counting only what returned
+ # compares a number with itself and always agrees.
+ declared = {
+ pkg
+ for pkg, d in index.items()
+ if pkg.startswith(prefix + ".") and d.reports_a_verdict
+ }
+ packages, _ = _package_values_from(raw)
+ verdicts = extract_results_from_packages(packages, policy_dir)
+
+ problems = []
+ if len(declared) < floor:
+ problems.append(
+ f"only {len(declared)} policies declare a verdict, floor is {floor}"
+ )
+ if len(verdicts) != len(declared):
+ silent = sorted(
+ declared - {p for p in packages if p.startswith(prefix + ".")}
+ )
+ problems.append(
+ f"{len(declared)} declared, {len(verdicts)} delivered; "
+ f"silent packages: {silent or 'none'}"
+ )
+ for problem in problems:
+ failures.append(f"{framework}: {problem}")
+ status = "ok" if not problems else "FAIL"
+ print(
+ f"{framework:<12}{len(declared):>9}{len(verdicts):>10}{floor:>7} {status}"
+ )
+
+ print()
+ if failures:
+ for f in failures:
+ print(f"FAIL: {f}", file=sys.stderr)
+ print(
+ f"\n{len(failures)} framework(s) did not deliver verdicts from the installed wheel. "
+ "Do not publish this build.",
+ file=sys.stderr,
+ )
+ return 1
+
+ print(
+ f"All {len(MINIMUM_VERDICTS)} frameworks delivered every verdict they declare, "
+ f"and each is above its floor."
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tests/test_framework_golden.py b/tests/test_framework_golden.py
new file mode 100644
index 0000000..e76ad8d
--- /dev/null
+++ b/tests/test_framework_golden.py
@@ -0,0 +1,289 @@
+"""
+Golden tests: every advertised framework must deliver the verdicts it declares.
+
+v0.8.0 fixed a defect where an EU AI Act evaluation reported 4 verdicts out of
+29, and UK, NIST, BFS, legal, healthcare and education evaluations reported none
+at all, while the process exited successfully. AICertify was querying
+`data..report_output`, which only four policies define. Nothing failed,
+because nothing asserted that a framework produces verdicts. The CLI still
+prints "OPA Evaluation: Successful" and exits 0 whether a run produced 29
+verdicts or none, so no exit code separates a working release from an empty one.
+
+These tests assert it, in three layers, because each catches something the
+others let through.
+
+ 1. The library is present. declared >= a floor per framework.
+ 2. Everything declared arrives. delivered == declared.
+ 3. Nothing is unaccounted for. every evaluated package is known to the index.
+
+What "declared" is measured against decides whether layer 2 works at all. It is
+taken from gopal's coverage.json, filtered to the framework, and never from the
+packages the evaluation returned. Deriving it from the returned packages is
+fail-open in a way that is easy to miss: a package that fails to evaluate is
+absent from the results and therefore absent from the expectation too, so the
+count agrees with itself and the test passes.
+
+That is not hypothetical. industry_specific.education.v1.fairness_and_equity
+raised an eval-time conflict and returned nothing, and the two counts agreed at
+4 and 4. Against the index, which says 5 packages declare a verdict for
+education, the same run reads 5 declared and 4 delivered.
+
+Layer 1 exists because layer 2 is vacuous on its own. With the policy submodule
+missing or empty, nothing is declared and nothing is delivered, and
+`delivered == declared` holds at zero: a green result meaning the question was
+never asked.
+
+Layer 3 covers the other direction. Layers 1 and 2 both read the index to decide
+what to expect, so a policy present in the tree but absent from coverage.json is
+invisible to both: never expected, never delivered, never missed. Layer 3
+compares against what OPA actually evaluated instead.
+
+These are slow. Each framework is a real OPA evaluation over the vendored policy
+library, which is what makes them worth having: no mock can regress the way the
+real query did.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from aicertify.opa_core.decision_index import load_index
+from aicertify.opa_core.evaluator import OpaEvaluator
+from aicertify.opa_core.extraction import (
+ _package_values_from,
+ extract_results_from_packages,
+)
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+POLICY_DIR = REPO_ROOT / "aicertify" / "opa_policies"
+COVERAGE_FILE = POLICY_DIR / "docs" / "coverage" / "coverage.json"
+CONTRACT_FILE = REPO_ROOT / "aicertify" / "_demo" / "sample_contract.json"
+
+#: A clone without --recursive has no policies to read. Skipping is right for a
+#: contributor's first checkout; CI fetches submodules and installs OPA, so
+#: these do run there, and a green CI without them would be the vacuous pass
+#: this file exists to prevent.
+needs_policies = pytest.mark.skipif(
+ not COVERAGE_FILE.is_file(),
+ reason="gopal submodule not checked out; run: git submodule update --init",
+)
+
+
+# Release-blocking floors: 80% of the count declared today, never below 1.
+#
+# One rule rather than fourteen judgements, and it is the only guard against the
+# policy library shrinking. Layer 2 does not cover that case: a policy removed
+# from gopal outright drops out of both the declared and the delivered count, so
+# the two still agree and the delivery check passes. The floor is what notices.
+#
+# 80% leaves room for a framework to be reorganised or a policy to be retired
+# without failing a release, while a framework being gutted or a submodule
+# pinned far behind still fails. Frameworks with a single policy floor at 1,
+# which is as meaningful as a floor on one policy can be: it exists or it does
+# not.
+#
+# Declared counts measured against aicertify/_demo/sample_contract.json and the
+# pinned submodule. Regenerate with the same evaluation if the library moves.
+MINIMUM_EXPECTED: dict[str, int] = {
+ # framework floor declared today
+ "eu_ai_act": 23, # 29
+ "aviation": 9, # 12
+ "uk": 4, # 6
+ "education": 4, # 5
+ "nist": 4, # 5
+ "global": 4, # 5
+ "bfs": 3, # 4
+ "operational": 3, # 4
+ "legal": 2, # 3
+ "healthcare": 1, # 2
+ "standards": 1, # 2
+ "india": 1, # 1
+ "brazil": 1, # 1
+ "automotive": 1, # 1
+}
+
+#: Frameworks known to deliver fewer verdicts than they declare, with the cause.
+#: Empty, and the mechanism is kept for the next one.
+#:
+#: Strict, so an exemption cannot outlive the defect: once the gap closes the
+#: test passes, and pytest reports an unexpected pass as a failure, forcing the
+#: entry out rather than leaving it as a permanent exception. That is how the
+#: education entry was removed. It covered
+#: industry_specific.education.v1.fairness_and_equity, which raised
+#: eval_conflict_error and delivered nothing for three policies; when the gopal
+#: submodule was bumped past the fix, this failed with XPASS(strict) and printed
+#: the instruction to delete it.
+KNOWN_DELIVERY_GAPS: dict[str, str] = {}
+
+
+def _delivery_params():
+ """Framework parameters for layer 2, with known gaps marked xfail(strict)."""
+ for framework in sorted(MINIMUM_EXPECTED):
+ reason = KNOWN_DELIVERY_GAPS.get(framework)
+ marks = [pytest.mark.xfail(reason=reason, strict=True)] if reason else []
+ yield pytest.param(framework, marks=marks)
+
+
+@pytest.fixture(scope="module")
+def contract() -> dict:
+ return json.loads(CONTRACT_FILE.read_text())
+
+
+@pytest.fixture(scope="module")
+def evaluator() -> OpaEvaluator:
+ ev = OpaEvaluator()
+ ev.load_policies()
+ return ev
+
+
+@pytest.fixture(scope="module")
+def index() -> dict:
+ return load_index(str(POLICY_DIR))
+
+
+def _package_prefix(evaluator: OpaEvaluator, framework: str) -> str:
+ """
+ The package prefix for a framework, taken from the folder the evaluator
+ actually resolves rather than from a hand-written map, so the two cannot
+ disagree about which policies belong to a framework.
+ """
+ folders = evaluator.find_matching_policy_folders(framework)
+ assert folders, f"{framework}: no policy folder matches this name"
+ return folders[0].split("opa_policies/")[-1].replace("/", ".")
+
+
+def _declared(index: dict, prefix: str) -> list[str]:
+ """Packages the policy library says reach a verdict for this framework."""
+ return [
+ pkg
+ for pkg, d in index.items()
+ if pkg.startswith(prefix + ".") and d.reports_a_verdict
+ ]
+
+
+#: One real OPA evaluation per framework, shared by the layers that need it.
+#: Three layers over fourteen frameworks is forty-two tests, and evaluating the
+#: library afresh for each would be forty-two full runs to answer fourteen
+#: questions. The fixtures below are module-scoped for the same reason.
+_EVALUATIONS: dict[str, tuple] = {}
+
+
+def _delivered(evaluator: OpaEvaluator, framework: str, contract: dict):
+ """(evaluated packages, verdicts) from a real OPA run, evaluated once."""
+ if framework not in _EVALUATIONS:
+ raw = evaluator.evaluate_by_folder_name(framework, contract)
+ assert not (
+ isinstance(raw, dict) and "error" in raw
+ ), f"{framework}: evaluation failed: {raw.get('error')}"
+ packages, _ = _package_values_from(raw)
+ _EVALUATIONS[framework] = (
+ packages,
+ extract_results_from_packages(packages, str(POLICY_DIR)),
+ )
+ return _EVALUATIONS[framework]
+
+
+@needs_policies
+@pytest.mark.slow
+class TestEveryAdvertisedFrameworkDeliversVerdicts:
+ @pytest.mark.parametrize("framework", sorted(MINIMUM_EXPECTED))
+ def test_the_policy_library_is_present(self, framework, evaluator, index):
+ """
+ Layer 1. Without this, the delivery check below passes against an empty
+ policy directory, which is the shape of the defect it guards.
+ """
+ declared = _declared(index, _package_prefix(evaluator, framework))
+ floor = MINIMUM_EXPECTED[framework]
+ assert len(declared) >= floor, (
+ f"{framework}: only {len(declared)} policies declare a verdict, "
+ f"floor is {floor}. The policy submodule may be missing, truncated "
+ f"or pinned behind."
+ )
+
+ @pytest.mark.parametrize("framework", _delivery_params())
+ def test_every_declared_verdict_is_delivered(
+ self, framework, evaluator, contract, index
+ ):
+ """
+ Layer 2, and the regression itself: 26 of the EU AI Act's 29 policies
+ declared a verdict, delivered nothing, and the run exited successfully.
+ """
+ prefix = _package_prefix(evaluator, framework)
+ declared = set(_declared(index, prefix))
+ packages, verdicts = _delivered(evaluator, framework, contract)
+
+ # Verdicts carry titles, so the shortfall is reported by package: a
+ # count alone does not say which policies went missing.
+ evaluated = {p for p in packages if p.startswith(prefix + ".")}
+ silent = sorted(declared - evaluated)
+
+ assert len(verdicts) == len(declared), (
+ f"{framework}: {len(declared)} policies declare a verdict but "
+ f"{len(verdicts)} were delivered. "
+ f"Packages that declare one and returned nothing: {silent or 'none'}. "
+ f"A package returning nothing is usually an evaluation-time error; "
+ f"run gopal's scripts/check-eval-conflicts.sh."
+ )
+
+ @pytest.mark.parametrize("framework", sorted(MINIMUM_EXPECTED))
+ def test_no_evaluated_package_is_unknown_to_the_index(
+ self, framework, evaluator, contract, index
+ ):
+ """
+ Layer 3. A package OPA evaluates but coverage.json does not describe is
+ dropped without trace, and the layers above cannot see it because they
+ both read the index to decide what to expect.
+ """
+ packages, _ = _delivered(evaluator, framework, contract)
+ unknown = sorted(p for p in packages if p not in index)
+ assert not unknown, (
+ f"{framework}: {len(unknown)} evaluated package(s) are absent from "
+ f"gopal's coverage.json and are silently excluded from every report: "
+ f"{unknown[:5]}. Regenerate coverage.json in gopal, or bump the "
+ f"submodule to a commit that includes them."
+ )
+
+
+@needs_policies
+class TestTheGoldenTableItself:
+ """
+ The floors are only useful if they cover what the product advertises. A
+ framework left out of the table is tested by nothing, which is how the
+ original defect went unnoticed across six frameworks at once.
+ """
+
+ def test_the_table_is_populated(self):
+ assert MINIMUM_EXPECTED, (
+ "MINIMUM_EXPECTED is empty, so every golden test above is "
+ "parametrised over nothing and passes without evaluating anything."
+ )
+
+ def test_no_framework_has_a_zero_floor(self):
+ zeroes = sorted(f for f, n in MINIMUM_EXPECTED.items() if n < 1)
+ assert not zeroes, (
+ f"{zeroes} have a floor below 1, which lets the delivery check pass "
+ f"against an empty policy library."
+ )
+
+ def test_known_gaps_name_a_framework_in_the_table(self):
+ """
+ A gap entry for a framework the table does not cover marks nothing, so
+ the exemption would look applied while the framework went unchecked.
+ """
+ stray = sorted(set(KNOWN_DELIVERY_GAPS) - set(MINIMUM_EXPECTED))
+ assert not stray, (
+ f"{stray} are listed in KNOWN_DELIVERY_GAPS but absent from "
+ f"MINIMUM_EXPECTED, so the exemption applies to no test."
+ )
+
+ def test_every_framework_resolves_to_a_policy_folder(self, evaluator):
+ unresolved = sorted(
+ f for f in MINIMUM_EXPECTED if not evaluator.find_matching_policy_folders(f)
+ )
+ assert not unresolved, (
+ f"{unresolved} have a floor but match no policy folder, so the "
+ f"golden tests for them would error rather than check anything."
+ )
diff --git a/tests/test_opa_invocation.py b/tests/test_opa_invocation.py
index 7f592ee..47961d9 100644
--- a/tests/test_opa_invocation.py
+++ b/tests/test_opa_invocation.py
@@ -149,3 +149,54 @@ def run(extra_flags):
assert (
'"result"' in with_flags.stdout
), f"expected a verdict for {query}, got: {with_flags.stdout[:400]}"
+
+
+class TestOpaPathSurvivesCI:
+ """
+ GitHub Actions sets CI=true, which set skip_opa_check, which left
+ OpaEvaluator.opa_path as None even with the binary installed.
+
+ None then went into argv[0]. Every call through evaluate_policy raised
+ "sequence item 0: expected str instance, NoneType found", and the folder
+ evaluation reported "No valid results from any policy evaluation". The other
+ path, _evaluate_with_local_opa, checks for None and returns a mock result,
+ so a run under CI reported fabricated verdicts as real ones.
+
+ Nothing caught it because the tests that exercise OPA resolve the binary
+ themselves with shutil.which and invoke it directly, so they never went
+ through the evaluator's own path.
+ """
+
+ def test_opa_path_resolves_when_ci_is_set(self, monkeypatch):
+ import shutil
+
+ from aicertify.opa_core.evaluator import OpaEvaluator
+
+ if shutil.which("opa") is None:
+ pytest.skip("opa binary not on PATH")
+
+ monkeypatch.setenv("CI", "true")
+ assert OpaEvaluator().opa_path is not None, (
+ "opa_path is None with CI set, so argv[0] is None and every "
+ "evaluation fails or returns a mock result"
+ )
+
+ def test_skipping_the_check_still_finds_the_binary(self, monkeypatch):
+ """The flag exists so a missing binary does not abort startup, not so a
+ present one goes unused."""
+ import shutil
+
+ from aicertify.opa_core.evaluator import OpaEvaluator
+
+ if shutil.which("opa") is None:
+ pytest.skip("opa binary not on PATH")
+
+ monkeypatch.delenv("CI", raising=False)
+ assert OpaEvaluator(skip_opa_check=True).opa_path == shutil.which("opa")
+
+ def test_external_server_still_needs_no_local_binary(self, monkeypatch):
+ """An external server is the one case where no local path is correct."""
+ from aicertify.opa_core.evaluator import OpaEvaluator
+
+ monkeypatch.delenv("CI", raising=False)
+ assert OpaEvaluator(use_external_server=True).opa_path is None