Publish on a tag, prove the wheel delivers verdicts, and fix OPA being unusable in CI

.github/workflows/release.yaml

@@ -0,0 +1,164 @@
+name: Release to PyPI
+
+# GitHub had v0.8.0 from 28 August while PyPI served v0.7.3 from 14 May, so
+# `pip install aicertify` did not install what this repository represented as
+# the product. Publishing was a manual step and manual steps get skipped.
+#
+# A tag now publishes. The tag is the only trigger, the version in the tag must
+# match the version in pyproject.toml, and the wheel is installed into a clean
+# environment and made to produce real verdicts before anything is uploaded.
+#
+# Authentication is PyPI Trusted Publishing (OIDC). There is no API token in
+# this repository. Configure the publisher once at
+# https://pypi.org/manage/project/aicertify/settings/publishing/ with:
+#   owner        Principled-Evolution
+#   repository   aicertify
+#   workflow     release.yaml
+#   environment  pypi
+
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Build and smoke-test only; do not publish"
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build and verify the wheel
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      # The policy library is a submodule. Without it the wheel builds and
+      # installs but contains no .rego files, so every evaluation returns
+      # nothing while still exiting successfully.
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python -
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Read the version from pyproject.toml
+        id: version
+        run: |
+          version="$(poetry version --short)"
+          echo "version=${version}" >> "$GITHUB_OUTPUT"
+          echo "pyproject version is ${version}"
+
+      # A tag that disagrees with pyproject.toml is how the published version
+      # and the represented version come apart in the first place.
+      - name: Check the tag matches the version
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          tag="${GITHUB_REF#refs/tags/v}"
+          version="${{ steps.version.outputs.version }}"
+          if [ "${tag}" != "${version}" ]; then
+            echo "::error::tag v${tag} does not match pyproject version ${version}" >&2
+            exit 1
+          fi
+          echo "tag v${tag} matches pyproject version ${version}"
+
+      - name: Build sdist and wheel
+        run: poetry build
+
+      # The policies are data files, not code, so a packaging change can drop
+      # them silently. An empty policy directory is indistinguishable from a
+      # passing run at the CLI, so it is checked here.
+      - name: Check the wheel carries the policy library
+        run: |
+          wheel="$(ls dist/*.whl)"
+          rego="$(unzip -l "${wheel}" | grep -c '\.rego' || true)"
+          echo "${rego} .rego files in ${wheel}"
+          if [ "${rego}" -lt 50 ]; then
+            echo "::error::wheel contains ${rego} .rego files; the submodule is missing or unpackaged" >&2
+            exit 1
+          fi
+          unzip -l "${wheel}" | grep -q 'opa_policies/docs/coverage/coverage.json' || {
+            echo "::error::wheel has no coverage.json; verdict extraction reads it to find each policy's decision rule" >&2
+            exit 1
+          }
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  smoke:
+    name: Clean-install smoke test
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      # Deliberately no repository checkout beyond the smoke script: the point
+      # is to exercise what a user gets from `pip install aicertify`, not what
+      # a developer gets from a working tree. A source checkout would mask a
+      # missing package data file, because the file would be on disk anyway.
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            scripts/smoke_test_wheel.py
+          sparse-checkout-cone-mode: false
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Install OPA
+        env:
+          OPA_VERSION: v1.20.1
+        run: |
+          curl -L -o opa "https://openpolicyagent.org/downloads/${OPA_VERSION}/opa_linux_amd64"
+          chmod 755 opa
+          sudo mv opa /usr/local/bin/
+          opa version
+
+      - name: Install the built wheel into a clean environment
+        run: |
+          python -m venv /tmp/smoke
+          /tmp/smoke/bin/pip install --upgrade pip
+          /tmp/smoke/bin/pip install dist/*.whl
+
+      # Runs a real framework evaluation and asserts on the number of verdicts.
+      # The CLI exits 0 and prints "OPA Evaluation: Successful" whether it
+      # produced 29 verdicts or none, so the exit code cannot detect the
+      # regression this is here to catch.
+      - name: Evaluate real frameworks from the installed wheel
+        run: /tmp/smoke/bin/python scripts/smoke_test_wheel.py
+
+  publish:
+    name: Publish to PyPI
+    needs: [build, smoke]
+    if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/aicertify/${{ needs.build.outputs.version }}/
+    permissions:
+      id-token: write   # Trusted Publishing exchanges this for a PyPI token.
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@release/v1

README-pypi.md

@@ -11,6 +11,7 @@
 
 <p align="center">
   <a href="https://pypi.org/project/aicertify/"><img src="https://img.shields.io/pypi/v/aicertify?style=flat-square&color=blue" alt="PyPI"></a>
+  <a href="https://pepy.tech/project/aicertify"><img src="https://img.shields.io/pepy/dt/aicertify?style=flat-square" alt="Downloads"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml"><img src="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml/badge.svg" alt="CI"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/stargazers"><img src="https://img.shields.io/github/stars/Principled-Evolution/aicertify?style=flat-square" alt="Stars"></a>
   <a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.12-blue.svg?style=flat-square" alt="Python 3.12"></a>

README.hi-IN.md

@@ -19,6 +19,7 @@
 
 <p align="center">
   <a href="https://pypi.org/project/aicertify/"><img src="https://img.shields.io/pypi/v/aicertify?style=flat-square&color=blue" alt="PyPI"></a>
+  <a href="https://pepy.tech/project/aicertify"><img src="https://img.shields.io/pepy/dt/aicertify?style=flat-square" alt="Downloads"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml"><img src="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml/badge.svg" alt="CI"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/stargazers"><img src="https://img.shields.io/github/stars/Principled-Evolution/aicertify?style=flat-square" alt="Stars"></a>
   <a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.12-blue.svg?style=flat-square" alt="Python 3.12"></a>

README.ja-JP.md

@@ -19,6 +19,7 @@
 
 <p align="center">
   <a href="https://pypi.org/project/aicertify/"><img src="https://img.shields.io/pypi/v/aicertify?style=flat-square&color=blue" alt="PyPI"></a>
+  <a href="https://pepy.tech/project/aicertify"><img src="https://img.shields.io/pepy/dt/aicertify?style=flat-square" alt="ダウンロード"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml"><img src="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml/badge.svg" alt="CI"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/stargazers"><img src="https://img.shields.io/github/stars/Principled-Evolution/aicertify?style=flat-square" alt="Stars"></a>
   <a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.12-blue.svg?style=flat-square" alt="Python 3.12"></a>

README.ko-KR.md

@@ -19,6 +19,7 @@
 
 <p align="center">
   <a href="https://pypi.org/project/aicertify/"><img src="https://img.shields.io/pypi/v/aicertify?style=flat-square&color=blue" alt="PyPI"></a>
+  <a href="https://pepy.tech/project/aicertify"><img src="https://img.shields.io/pepy/dt/aicertify?style=flat-square" alt="다운로드"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml"><img src="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml/badge.svg" alt="CI"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/stargazers"><img src="https://img.shields.io/github/stars/Principled-Evolution/aicertify?style=flat-square" alt="Stars"></a>
   <a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.12-blue.svg?style=flat-square" alt="Python 3.12"></a>

README.md

@@ -19,6 +19,7 @@
 
 <p align="center">
   <a href="https://pypi.org/project/aicertify/"><img src="https://img.shields.io/pypi/v/aicertify?style=flat-square&color=blue" alt="PyPI"></a>
+  <a href="https://pepy.tech/project/aicertify"><img src="https://img.shields.io/pepy/dt/aicertify?style=flat-square" alt="Downloads"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml"><img src="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml/badge.svg" alt="CI"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/stargazers"><img src="https://img.shields.io/github/stars/Principled-Evolution/aicertify?style=flat-square" alt="Stars"></a>
   <a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.12-blue.svg?style=flat-square" alt="Python 3.12"></a>

README.zh-CN.md

@@ -19,6 +19,7 @@
 
 <p align="center">
   <a href="https://pypi.org/project/aicertify/"><img src="https://img.shields.io/pypi/v/aicertify?style=flat-square&color=blue" alt="PyPI"></a>
+  <a href="https://pepy.tech/project/aicertify"><img src="https://img.shields.io/pepy/dt/aicertify?style=flat-square" alt="下载量"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml"><img src="https://github.com/Principled-Evolution/aicertify/actions/workflows/aicertify-ci.yaml/badge.svg" alt="持续集成"></a>
   <a href="https://github.com/Principled-Evolution/aicertify/stargazers"><img src="https://img.shields.io/github/stars/Principled-Evolution/aicertify?style=flat-square" alt="Star 数"></a>
   <a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.12-blue.svg?style=flat-square" alt="Python 3.12"></a>

aicertify/opa_core/evaluator.py

@@ -128,11 +128,26 @@ def __init__(
         )
 
         self.policy_loader = PolicyLoader()
-        self.opa_path = (
-            None
-            if (use_external_server or skip_opa_check)
-            else self._verify_opa_installation()
-        )
+
+        # Skipping the installation check must not mean discarding the path.
+        #
+        # GitHub Actions sets CI=true, which set skip_opa_check and left
+        # opa_path as None. None then went into argv[0], so every call through
+        # evaluate_policy raised "sequence item 0: expected str instance,
+        # NoneType found" and the folder evaluation reported "No valid results
+        # from any policy evaluation". The path through
+        # _evaluate_with_local_opa is worse: it checks for None and returns a
+        # mock result, so a run in CI reported fabricated verdicts as real ones.
+        #
+        # The flag exists so a missing binary does not abort startup, not so a
+        # present one goes unused. Resolve it either way and let the call site
+        # fail on a real missing executable.
+        if use_external_server:
+            self.opa_path = None
+        elif skip_opa_check:
+            self.opa_path = shutil.which("opa") or os.environ.get("OPA_PATH") or "opa"
+        else:
+            self.opa_path = self._verify_opa_installation()
         self.use_external_server = use_external_server
         self.server_url = server_url
         self.policies_loaded = False

pyproject.toml

@@ -142,6 +142,12 @@ black = ">=26.3.1,<27.0.0"
 # broken, that autoload crashes pytest itself, regardless of whether any test
 # imports deepeval. Disabling it here is unrelated to test content.
 addopts = "-p no:plugins"
+markers = [
+    # Real OPA evaluations over the vendored policy library. Deselect with
+    # -m "not slow" for a fast loop; CI runs them, because they are the only
+    # tests that can catch a framework silently returning no verdicts.
+    "slow: runs a real OPA evaluation against the policy library",
+]
 
 [tool.ruff]
 line-length = 88