From acb168f50887642b02de0630da2b78b17740645d Mon Sep 17 00:00:00 2001 From: "kapil.madan" Date: Mon, 31 Aug 2026 12:18:50 +0530 Subject: [PATCH] fix(examples): stop the examples spending money, and gate README flag drift Two things a first-time contributor hits before writing a line. The three example run.py scripts call OpenAI. DeepEval judges toxicity with an LLM and activates off the presence of OPENAI_API_KEY, so anyone with a key in their shell gets billed for running an example, and anyone with an exhausted quota watches nine retries per interaction scroll past and concludes the example is broken. The bundled demo already guards against this by popping the key unless --with-llm-metrics is passed; the examples never got the same treatment. They do now, opting in through AICERTIFY_WITH_LLM_METRICS=1, which each example README documents. customer-support-bot goes from a wall of 429s to zero OpenAI calls and still produces its report. The guard has to run before `from aicertify import ...`, because the evaluator stack reads the key at import time. That puts the import below module-level code, which ruff flags as E402; the ordering is load-bearing, so it carries a noqa and a comment saying why rather than being reordered into correctness bug. Separately, the README flag tables had drifted in both directions: --evaluators removed in #91, still documented in all four translations --params has always existed, documented in none of the five READMEs Neither shows up in a test, a lint or a build. A reader following the Japanese README gets `unrecognized arguments` and reasonably concludes the tool is broken. tests/test_readme_flags.py now asserts every documented flag exists, across all five languages, and that the English table covers every `evaluate` flag. Mutation-tested in both directions before committing. The translations are still missing a --params row. That needs a native speaker rather than a machine translation nobody can vouch for, so the test does not demand it and it stays a contribution task. --- README.hi-IN.md | 1 - README.ja-JP.md | 1 - README.ko-KR.md | 1 - README.md | 1 + README.zh-CN.md | 1 - examples/customer-support-bot/README.md | 14 +++ examples/customer-support-bot/run.py | 23 ++++- examples/healthcare-triage-bot/README.md | 14 +++ examples/healthcare-triage-bot/run.py | 23 ++++- examples/hiring-screening-bot/README.md | 14 +++ examples/hiring-screening-bot/run.py | 23 ++++- tests/test_readme_flags.py | 107 +++++++++++++++++++++++ 12 files changed, 216 insertions(+), 7 deletions(-) create mode 100644 tests/test_readme_flags.py diff --git a/README.hi-IN.md b/README.hi-IN.md index 7bb15cc..184d9fb 100644 --- a/README.hi-IN.md +++ b/README.hi-IN.md @@ -208,7 +208,6 @@ python -m aicertify.cli \ | `--contract` | AI एप्लिकेशन कॉन्ट्रैक्ट JSON का पथ | | `--policy` | जिसके विरुद्ध मूल्यांकन करना है उस OPA पॉलिसी फ़ोल्डर का पथ | | `--report-format` | `pdf`, `markdown`, `json`, `html` (डिफ़ॉल्ट: `pdf`) | -| `--evaluators` | विशिष्ट evaluators तक सीमित करें (जैसे `Fairness ContentSafety`) | | `--output-dir` | जहाँ रिपोर्ट्स लैंड होती हैं (डिफ़ॉल्ट: `./reports`) | | `--verbose` | वर्बोज़ लॉगिंग | diff --git a/README.ja-JP.md b/README.ja-JP.md index 6eb15c0..e2603bb 100644 --- a/README.ja-JP.md +++ b/README.ja-JP.md @@ -208,7 +208,6 @@ python -m aicertify.cli \ | `--contract` | AI アプリケーション契約 JSON のパス | | `--policy` | 評価対象とする OPA ポリシーフォルダのパス | | `--report-format` | `pdf`、`markdown`、`json`、`html` (デフォルト: `pdf`) | -| `--evaluators` | 特定の評価器に限定 (例: `Fairness ContentSafety`) | | `--output-dir` | レポート出力先 (デフォルト: `./reports`) | | `--verbose` | 詳細ログ出力 | diff --git a/README.ko-KR.md b/README.ko-KR.md index 1811bd7..5c23f41 100644 --- a/README.ko-KR.md +++ b/README.ko-KR.md @@ -208,7 +208,6 @@ python -m aicertify.cli \ | `--contract` | AI 애플리케이션 계약 JSON 파일 경로 | | `--policy` | 평가에 사용할 OPA 정책 폴더 경로 | | `--report-format` | `pdf`, `markdown`, `json`, `html` (기본값: `pdf`) | -| `--evaluators` | 특정 평가기로 제한 (예: `Fairness ContentSafety`) | | `--output-dir` | 리포트 출력 위치 (기본값: `./reports`) | | `--verbose` | 상세 로깅 | diff --git a/README.md b/README.md index db0042e..5907806 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,7 @@ Useful flags: | `--policy` | Framework name or path to an OPA policy folder | | `--report-format` | `pdf`, `markdown`, `json`, `html` (default: `pdf`) | | `--output-dir` | Where reports land (default: `./reports`) | +| `--params` | JSON string or file overriding a policy's documented thresholds | | `--verbose` | Verbose logging | See [`examples/quickstart.py`](examples/quickstart.py) for the full Python API. diff --git a/README.zh-CN.md b/README.zh-CN.md index 20cfc7e..ed60029 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -211,7 +211,6 @@ python -m aicertify.cli \ | `--contract` | AI 应用合约 JSON 的路径 | | `--policy` | 用于评估的 OPA 策略目录路径 | | `--report-format` | `pdf`、`markdown`、`json`、`html`(默认:`pdf`) | -| `--evaluators` | 限定使用特定评估器(例如 `Fairness ContentSafety`) | | `--output-dir` | 报告输出目录(默认:`./reports`) | | `--verbose` | 输出详细日志 | diff --git a/examples/customer-support-bot/README.md b/examples/customer-support-bot/README.md index 9f96ac3..b4d1e32 100644 --- a/examples/customer-support-bot/README.md +++ b/examples/customer-support-bot/README.md @@ -59,3 +59,17 @@ For more elaborate setups, see the sibling examples: - [`healthcare-triage-bot/`](../healthcare-triage-bot/) — medical AI evaluated for patient safety - [`hiring-screening-bot/`](../hiring-screening-bot/) — recruiting AI evaluated for fair-employment compliance + +### LLM-judged metrics are off by default + +This example computes what it can locally and skips the toxicity and fairness +metrics that DeepEval judges with an LLM, so it costs nothing and behaves the +same on every machine. `OPENAI_API_KEY` is ignored even if your shell has one. + +To opt in: + +```bash +AICERTIFY_WITH_LLM_METRICS=1 python examples/customer-support-bot/run.py +``` + +That makes billable API calls and takes noticeably longer. diff --git a/examples/customer-support-bot/run.py b/examples/customer-support-bot/run.py index 6d470cf..1a48dbb 100644 --- a/examples/customer-support-bot/run.py +++ b/examples/customer-support-bot/run.py @@ -12,10 +12,31 @@ import asyncio import json +import os import sys from pathlib import Path -from aicertify import application, regulations +# The LLM-judged evaluators activate off the presence of OPENAI_API_KEY, and +# this example is something a first-time reader copies and runs. Hide the key +# unless they ask for those metrics, so the example behaves the same on every +# machine and never spends somebody's money without being asked. Same treatment +# the bundled demo gets in aicertify/_demo/runner.py. +# +# Without it, a reader with an exhausted quota watches nine retries per +# interaction scroll past and reasonably concludes the example is broken. +WITH_LLM_METRICS = os.environ.get("AICERTIFY_WITH_LLM_METRICS") == "1" +if not WITH_LLM_METRICS: + os.environ.pop("OPENAI_API_KEY", None) + +# Not exposed, to match examples/quickstart.py and keep the run reproducible +# across machines with and without GPUs. +os.environ.setdefault("CUDA_VISIBLE_DEVICES", "") + + +# Imported after the environment guard above, not before: the evaluator +# stack reads OPENAI_API_KEY at import time, so popping it afterwards +# would be too late. Same ordering as aicertify/_demo/runner.py. +from aicertify import application, regulations # noqa: E402 EXAMPLE_DIR = Path(__file__).resolve().parent CONTRACT_PATH = EXAMPLE_DIR / "input_contract.json" diff --git a/examples/healthcare-triage-bot/README.md b/examples/healthcare-triage-bot/README.md index d54ae55..e902735 100644 --- a/examples/healthcare-triage-bot/README.md +++ b/examples/healthcare-triage-bot/README.md @@ -60,3 +60,17 @@ A green report does **not** clear the application for clinical deployment. It de | `policy_config.yaml` | EU AI Act + healthcare/v1 policy bundle | | `run.py` | Runnable AICertify Python API script | | `expected_report.md` | What a successful run looks like | + +### LLM-judged metrics are off by default + +This example computes what it can locally and skips the toxicity and fairness +metrics that DeepEval judges with an LLM, so it costs nothing and behaves the +same on every machine. `OPENAI_API_KEY` is ignored even if your shell has one. + +To opt in: + +```bash +AICERTIFY_WITH_LLM_METRICS=1 python examples/healthcare-triage-bot/run.py +``` + +That makes billable API calls and takes noticeably longer. diff --git a/examples/healthcare-triage-bot/run.py b/examples/healthcare-triage-bot/run.py index d0ea46b..b0cb492 100644 --- a/examples/healthcare-triage-bot/run.py +++ b/examples/healthcare-triage-bot/run.py @@ -15,10 +15,31 @@ import asyncio import json +import os import sys from pathlib import Path -from aicertify import application, regulations +# The LLM-judged evaluators activate off the presence of OPENAI_API_KEY, and +# this example is something a first-time reader copies and runs. Hide the key +# unless they ask for those metrics, so the example behaves the same on every +# machine and never spends somebody's money without being asked. Same treatment +# the bundled demo gets in aicertify/_demo/runner.py. +# +# Without it, a reader with an exhausted quota watches nine retries per +# interaction scroll past and reasonably concludes the example is broken. +WITH_LLM_METRICS = os.environ.get("AICERTIFY_WITH_LLM_METRICS") == "1" +if not WITH_LLM_METRICS: + os.environ.pop("OPENAI_API_KEY", None) + +# Not exposed, to match examples/quickstart.py and keep the run reproducible +# across machines with and without GPUs. +os.environ.setdefault("CUDA_VISIBLE_DEVICES", "") + + +# Imported after the environment guard above, not before: the evaluator +# stack reads OPENAI_API_KEY at import time, so popping it afterwards +# would be too late. Same ordering as aicertify/_demo/runner.py. +from aicertify import application, regulations # noqa: E402 EXAMPLE_DIR = Path(__file__).resolve().parent CONTRACT_PATH = EXAMPLE_DIR / "input_contract.json" diff --git a/examples/hiring-screening-bot/README.md b/examples/hiring-screening-bot/README.md index 496c7f5..fd81116 100644 --- a/examples/hiring-screening-bot/README.md +++ b/examples/hiring-screening-bot/README.md @@ -59,3 +59,17 @@ See [docs/why-aicertify.md](../../docs/why-aicertify.md) for what AICertify *doe | `policy_config.yaml` | EU AI Act + fair-lending proxy + global bundle | | `run.py` | Runnable AICertify Python API script | | `expected_report.md` | What a successful run looks like | + +### LLM-judged metrics are off by default + +This example computes what it can locally and skips the toxicity and fairness +metrics that DeepEval judges with an LLM, so it costs nothing and behaves the +same on every machine. `OPENAI_API_KEY` is ignored even if your shell has one. + +To opt in: + +```bash +AICERTIFY_WITH_LLM_METRICS=1 python examples/hiring-screening-bot/run.py +``` + +That makes billable API calls and takes noticeably longer. diff --git a/examples/hiring-screening-bot/run.py b/examples/hiring-screening-bot/run.py index dcecb59..58041bb 100644 --- a/examples/hiring-screening-bot/run.py +++ b/examples/hiring-screening-bot/run.py @@ -17,10 +17,31 @@ import asyncio import json +import os import sys from pathlib import Path -from aicertify import application, regulations +# The LLM-judged evaluators activate off the presence of OPENAI_API_KEY, and +# this example is something a first-time reader copies and runs. Hide the key +# unless they ask for those metrics, so the example behaves the same on every +# machine and never spends somebody's money without being asked. Same treatment +# the bundled demo gets in aicertify/_demo/runner.py. +# +# Without it, a reader with an exhausted quota watches nine retries per +# interaction scroll past and reasonably concludes the example is broken. +WITH_LLM_METRICS = os.environ.get("AICERTIFY_WITH_LLM_METRICS") == "1" +if not WITH_LLM_METRICS: + os.environ.pop("OPENAI_API_KEY", None) + +# Not exposed, to match examples/quickstart.py and keep the run reproducible +# across machines with and without GPUs. +os.environ.setdefault("CUDA_VISIBLE_DEVICES", "") + + +# Imported after the environment guard above, not before: the evaluator +# stack reads OPENAI_API_KEY at import time, so popping it afterwards +# would be too late. Same ordering as aicertify/_demo/runner.py. +from aicertify import application, regulations # noqa: E402 EXAMPLE_DIR = Path(__file__).resolve().parent CONTRACT_PATH = EXAMPLE_DIR / "input_contract.json" diff --git a/tests/test_readme_flags.py b/tests/test_readme_flags.py new file mode 100644 index 0000000..b964228 --- /dev/null +++ b/tests/test_readme_flags.py @@ -0,0 +1,107 @@ +"""Every command-line flag a README documents must exist. + +This exists because two flags drifted in one week and neither was caught. + +`--evaluators` was removed in #91 for never having affected a run. The English +README dropped it; the four translations kept documenting it, in four +languages, for a flag that no longer parsed. And `--params` has always existed +and was documented nowhere at all. + +Neither shows up in a test run, a lint, or a build. A reader following the +translated README simply gets `unrecognized arguments` and concludes the tool +is broken. So the check belongs somewhere that runs on every commit, which is +here rather than in a workflow step, so it also fails on a laptop. + +The reverse direction is deliberately not asserted. A flag missing from a +translation needs a native speaker to add a row, and failing the build until +somebody translates a sentence would either block the release or invite a +machine translation nobody can vouch for. Undocumented flags are reported by +`test_english_readme_documents_every_flag` for English only, where we can +actually write the sentence. +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +import pytest + +from aicertify.cli import _build_parser + +REPO = Path(__file__).resolve().parent.parent +READMES = sorted(REPO.glob("README*.md")) + +# A row in a flag table: `| `--flag` | description |` +FLAG_ROW = re.compile(r"^\|\s*`(--[a-z][a-z-]*)`", re.MULTILINE) + + +def _real_flags() -> set[str]: + """Every option string the parser accepts, across all subcommands.""" + found: set[str] = set() + + def collect(parser: argparse.ArgumentParser) -> None: + for action in parser._actions: # noqa: SLF001 - argparse has no public API + found.update(opt for opt in action.option_strings if opt.startswith("--")) + if isinstance(action, argparse._SubParsersAction): # noqa: SLF001 + for sub in action.choices.values(): + collect(sub) + + collect(_build_parser()) + return found + + +def test_the_parser_exposes_flags_at_all() -> None: + """Guards the guard: if introspection breaks, the tests below pass vacuously.""" + flags = _real_flags() + assert "--contract" in flags, f"parser introspection returned {flags}" + assert len(flags) > 5 + + +@pytest.mark.parametrize("readme", READMES, ids=lambda p: p.name) +def test_readme_documents_no_flag_that_does_not_exist(readme: Path) -> None: + documented = set(FLAG_ROW.findall(readme.read_text(encoding="utf-8"))) + if not documented: + pytest.skip(f"{readme.name} documents no flags") + + unreal = sorted(documented - _real_flags()) + assert not unreal, ( + f"{readme.name} documents {unreal}, which the CLI does not accept. " + "A reader following it gets 'unrecognized arguments'." + ) + + +def _flags_of(command: str) -> set[str]: + """The `--flags` one subcommand accepts.""" + parser = _build_parser() + for action in parser._actions: # noqa: SLF001 + if isinstance(action, argparse._SubParsersAction): # noqa: SLF001 + sub = action.choices[command] + return { + opt + for a in sub._actions # noqa: SLF001 + for opt in a.option_strings + if opt.startswith("--") + } + raise AssertionError(f"no subcommand {command}") + + +def test_english_readme_documents_every_evaluate_flag() -> None: + """The flag table documents `evaluate`, so hold it to that command. + + Scoped rather than allow-listed. An allow-list of "flags documented + elsewhere" is a place to quietly park anything inconvenient, and it grows + until the check means nothing. `demo`, `init-contract`, `explain` and + `score-card` carry their own flags and are described in prose, so asserting + the table lists those too would only invite the exemption. + """ + readme = REPO / "README.md" + documented = set(FLAG_ROW.findall(readme.read_text(encoding="utf-8"))) + + # --help and --verbose are on every subcommand and are not worth a row. + missing = sorted(_flags_of("evaluate") - documented - {"--help", "--verbose"}) + assert not missing, ( + f"README.md documents the `evaluate` flag table but omits {missing}. " + "`--params` was missing from all five READMEs for exactly this reason." + )