diff --git a/README.hi-IN.md b/README.hi-IN.md index 7bb15cc..184d9fb 100644 --- a/README.hi-IN.md +++ b/README.hi-IN.md @@ -208,7 +208,6 @@ python -m aicertify.cli \ | `--contract` | AI एप्लिकेशन कॉन्ट्रैक्ट JSON का पथ | | `--policy` | जिसके विरुद्ध मूल्यांकन करना है उस OPA पॉलिसी फ़ोल्डर का पथ | | `--report-format` | `pdf`, `markdown`, `json`, `html` (डिफ़ॉल्ट: `pdf`) | -| `--evaluators` | विशिष्ट evaluators तक सीमित करें (जैसे `Fairness ContentSafety`) | | `--output-dir` | जहाँ रिपोर्ट्स लैंड होती हैं (डिफ़ॉल्ट: `./reports`) | | `--verbose` | वर्बोज़ लॉगिंग | diff --git a/README.ja-JP.md b/README.ja-JP.md index 6eb15c0..e2603bb 100644 --- a/README.ja-JP.md +++ b/README.ja-JP.md @@ -208,7 +208,6 @@ python -m aicertify.cli \ | `--contract` | AI アプリケーション契約 JSON のパス | | `--policy` | 評価対象とする OPA ポリシーフォルダのパス | | `--report-format` | `pdf`、`markdown`、`json`、`html` (デフォルト: `pdf`) | -| `--evaluators` | 特定の評価器に限定 (例: `Fairness ContentSafety`) | | `--output-dir` | レポート出力先 (デフォルト: `./reports`) | | `--verbose` | 詳細ログ出力 | diff --git a/README.ko-KR.md b/README.ko-KR.md index 1811bd7..5c23f41 100644 --- a/README.ko-KR.md +++ b/README.ko-KR.md @@ -208,7 +208,6 @@ python -m aicertify.cli \ | `--contract` | AI 애플리케이션 계약 JSON 파일 경로 | | `--policy` | 평가에 사용할 OPA 정책 폴더 경로 | | `--report-format` | `pdf`, `markdown`, `json`, `html` (기본값: `pdf`) | -| `--evaluators` | 특정 평가기로 제한 (예: `Fairness ContentSafety`) | | `--output-dir` | 리포트 출력 위치 (기본값: `./reports`) | | `--verbose` | 상세 로깅 | diff --git a/README.md b/README.md index db0042e..5907806 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,7 @@ Useful flags: | `--policy` | Framework name or path to an OPA policy folder | | `--report-format` | `pdf`, `markdown`, `json`, `html` (default: `pdf`) | | `--output-dir` | Where reports land (default: `./reports`) | +| `--params` | JSON string or file overriding a policy's documented thresholds | | `--verbose` | Verbose logging | See [`examples/quickstart.py`](examples/quickstart.py) for the full Python API. diff --git a/README.zh-CN.md b/README.zh-CN.md index 20cfc7e..ed60029 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -211,7 +211,6 @@ python -m aicertify.cli \ | `--contract` | AI 应用合约 JSON 的路径 | | `--policy` | 用于评估的 OPA 策略目录路径 | | `--report-format` | `pdf`、`markdown`、`json`、`html`(默认:`pdf`) | -| `--evaluators` | 限定使用特定评估器(例如 `Fairness ContentSafety`) | | `--output-dir` | 报告输出目录(默认:`./reports`) | | `--verbose` | 输出详细日志 | diff --git a/examples/customer-support-bot/README.md b/examples/customer-support-bot/README.md index 9f96ac3..b4d1e32 100644 --- a/examples/customer-support-bot/README.md +++ b/examples/customer-support-bot/README.md @@ -59,3 +59,17 @@ For more elaborate setups, see the sibling examples: - [`healthcare-triage-bot/`](../healthcare-triage-bot/) — medical AI evaluated for patient safety - [`hiring-screening-bot/`](../hiring-screening-bot/) — recruiting AI evaluated for fair-employment compliance + +### LLM-judged metrics are off by default + +This example computes what it can locally and skips the toxicity and fairness +metrics that DeepEval judges with an LLM, so it costs nothing and behaves the +same on every machine. `OPENAI_API_KEY` is ignored even if your shell has one. + +To opt in: + +```bash +AICERTIFY_WITH_LLM_METRICS=1 python examples/customer-support-bot/run.py +``` + +That makes billable API calls and takes noticeably longer. diff --git a/examples/customer-support-bot/run.py b/examples/customer-support-bot/run.py index 6d470cf..1a48dbb 100644 --- a/examples/customer-support-bot/run.py +++ b/examples/customer-support-bot/run.py @@ -12,10 +12,31 @@ import asyncio import json +import os import sys from pathlib import Path -from aicertify import application, regulations +# The LLM-judged evaluators activate off the presence of OPENAI_API_KEY, and +# this example is something a first-time reader copies and runs. Hide the key +# unless they ask for those metrics, so the example behaves the same on every +# machine and never spends somebody's money without being asked. Same treatment +# the bundled demo gets in aicertify/_demo/runner.py. +# +# Without it, a reader with an exhausted quota watches nine retries per +# interaction scroll past and reasonably concludes the example is broken. +WITH_LLM_METRICS = os.environ.get("AICERTIFY_WITH_LLM_METRICS") == "1" +if not WITH_LLM_METRICS: + os.environ.pop("OPENAI_API_KEY", None) + +# Not exposed, to match examples/quickstart.py and keep the run reproducible +# across machines with and without GPUs. +os.environ.setdefault("CUDA_VISIBLE_DEVICES", "") + + +# Imported after the environment guard above, not before: the evaluator +# stack reads OPENAI_API_KEY at import time, so popping it afterwards +# would be too late. Same ordering as aicertify/_demo/runner.py. +from aicertify import application, regulations # noqa: E402 EXAMPLE_DIR = Path(__file__).resolve().parent CONTRACT_PATH = EXAMPLE_DIR / "input_contract.json" diff --git a/examples/healthcare-triage-bot/README.md b/examples/healthcare-triage-bot/README.md index d54ae55..e902735 100644 --- a/examples/healthcare-triage-bot/README.md +++ b/examples/healthcare-triage-bot/README.md @@ -60,3 +60,17 @@ A green report does **not** clear the application for clinical deployment. It de | `policy_config.yaml` | EU AI Act + healthcare/v1 policy bundle | | `run.py` | Runnable AICertify Python API script | | `expected_report.md` | What a successful run looks like | + +### LLM-judged metrics are off by default + +This example computes what it can locally and skips the toxicity and fairness +metrics that DeepEval judges with an LLM, so it costs nothing and behaves the +same on every machine. `OPENAI_API_KEY` is ignored even if your shell has one. + +To opt in: + +```bash +AICERTIFY_WITH_LLM_METRICS=1 python examples/healthcare-triage-bot/run.py +``` + +That makes billable API calls and takes noticeably longer. diff --git a/examples/healthcare-triage-bot/run.py b/examples/healthcare-triage-bot/run.py index d0ea46b..b0cb492 100644 --- a/examples/healthcare-triage-bot/run.py +++ b/examples/healthcare-triage-bot/run.py @@ -15,10 +15,31 @@ import asyncio import json +import os import sys from pathlib import Path -from aicertify import application, regulations +# The LLM-judged evaluators activate off the presence of OPENAI_API_KEY, and +# this example is something a first-time reader copies and runs. Hide the key +# unless they ask for those metrics, so the example behaves the same on every +# machine and never spends somebody's money without being asked. Same treatment +# the bundled demo gets in aicertify/_demo/runner.py. +# +# Without it, a reader with an exhausted quota watches nine retries per +# interaction scroll past and reasonably concludes the example is broken. +WITH_LLM_METRICS = os.environ.get("AICERTIFY_WITH_LLM_METRICS") == "1" +if not WITH_LLM_METRICS: + os.environ.pop("OPENAI_API_KEY", None) + +# Not exposed, to match examples/quickstart.py and keep the run reproducible +# across machines with and without GPUs. +os.environ.setdefault("CUDA_VISIBLE_DEVICES", "") + + +# Imported after the environment guard above, not before: the evaluator +# stack reads OPENAI_API_KEY at import time, so popping it afterwards +# would be too late. Same ordering as aicertify/_demo/runner.py. +from aicertify import application, regulations # noqa: E402 EXAMPLE_DIR = Path(__file__).resolve().parent CONTRACT_PATH = EXAMPLE_DIR / "input_contract.json" diff --git a/examples/hiring-screening-bot/README.md b/examples/hiring-screening-bot/README.md index 496c7f5..fd81116 100644 --- a/examples/hiring-screening-bot/README.md +++ b/examples/hiring-screening-bot/README.md @@ -59,3 +59,17 @@ See [docs/why-aicertify.md](../../docs/why-aicertify.md) for what AICertify *doe | `policy_config.yaml` | EU AI Act + fair-lending proxy + global bundle | | `run.py` | Runnable AICertify Python API script | | `expected_report.md` | What a successful run looks like | + +### LLM-judged metrics are off by default + +This example computes what it can locally and skips the toxicity and fairness +metrics that DeepEval judges with an LLM, so it costs nothing and behaves the +same on every machine. `OPENAI_API_KEY` is ignored even if your shell has one. + +To opt in: + +```bash +AICERTIFY_WITH_LLM_METRICS=1 python examples/hiring-screening-bot/run.py +``` + +That makes billable API calls and takes noticeably longer. diff --git a/examples/hiring-screening-bot/run.py b/examples/hiring-screening-bot/run.py index dcecb59..58041bb 100644 --- a/examples/hiring-screening-bot/run.py +++ b/examples/hiring-screening-bot/run.py @@ -17,10 +17,31 @@ import asyncio import json +import os import sys from pathlib import Path -from aicertify import application, regulations +# The LLM-judged evaluators activate off the presence of OPENAI_API_KEY, and +# this example is something a first-time reader copies and runs. Hide the key +# unless they ask for those metrics, so the example behaves the same on every +# machine and never spends somebody's money without being asked. Same treatment +# the bundled demo gets in aicertify/_demo/runner.py. +# +# Without it, a reader with an exhausted quota watches nine retries per +# interaction scroll past and reasonably concludes the example is broken. +WITH_LLM_METRICS = os.environ.get("AICERTIFY_WITH_LLM_METRICS") == "1" +if not WITH_LLM_METRICS: + os.environ.pop("OPENAI_API_KEY", None) + +# Not exposed, to match examples/quickstart.py and keep the run reproducible +# across machines with and without GPUs. +os.environ.setdefault("CUDA_VISIBLE_DEVICES", "") + + +# Imported after the environment guard above, not before: the evaluator +# stack reads OPENAI_API_KEY at import time, so popping it afterwards +# would be too late. Same ordering as aicertify/_demo/runner.py. +from aicertify import application, regulations # noqa: E402 EXAMPLE_DIR = Path(__file__).resolve().parent CONTRACT_PATH = EXAMPLE_DIR / "input_contract.json" diff --git a/tests/test_readme_flags.py b/tests/test_readme_flags.py new file mode 100644 index 0000000..b964228 --- /dev/null +++ b/tests/test_readme_flags.py @@ -0,0 +1,107 @@ +"""Every command-line flag a README documents must exist. + +This exists because two flags drifted in one week and neither was caught. + +`--evaluators` was removed in #91 for never having affected a run. The English +README dropped it; the four translations kept documenting it, in four +languages, for a flag that no longer parsed. And `--params` has always existed +and was documented nowhere at all. + +Neither shows up in a test run, a lint, or a build. A reader following the +translated README simply gets `unrecognized arguments` and concludes the tool +is broken. So the check belongs somewhere that runs on every commit, which is +here rather than in a workflow step, so it also fails on a laptop. + +The reverse direction is deliberately not asserted. A flag missing from a +translation needs a native speaker to add a row, and failing the build until +somebody translates a sentence would either block the release or invite a +machine translation nobody can vouch for. Undocumented flags are reported by +`test_english_readme_documents_every_flag` for English only, where we can +actually write the sentence. +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +import pytest + +from aicertify.cli import _build_parser + +REPO = Path(__file__).resolve().parent.parent +READMES = sorted(REPO.glob("README*.md")) + +# A row in a flag table: `| `--flag` | description |` +FLAG_ROW = re.compile(r"^\|\s*`(--[a-z][a-z-]*)`", re.MULTILINE) + + +def _real_flags() -> set[str]: + """Every option string the parser accepts, across all subcommands.""" + found: set[str] = set() + + def collect(parser: argparse.ArgumentParser) -> None: + for action in parser._actions: # noqa: SLF001 - argparse has no public API + found.update(opt for opt in action.option_strings if opt.startswith("--")) + if isinstance(action, argparse._SubParsersAction): # noqa: SLF001 + for sub in action.choices.values(): + collect(sub) + + collect(_build_parser()) + return found + + +def test_the_parser_exposes_flags_at_all() -> None: + """Guards the guard: if introspection breaks, the tests below pass vacuously.""" + flags = _real_flags() + assert "--contract" in flags, f"parser introspection returned {flags}" + assert len(flags) > 5 + + +@pytest.mark.parametrize("readme", READMES, ids=lambda p: p.name) +def test_readme_documents_no_flag_that_does_not_exist(readme: Path) -> None: + documented = set(FLAG_ROW.findall(readme.read_text(encoding="utf-8"))) + if not documented: + pytest.skip(f"{readme.name} documents no flags") + + unreal = sorted(documented - _real_flags()) + assert not unreal, ( + f"{readme.name} documents {unreal}, which the CLI does not accept. " + "A reader following it gets 'unrecognized arguments'." + ) + + +def _flags_of(command: str) -> set[str]: + """The `--flags` one subcommand accepts.""" + parser = _build_parser() + for action in parser._actions: # noqa: SLF001 + if isinstance(action, argparse._SubParsersAction): # noqa: SLF001 + sub = action.choices[command] + return { + opt + for a in sub._actions # noqa: SLF001 + for opt in a.option_strings + if opt.startswith("--") + } + raise AssertionError(f"no subcommand {command}") + + +def test_english_readme_documents_every_evaluate_flag() -> None: + """The flag table documents `evaluate`, so hold it to that command. + + Scoped rather than allow-listed. An allow-list of "flags documented + elsewhere" is a place to quietly park anything inconvenient, and it grows + until the check means nothing. `demo`, `init-contract`, `explain` and + `score-card` carry their own flags and are described in prose, so asserting + the table lists those too would only invite the exemption. + """ + readme = REPO / "README.md" + documented = set(FLAG_ROW.findall(readme.read_text(encoding="utf-8"))) + + # --help and --verbose are on every subcommand and are not worth a row. + missing = sorted(_flags_of("evaluate") - documented - {"--help", "--verbose"}) + assert not missing, ( + f"README.md documents the `evaluate` flag table but omits {missing}. " + "`--params` was missing from all five READMEs for exactly this reason." + )