diff --git a/aicertify/opa_policies b/aicertify/opa_policies index 9950848..0482ec9 160000 --- a/aicertify/opa_policies +++ b/aicertify/opa_policies @@ -1 +1 @@ -Subproject commit 995084816700e675d0d025357df27f3025a802a8 +Subproject commit 0482ec9b4bb2a237f2a75e8e12353a11d0b901b7 diff --git a/tests/test_metric_gap_report.py b/tests/test_metric_gap_report.py index f6d4c51..50a5c18 100644 --- a/tests/test_metric_gap_report.py +++ b/tests/test_metric_gap_report.py @@ -186,24 +186,35 @@ class TestTheAliasTableIsActuallyRead: list, so widening was a silent no-op and the report under-counted coverage. An empty list per entry looks like a working parser from the outside, which is why these assert content rather than shape. + + GOPAL 2.0.0 retired the 20 legacy spellings, so every entry now lists + exactly one path: its own. The parser still has to read the paths rather + than the keys, because an entry parsed as empty and an entry parsed as + itself are indistinguishable by count alone once the table is one-to-one. """ def test_aliases_are_parsed_not_just_the_keys(self): table = gap.load_alias_table() assert table, "no alias table parsed from the pinned gopal checkout" + assert all(v for v in table.values()), "some entry parsed as an empty path list" total = sum(len(v) for v in table.values()) - assert total >= 20, f"only {total} aliases parsed across {len(table)} metrics" + assert total == len( + table + ), f"{total} paths across {len(table)} metrics; GOPAL 2.0.0 is one-to-one" def test_every_entry_lists_itself_first(self): for name, paths in gap.load_alias_table().items(): assert paths and paths[0] == name, f"{name} does not list itself first" - def test_a_known_legacy_spelling_is_present(self): + def test_retired_spellings_are_absent(self): + """GOPAL 2.0.0 removed these. A gap report that still resolved them + would report a metric as supplied by a policy that cannot read it.""" table = gap.load_alias_table() - assert ( - "documentation.model_card.completeness_score" - in table["metrics.model_card.completeness"] - ) + for retired in ( + "documentation.model_card.completeness_score", + "documentation.model_card.completeness", + ): + assert retired not in table["metrics.model_card.completeness"] def test_the_two_toxicity_statistics_stay_separate(self): """gopal split these deliberately; merging them here would undo it.""" @@ -263,9 +274,19 @@ def test_a_computed_metric_is_not_reported_as_a_gap(self): aliases = gap.load_alias_table() provided = gap.provided_metrics() assert gap._policy_for("metrics.model_card.completeness", provided, aliases) - assert gap._policy_for( - "documentation.model_card.completeness_score", provided, aliases - ), "the legacy spelling should resolve to the same policy" + + def test_a_retired_spelling_resolves_to_nothing(self): + """It resolved to the scoring policy until GOPAL 2.0.0 retired it. + Continuing to resolve it would mark a requirement as met while the + value never reaches a rule.""" + aliases = gap.load_alias_table() + provided = gap.provided_metrics() + assert ( + gap._policy_for( + "documentation.model_card.completeness_score", provided, aliases + ) + is None + ) def test_clinical_metrics_stay_genuine_gaps(self): """ diff --git a/tests/test_rego_parser.py b/tests/test_rego_parser.py index 2d0532c..f7bf64c 100644 --- a/tests/test_rego_parser.py +++ b/tests/test_rego_parser.py @@ -206,6 +206,20 @@ def test_parsed_fields_match_what_the_policy_reads(self): pytest.skip("gopal submodule not checked out") source = policy.read_text(encoding="utf-8") - read_paths = set(re.findall(r"input\.([a-z_]+\.[a-z_]+)", source)) - for metric in parse_rego_file_metadata(str(policy)).required_metrics: + + # Two read forms. `input.a.b` was the only one until GOPAL's + # declarations migration rewrote every declaration read as + # `declarations.resolve(input, ["a", "b"])`. Matching only the first + # left read_paths empty, and a loop over an empty set asserts nothing, + # so this test passed while checking nothing at all. + read_paths = set(re.findall(r"input\.([a-z_0-9]+\.[a-z_0-9]+)", source)) + for args in re.findall(r"resolve(?:_or)?\(\s*input,\s*\[([^\]]+)\]", source): + parts = re.findall(r'"([^"]+)"', args) + if len(parts) >= 2: + read_paths.add(".".join(parts[:2])) + + declared = parse_rego_file_metadata(str(policy)).required_metrics + assert declared, "nothing parsed; the assertion below would be vacuous" + assert read_paths, "no reads found; the parser or the read form changed" + for metric in declared: assert metric in read_paths, f"{metric} is declared but never read"