From fa84ed4279af8baf6ab81b77d63e1c130bb8407b Mon Sep 17 00:00:00 2001 From: "kapil.madan" <3740365+kmadan@users.noreply.github.com> Date: Sat, 29 Aug 2026 17:05:22 +0530 Subject: [PATCH] chore: pin GOPAL 2.0.0, and repair what the bump exposed The submodule was 17 commits behind and had not moved since before the declarations migration, so two separate breakages surfaced together. From GOPAL 2.0.0, the alias table is one-to-one. Three tests asserted the old contract: at least 20 aliases, a named legacy spelling present, and a legacy spelling resolving to the policy that computes the metric. All three now assert the opposite, because a gap report that still resolved a retired spelling would mark a requirement as met while the value never reaches a rule. From the earlier declarations migration, test_parsed_fields_match_what_the_policy_reads matched only `input.a.b`. Every declaration read is now `declarations.resolve(input, ["a", "b"])`, so read_paths came back empty and the loop over it asserted nothing. The test had been passing without checking anything. It reads both forms now, and refuses to run on an empty set either side. Verified against both pins: 45 passed on 1.x, 4 failed on 2.0.0, and 216 pass after these fixes. --- aicertify/opa_policies | 2 +- tests/test_metric_gap_report.py | 39 +++++++++++++++++++++++++-------- tests/test_rego_parser.py | 18 +++++++++++++-- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/aicertify/opa_policies b/aicertify/opa_policies index 9950848..0482ec9 160000 --- a/aicertify/opa_policies +++ b/aicertify/opa_policies @@ -1 +1 @@ -Subproject commit 995084816700e675d0d025357df27f3025a802a8 +Subproject commit 0482ec9b4bb2a237f2a75e8e12353a11d0b901b7 diff --git a/tests/test_metric_gap_report.py b/tests/test_metric_gap_report.py index f6d4c51..50a5c18 100644 --- a/tests/test_metric_gap_report.py +++ b/tests/test_metric_gap_report.py @@ -186,24 +186,35 @@ class TestTheAliasTableIsActuallyRead: list, so widening was a silent no-op and the report under-counted coverage. An empty list per entry looks like a working parser from the outside, which is why these assert content rather than shape. + + GOPAL 2.0.0 retired the 20 legacy spellings, so every entry now lists + exactly one path: its own. The parser still has to read the paths rather + than the keys, because an entry parsed as empty and an entry parsed as + itself are indistinguishable by count alone once the table is one-to-one. """ def test_aliases_are_parsed_not_just_the_keys(self): table = gap.load_alias_table() assert table, "no alias table parsed from the pinned gopal checkout" + assert all(v for v in table.values()), "some entry parsed as an empty path list" total = sum(len(v) for v in table.values()) - assert total >= 20, f"only {total} aliases parsed across {len(table)} metrics" + assert total == len( + table + ), f"{total} paths across {len(table)} metrics; GOPAL 2.0.0 is one-to-one" def test_every_entry_lists_itself_first(self): for name, paths in gap.load_alias_table().items(): assert paths and paths[0] == name, f"{name} does not list itself first" - def test_a_known_legacy_spelling_is_present(self): + def test_retired_spellings_are_absent(self): + """GOPAL 2.0.0 removed these. A gap report that still resolved them + would report a metric as supplied by a policy that cannot read it.""" table = gap.load_alias_table() - assert ( - "documentation.model_card.completeness_score" - in table["metrics.model_card.completeness"] - ) + for retired in ( + "documentation.model_card.completeness_score", + "documentation.model_card.completeness", + ): + assert retired not in table["metrics.model_card.completeness"] def test_the_two_toxicity_statistics_stay_separate(self): """gopal split these deliberately; merging them here would undo it.""" @@ -263,9 +274,19 @@ def test_a_computed_metric_is_not_reported_as_a_gap(self): aliases = gap.load_alias_table() provided = gap.provided_metrics() assert gap._policy_for("metrics.model_card.completeness", provided, aliases) - assert gap._policy_for( - "documentation.model_card.completeness_score", provided, aliases - ), "the legacy spelling should resolve to the same policy" + + def test_a_retired_spelling_resolves_to_nothing(self): + """It resolved to the scoring policy until GOPAL 2.0.0 retired it. + Continuing to resolve it would mark a requirement as met while the + value never reaches a rule.""" + aliases = gap.load_alias_table() + provided = gap.provided_metrics() + assert ( + gap._policy_for( + "documentation.model_card.completeness_score", provided, aliases + ) + is None + ) def test_clinical_metrics_stay_genuine_gaps(self): """ diff --git a/tests/test_rego_parser.py b/tests/test_rego_parser.py index 2d0532c..f7bf64c 100644 --- a/tests/test_rego_parser.py +++ b/tests/test_rego_parser.py @@ -206,6 +206,20 @@ def test_parsed_fields_match_what_the_policy_reads(self): pytest.skip("gopal submodule not checked out") source = policy.read_text(encoding="utf-8") - read_paths = set(re.findall(r"input\.([a-z_]+\.[a-z_]+)", source)) - for metric in parse_rego_file_metadata(str(policy)).required_metrics: + + # Two read forms. `input.a.b` was the only one until GOPAL's + # declarations migration rewrote every declaration read as + # `declarations.resolve(input, ["a", "b"])`. Matching only the first + # left read_paths empty, and a loop over an empty set asserts nothing, + # so this test passed while checking nothing at all. + read_paths = set(re.findall(r"input\.([a-z_0-9]+\.[a-z_0-9]+)", source)) + for args in re.findall(r"resolve(?:_or)?\(\s*input,\s*\[([^\]]+)\]", source): + parts = re.findall(r'"([^"]+)"', args) + if len(parts) >= 2: + read_paths.add(".".join(parts[:2])) + + declared = parse_rego_file_metadata(str(policy)).required_metrics + assert declared, "nothing parsed; the assertion below would be vacuous" + assert read_paths, "no reads found; the parser or the read form changed" + for metric in declared: assert metric in read_paths, f"{metric} is declared but never read"