diff --git a/launchpad/review-agent/run_adjudication.py b/launchpad/review-agent/run_adjudication.py index 67b0ecc6990..ae8b5ef4230 100644 --- a/launchpad/review-agent/run_adjudication.py +++ b/launchpad/review-agent/run_adjudication.py @@ -1,4 +1,5 @@ -"""The adjudication stage's CLI. Implements launchpad-26/buzz#118 STEP 3. +"""The adjudication stage's CLI. Implements launchpad-26/buzz#118 STEP 3 and +STEP 4 (the nonce check and the `stages` manifest -- see that section below). Reads one #117 **merged document** on stdin, adjudicates every finding with an **injected judge callable** -- defaulting to a stub that returns ``UNPROVEN`` @@ -73,6 +74,60 @@ through the protocol here (symmetric with how a future ``severity_reason`` would be) or amend ``adjudicator.md`` to say the channel is deferred. Not decided in this step; named so it cannot be merged past unnoticed. + +**STEP 4 -- the nonce check and the `stages` manifest.** Two more things +``adjudicate()`` does, on top of STEP 3's pass-through/anchor/validation-order +guarantees above, both implemented in this module because STEP 3 and STEP 4 +are two facets of one CLI: + +The top-level ``nonce`` is checked and passed through, **never generated**. +``_verify_nonce`` runs BEFORE #117's own ``findings.validate`` -- deliberately +reordered from STEP 3's original sequence, and this is why: ``findings. +validate`` independently rejects a document whose marker disagrees with the +top-level key too, but it does so with one generic per-report message, +identical whether the reports disagree with EACH OTHER or merely with the +top-level key. Running it first would mean ``_verify_nonce``'s three distinct +refusals below could never actually surface through ``main()`` -- every +document that would trigger one of them already fails ``findings.validate`` +first, so the operator would only ever see the generic message and never the +category. Checking the nonce first makes the three refusals genuinely +observable end to end, which is what ADJUDICATION.md's "own reason, distinct +from the first" requirement means in practice. ``findings.validate`` still +runs -- second now, but still before a single finding reaches the judge loop, +which is STEP 3's actual guarantee, not "first" in an absolute sense. + +One exception: a document whose ``reports`` key is missing, non-list, or +empty defers straight to ``findings.validate`` instead of ``_verify_nonce`` +-- there is nothing for nonce verification to compare against, and calling +that "absent provenance" would bury ``findings.validate``'s more specific, +more useful message for exactly that shape defect. See ``adjudicate``'s own +body for the precise condition. + +Three refusals, checked in this fixed order because one document can satisfy +more than one at once: + 1. ``"absent provenance"`` -- no top-level ``nonce``, or no report carries a + parseable completion marker. Checked first because the other two need a + value to compare against. + 2. ``"mixed document"`` -- the reports disagree with EACH OTHER. Checked + second, and wins over 3 when a document exhibits both: a mixed document + is the larger fact and a header mismatch is its consequence. + 3. ``"mismatched envelope"`` -- the reports agree with each other but not + with the top-level key. +This module never picks a winner among disagreeing nonces and never accepts +a caller-supplied one -- ``_verify_nonce`` reads only ``document`` itself. + +The ``stages`` manifest. #117 emits no top-level ``stages`` array; it is the +manifest #119 reads for stages -- #116's pre-flight, this one -- that produce +no envelope of their own. ``adjudicate()`` copies through every entry already +present on input and appends exactly one new ``{name: "adjudication", status, +reason}`` entry. An input already carrying an ``adjudication`` entry is a +re-run against an already-adjudicated document, and is refused outright +(``AlreadyAdjudicatedError``) rather than silently overwritten. + +``status`` is ``"complete"`` only when every finding received a verdict and +the nonce was established. STEP 6's total-refutation flag does not exist yet +-- when it lands it becomes a third condition ANDed into ``adjudicate()``'s +``stage_complete`` computation below, not a rewrite of it. """ from __future__ import annotations @@ -107,6 +162,186 @@ def __init__(self, violations: list[str]): super().__init__("input document fails findings.validate: " + "; ".join(violations)) +class NonceVerificationError(ValueError): + """Raised by ``_verify_nonce`` when the document's provenance cannot be + established -- one of the three refusals ADJUDICATION.md § The + ``adjudication`` block names. ``reason`` is one of ``"absent provenance"``, + ``"mixed document"`` or ``"mismatched envelope"``; ``detail`` is the + human-readable specifics. Kept as two separate attributes (rather than one + formatted string) so a caller -- ``main`` below, or a future control -- + can assert on the *category* without parsing prose. + """ + + def __init__(self, reason: str, detail: str): + self.reason = reason + self.detail = detail + super().__init__(f"{reason}: {detail}") + + +class AlreadyAdjudicatedError(ValueError): + """Raised when ``input_document["stages"]`` already carries an + ``"adjudication"`` entry. That shape means this exact document has + already been through this stage once -- a re-run -- and ADJUDICATION.md + § The ``stages`` entry requires refusing it outright rather than + silently overwriting the earlier result. + """ + + +class StagesShapeError(ValueError): + """Raised when ``input_document["stages"]`` is present but malformed -- + not a list, or carrying an entry that is not an object with a string + ``name``. + + Absent is legal and stays legal: #117 emits no top-level ``stages`` key + at all, so "no manifest yet" is the normal case. What is refused is a + manifest that exists in a shape this stage cannot honour. Treating that + as absent -- which both readers previously did -- loses data twice over: + the re-run guard has nothing to scan so a duplicate ``adjudication`` + entry slips through, and every entry already recorded is dropped, so a + ``blocked`` pre-flight disappears and the document publishes as + ``complete``. + + ADJUDICATION.md's rule is unconditional, and this stage cannot lean on + its producer to keep it: the ``stages`` manifest is explicitly an output + #117 does NOT emit, so there is no upstream guarantee to inherit. Neither + ``findings.validate`` nor ``verdicts.validate`` inspects ``stages``. + """ + + +def _input_stages(document: dict) -> list: + """``document["stages"]`` as a list, or ``[]`` when absent or null. + + The single definition of "a well-shaped input manifest", used by both + ``_check_not_already_adjudicated`` and the manifest builder in + ``adjudicate()``. One function on purpose: the two readers each had their + own inline ``isinstance(..., list)`` test and each treated a malformed + container as absent, which is how one shape defect became two independent + failures. A second copy of a rule is a second chance to disagree with it. + + Raises ``StagesShapeError`` on a present-but-malformed manifest. + """ + if "stages" not in document: + return [] + stages = document["stages"] + if stages is None: + # An explicit null is "no manifest", same as omitting the key -- the + # reading that keeps absence legal without admitting a wrong type. + return [] + if not isinstance(stages, list): + raise StagesShapeError( + "input document's `stages` is present but is not an array " + f"(got {type(stages).__name__}) -- refusing rather than treating a " + "malformed manifest as an absent one, which would discard every " + "entry already recorded in it" + ) + for index, entry in enumerate(stages): + if not isinstance(entry, dict): + raise StagesShapeError( + f"input document's `stages`[{index}] is not an object " + f"(got {type(entry).__name__}) -- every manifest entry is " + "`{name, status, reason}` per ADJUDICATION.md" + ) + name = entry.get("name") + if not isinstance(name, str): + raise StagesShapeError( + f"input document's `stages`[{index}] has a non-string `name` " + f"(got {type(name).__name__}) -- an entry that cannot be " + "identified by name cannot be checked against this stage's own" + ) + return stages + + +def _report_marker_nonce(report: dict) -> str | None: + """Extract the nonce embedded in one report's ``completion_marker``, or + ``None`` when the marker is missing, non-string, or does not parse as + ``BUZZ-DIMENSION-COMPLETE:{dimension}:{nonce}`` -- the exact format + ``findings.py``'s own ``_validate_report`` parses, matched here rather + than re-invented, since #117 is this format's one producer. + """ + marker = report.get("completion_marker") + if not isinstance(marker, str): + return None + parts = marker.split(":", 2) + if len(parts) != 3 or parts[0] != "BUZZ-DIMENSION-COMPLETE": + return None + return parts[2] + + +def _verify_nonce(document: dict) -> str: + """Verify the document's top-level ``nonce`` against every report's own + completion marker and return it. Raises ``NonceVerificationError`` -- + naming exactly one of the three refusals, in the fixed order + ADJUDICATION.md states -- when it cannot be established. Never invents a + nonce and never accepts one from anywhere but ``document`` itself. + + Reads ``document`` directly rather than trusting ``findings.validate`` + to have run first -- it runs BEFORE ``findings.validate`` in + ``adjudicate()`` (see the module docstring's STEP 4 section for why): a + stage agnostic about its producer verifies this itself. + """ + top_nonce = document.get("nonce") + reports_raw = document.get("reports") + reports = reports_raw if isinstance(reports_raw, list) else [] + report_nonces = [ + _report_marker_nonce(report) if isinstance(report, dict) else None for report in reports + ] + + # Refusal 1: ABSENT PROVENANCE. No top-level nonce, or at least one + # report's marker does not parse -- "must equal the nonce embedded in + # EVERY report's completion marker" cannot be checked for a report whose + # marker cannot even be read, so one unparseable report is enough to + # withhold provenance for the whole document, not just that report. + # Checked first: refusals 2 and 3 both need a value to compare against. + if not top_nonce: + raise NonceVerificationError("absent provenance", "no top-level `nonce` is present") + if not report_nonces or any(n is None for n in report_nonces): + raise NonceVerificationError( + "absent provenance", + "at least one report carries no parseable completion marker to compare against", + ) + + # Refusal 2: MIXED DOCUMENT. The reports disagree with each other. Wins + # over refusal 3 even when every report also disagrees with the + # top-level key: a mixed document is the larger fact, and the header + # mismatch that also follows from it is that fact's consequence, not a + # second, independent finding. + distinct_report_nonces = set(report_nonces) + if len(distinct_report_nonces) > 1: + raise NonceVerificationError( + "mixed document", + "reports carry different nonces in their completion markers: " + f"{sorted(distinct_report_nonces)}", + ) + + # Refusal 3: MISMATCHED ENVELOPE. The reports agree with each other but + # not with the top-level key -- one run's reports under another run's + # header. + (agreed_nonce,) = distinct_report_nonces + if agreed_nonce != top_nonce: + raise NonceVerificationError( + "mismatched envelope", + f"every report's completion marker carries nonce {agreed_nonce!r}, which does " + f"not match the top-level nonce {top_nonce!r}", + ) + + return top_nonce + + +def _check_not_already_adjudicated(document: dict) -> None: + """Raise ``AlreadyAdjudicatedError`` when ``document["stages"]`` already + carries an entry named ``"adjudication"``. Run before ``_verify_nonce`` + in ``adjudicate()`` -- a re-run is a structural defect in the request + itself, independent of whether this particular re-run's nonce happens to + check out. + """ + for entry in _input_stages(document): + if entry.get("name") == "adjudication": + raise AlreadyAdjudicatedError( + "input document's `stages` array already carries an `adjudication` entry -- " + "refusing to re-run adjudication over an already-adjudicated document" + ) + + def _location_description(finding: dict) -> str: """Describe where a finding is anchored, branching on ``anchor`` FIRST -- never assuming ``file``/``line`` exist. Anchor ``"pr"`` is a normal, valid @@ -227,11 +462,23 @@ def adjudicate(input_document: dict, judge: Judge) -> dict: """Adjudicate every finding in ``input_document`` with ``judge`` and return the adjudicated output document. Never mutates ``input_document``. + Raises ``StagesShapeError`` when ``input_document["stages"]`` is present + but malformed -- not a list, or an entry that is not an object with a + string ``name``. Absent or null stays legal. + + Raises ``AlreadyAdjudicatedError`` when ``input_document["stages"]`` + already carries an ``adjudication`` entry (a re-run), and + ``NonceVerificationError`` when the top-level ``nonce`` cannot be + established against every report's completion marker (STEP 4; see the + module docstring for why these run BEFORE ``findings.validate`` now). + Raises ``InputValidationError`` -- adjudicating nothing, calling ``judge`` zero times -- when ``input_document`` fails #117's own ``findings.validate``. This is the boundary STEP 1/STEP 2 call load-bearing: a finding whose ``severity`` already arrived illegal is refused here, wholesale, rather than reaching a per-finding fallback with no good answer. + Checked after the two STEP 4 gates above, but still before any finding + reaches the judge loop -- STEP 3's actual guarantee. Pass-through fields (``pr``, ``merge_base_sha``, ``head_sha``, ``containment``) are never touched: the output starts as a @@ -241,12 +488,37 @@ def adjudicate(input_document: dict, judge: Judge) -> dict: here is left exactly equal to its ``reported_severity``, and ``duplicate_of`` is always null. """ - violations = findings.validate(input_document) - if violations: - raise InputValidationError(violations) + _check_not_already_adjudicated(input_document) + + # `_verify_nonce`'s job is provenance, not `reports`'s basic shape. A + # document whose `reports` key is missing, non-list, or empty has nothing + # for nonce verification to compare against -- `_verify_nonce` would call + # that "absent provenance", which is technically true but masks the more + # specific, more useful message findings.validate already gives for + # exactly this ("missing required key 'reports'", "must not be empty", + # "expected an array"). So a document this malformed defers straight to + # findings.validate instead of being told the wrong subsystem is broken. + reports_raw = input_document.get("reports") + reports_present_and_nonempty = isinstance(reports_raw, list) and len(reports_raw) > 0 + + if reports_present_and_nonempty: + nonce = _verify_nonce(input_document) + violations = findings.validate(input_document) + if violations: + raise InputValidationError(violations) + else: + violations = findings.validate(input_document) + if violations: + raise InputValidationError(violations) + # Unreachable in practice: a missing, non-list, or empty `reports` + # always fails findings.validate above, on one of the three grounds + # named in this branch's comment. Kept as a real call, not asserted + # away, the same "real branch" discipline `stage_complete`'s + # nonce_established condition already uses below for STEP 6's + # not-yet-built flag. + nonce = _verify_nonce(input_document) output_document = copy.deepcopy(input_document) - nonce = output_document.get("nonce") verdict_counts = {"CONFIRMED": 0, "REFUTED": 0, "UNPROVEN": 0} findings_in = 0 @@ -288,6 +560,50 @@ def adjudicate(input_document: dict, judge: Judge) -> dict: completion_marker=f"BUZZ-ADJUDICATION-COMPLETE:{nonce}", ).as_dict() + # The `stages` manifest (STEP 4). Every entry already on input, in order, + # plus exactly one new `adjudication` entry -- `_check_not_already_ + # adjudicated` above already guarantees none of the input entries is + # itself named `adjudication`. That guarantee is real only because both it + # and this line read the manifest through `_input_stages`, which refuses a + # present-but-malformed shape instead of quietly reading it as absent. + input_stages = copy.deepcopy(_input_stages(input_document)) + + # `status` is "complete" only when every finding received a verdict AND + # the nonce was established. The nonce condition is always True here -- + # `_verify_nonce` above would have raised otherwise -- named explicitly + # anyway so the AND reads as the real, multi-condition guarantee + # ADJUDICATION.md states rather than a constant. `every_finding_has_ + # verdict` is read back off `output_document` itself (not tracked as a + # separate counter through the loop above) so it is a check ON the + # produced data rather than a second bookkeeping path that could drift + # from it. STEP 6's total-refutation flag is a third condition this stage + # does not build yet -- its absence must not make `stage_complete` wrongly + # unconditional, which is why it is named as its own boolean rather than + # inlined into one `and` chain that silently drops it. + nonce_established = True + every_finding_has_verdict = all( + finding.get("verdict") in verdicts.VERDICTS + for report in output_document.get("reports", []) + for finding in report.get("findings", []) + ) + stage_complete = every_finding_has_verdict and nonce_established + + if stage_complete: + stage_status, stage_reason = "complete", None + else: + # Unreachable today: `_run_judge_safely` always returns a legal + # verdict, so `every_finding_has_verdict` is always True by the time + # this runs, and a False `nonce_established` would already have + # raised above. Kept as a real branch, not asserted away, so STEP 6 + # can add its own condition here without restructuring this function. + stage_status = "incomplete" + stage_reason = "not every finding received a verdict" + + output_document["stages"] = [ + *input_stages, + {"name": "adjudication", "status": stage_status, "reason": stage_reason}, + ] + return output_document @@ -355,6 +671,15 @@ def main(argv: list[str] | None = None) -> int: for violation in exc.violations: print(f"run_adjudication: {violation}", file=sys.stderr) return 1 + except AlreadyAdjudicatedError as exc: + print(f"run_adjudication: {exc}", file=sys.stderr) + return 1 + except StagesShapeError as exc: + print(f"run_adjudication: {exc}", file=sys.stderr) + return 1 + except NonceVerificationError as exc: + print(f"run_adjudication: {exc.reason}: {exc.detail}", file=sys.stderr) + return 1 print(json.dumps(output_document)) return 0 diff --git a/launchpad/review-agent/test_run_adjudication.py b/launchpad/review-agent/test_run_adjudication.py index c91ceb4636c..0c8bd6a717d 100644 --- a/launchpad/review-agent/test_run_adjudication.py +++ b/launchpad/review-agent/test_run_adjudication.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Controls for run_adjudication.py -- issue #118 STEP 3's CLI. +"""Controls for run_adjudication.py -- issue #118 STEP 3 and STEP 4's CLI. Exercises every behaviour named in STEP 3's own done-when list in launchpad/plans/2026-08-13-issue-118-adjudication.md: byte-identical @@ -10,13 +10,25 @@ adjudicated (the injected judge's own call count proves the refusal happens first), and no ``gh`` subprocess or HTTP client invoked during a stub run. +Also exercises STEP 4's own done-when: the nonce check (three refusals -- +``"absent provenance"``, ``"mixed document"``, ``"mismatched envelope"`` -- +in that fixed order) and the ``stages`` manifest. The three refusals are +proven two ways: directly against ``_verify_nonce`` with hand-built +documents (``NonceVerificationDirectTests`` below), which is the only way to +observe their distinct reasons at all, and end to end through ``main`` with +realistic fixtures (``NonceVerificationEndToEndTests``), where every one of +them is ALSO already caught by #117's own ``findings.validate`` -- which +``adjudicate`` runs first -- with its own, less specific, message. Both are +tested because both are true: the dedicated check is real defence in depth, +per ADJUDICATION.md's own reasoning, and it does not change what ``main`` +reports for any fixture that also happens to fail #117's own contract, which +is every reachable fixture today. + Deliberately NOT exercised here (later steps' territory, per the plan): -the nonce three-way disagreement diagnosis and the ``stages`` manifest -(STEP 4), the escalate-only guard and downgrade recording for a judge that -actually re-rates severity (STEP 6), and dedupe (STEP 7). Every fixture -below either omits a re-rating entirely or only ever asserts that this -stage's own severity pass-through (``severity == reported_severity``, -always) holds. +the escalate-only guard and downgrade recording for a judge that actually +re-rates severity (STEP 6), and dedupe (STEP 7). Every fixture below either +omits a re-rating entirely or only ever asserts that this stage's own +severity pass-through (``severity == reported_severity``, always) holds. This file is scoped to `run_adjudication.py` alone and is deliberately not wired into `run_controls.py`'s CONTROLS list -- that is STEP 10's control @@ -539,5 +551,412 @@ def test_real_process_illegal_severity_exits_nonzero_no_stdout(self): self.assertEqual(proc.stdout, "") +class NonceVerificationDirectTests(unittest.TestCase): + """Direct, unit-level tests of ``_verify_nonce`` in isolation -- the + three refusals' distinct reasons and their fixed precedence, without the + rest of ``adjudicate`` around them. ``NonceVerificationEndToEndTests`` + below proves the same three reasons surface through the real CLI, now + that ``_verify_nonce`` runs before ``findings.validate`` in + ``adjudicate`` (see the module docstring's STEP 4 section). + """ + + def test_no_top_level_nonce_is_absent_provenance(self): + doc = make_document(reports=[make_report(nonce=NONCE)], nonce=NONCE) + del doc["nonce"] + with self.assertRaises(run_adjudication.NonceVerificationError) as ctx: + run_adjudication._verify_nonce(doc) + self.assertEqual(ctx.exception.reason, "absent provenance") + + def test_report_with_no_parseable_marker_is_absent_provenance(self): + report = make_report(nonce=NONCE) + del report["completion_marker"] + doc = make_document(reports=[report], nonce=NONCE) + with self.assertRaises(run_adjudication.NonceVerificationError) as ctx: + run_adjudication._verify_nonce(doc) + self.assertEqual(ctx.exception.reason, "absent provenance") + + def test_one_unparseable_marker_among_otherwise_agreeing_reports_is_absent_provenance(self): + # "must equal EVERY report's completion marker" cannot be checked for + # a report whose marker cannot be read -- one bad report withholds + # provenance for the whole document, it does not just drop out of + # the comparison. + good = make_report(dimension="a", nonce=NONCE) + unreadable = make_report(dimension="b", nonce=NONCE) + del unreadable["completion_marker"] + doc = make_document(reports=[good, unreadable], nonce=NONCE) + with self.assertRaises(run_adjudication.NonceVerificationError) as ctx: + run_adjudication._verify_nonce(doc) + self.assertEqual(ctx.exception.reason, "absent provenance") + + def test_reports_disagreeing_with_each_other_is_mixed_document(self): + doc = make_document( + reports=[make_report(dimension="a", nonce="N1"), make_report(dimension="b", nonce="N2")], + nonce=NONCE, + ) + with self.assertRaises(run_adjudication.NonceVerificationError) as ctx: + run_adjudication._verify_nonce(doc) + self.assertEqual(ctx.exception.reason, "mixed document") + + def test_reports_agreeing_but_not_with_top_level_is_mismatched_envelope(self): + doc = make_document( + reports=[make_report(dimension="a", nonce="N1"), make_report(dimension="b", nonce="N1")], + nonce=NONCE, # top-level differs from both reports' shared "N1" + ) + with self.assertRaises(run_adjudication.NonceVerificationError) as ctx: + run_adjudication._verify_nonce(doc) + self.assertEqual(ctx.exception.reason, "mismatched envelope") + + def test_mixed_document_wins_over_mismatched_envelope_when_both_apply(self): + # Every report disagrees with the top-level key AND with each other: + # satisfies both "mixed document" and "mismatched envelope" at once. + # ADJUDICATION.md states the mixed document wins. + doc = make_document( + reports=[make_report(dimension="a", nonce="N1"), make_report(dimension="b", nonce="N2")], + nonce="N3", + ) + with self.assertRaises(run_adjudication.NonceVerificationError) as ctx: + run_adjudication._verify_nonce(doc) + self.assertEqual(ctx.exception.reason, "mixed document") + + def test_matching_nonce_is_returned_unchanged_and_never_invented(self): + doc = make_document(reports=[make_report(nonce=NONCE)], nonce=NONCE) + self.assertEqual(run_adjudication._verify_nonce(doc), NONCE) + + +class NonceVerificationEndToEndTests(unittest.TestCase): + """The same three refusals, through `main` with realistic fixtures -- + proving the DISTINCT reason each one names is actually observable end to + end, not just from calling `_verify_nonce` directly. + + This is only true because `adjudicate()` runs `_verify_nonce` BEFORE + #117's own `findings.validate`. `findings.validate` independently rejects + the same documents, but with one generic per-report message that does not + distinguish "reports disagree with each other" from "reports agree with + each other but not the top-level key" -- see the module docstring's STEP 4 + section. Checking the ordering here, not just the exit code, is the whole + point of this class: a regression that reverts the ordering would still + pass a test that only asserts `stdout == ""`. + """ + + def _run_main_with_document(self, document: dict) -> tuple[int, str, str]: + stdout, stderr = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdin", io.StringIO(json.dumps(document))), \ + contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exit_code = run_adjudication.main([]) + return exit_code, stdout.getvalue(), stderr.getvalue() + + def test_two_reports_with_different_nonces_exits_nonzero_no_document(self): + doc = make_document( + reports=[make_report(dimension="a", nonce="N1"), make_report(dimension="b", nonce="N2")], + nonce=NONCE, + ) + exit_code, stdout, stderr = self._run_main_with_document(doc) + self.assertNotEqual(exit_code, 0) + self.assertEqual(stdout, "") + self.assertIn("mixed document", stderr, stderr) + + def test_reports_agreeing_but_not_top_level_exits_nonzero_no_document(self): + doc = make_document( + reports=[make_report(dimension="a", nonce="N1"), make_report(dimension="b", nonce="N1")], + nonce=NONCE, + ) + exit_code, stdout, stderr = self._run_main_with_document(doc) + self.assertNotEqual(exit_code, 0) + self.assertEqual(stdout, "") + self.assertIn("mismatched envelope", stderr, stderr) + self.assertNotIn("mixed document", stderr, stderr) + + def test_no_top_level_nonce_exits_nonzero_and_invents_nothing(self): + doc = make_document(reports=[make_report(nonce=NONCE)], nonce=NONCE) + del doc["nonce"] + exit_code, stdout, stderr = self._run_main_with_document(doc) + self.assertNotEqual(exit_code, 0) + self.assertEqual(stdout, "") # nothing printed means no nonce was invented + self.assertIn("absent provenance", stderr, stderr) + + def test_report_with_no_marker_exits_nonzero_and_never_reports_complete(self): + report = make_report(nonce=NONCE) + del report["completion_marker"] + doc = make_document(reports=[report], nonce=NONCE) + exit_code, stdout, stderr = self._run_main_with_document(doc) + self.assertNotEqual(exit_code, 0) + # No document at all is printed, so no stage status is ever emitted -- + # "complete" in particular is never among them. + self.assertEqual(stdout, "") + self.assertIn("absent provenance", stderr, stderr) + + +class MalformedReportsDefersToFindingsValidateTests(unittest.TestCase): + """A `reports`-shape defect (missing, non-list, empty) is not a nonce + problem -- `_verify_nonce` would call it "absent provenance", which + buries `findings.validate`'s more specific, more useful message. These + three shapes must all defer to that message instead. + """ + + def _run_main_with_document(self, document: dict) -> tuple[int, str, str]: + stdout, stderr = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdin", io.StringIO(json.dumps(document))), \ + contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exit_code = run_adjudication.main([]) + return exit_code, stdout.getvalue(), stderr.getvalue() + + def test_missing_reports_key_names_the_missing_key_not_provenance(self): + doc = make_document(reports=[make_report(nonce=NONCE)], nonce=NONCE) + del doc["reports"] + exit_code, stdout, stderr = self._run_main_with_document(doc) + self.assertNotEqual(exit_code, 0) + self.assertEqual(stdout, "") + self.assertIn("missing required key 'reports'", stderr, stderr) + self.assertNotIn("absent provenance", stderr, stderr) + + def test_empty_reports_array_names_the_empty_array_not_provenance(self): + doc = make_document(reports=[], nonce=NONCE) + exit_code, stdout, stderr = self._run_main_with_document(doc) + self.assertNotEqual(exit_code, 0) + self.assertEqual(stdout, "") + self.assertIn("must not be empty", stderr, stderr) + self.assertNotIn("absent provenance", stderr, stderr) + + def test_non_list_reports_names_the_wrong_type_not_provenance(self): + doc = make_document(reports=[make_report(nonce=NONCE)], nonce=NONCE) + doc["reports"] = "not-a-list" + exit_code, stdout, stderr = self._run_main_with_document(doc) + self.assertNotEqual(exit_code, 0) + self.assertEqual(stdout, "") + self.assertIn("expected an array", stderr, stderr) + self.assertNotIn("absent provenance", stderr, stderr) + + +class StagesManifestTests(unittest.TestCase): + """The top-level `stages` array STEP 4 adds: every entry already on + input, in order, plus exactly one new `adjudication` entry. + """ + + def test_happy_path_output_nonce_unchanged_and_marker_is_last_key(self): + reports = [make_report(dimension=d, nonce=NONCE) for d in ("a", "b", "c")] + input_doc = make_document(reports=reports, nonce=NONCE) + + output_doc = run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + + self.assertEqual(output_doc["nonce"], NONCE) + adjudication = output_doc["adjudication"] + keys = list(adjudication.keys()) + self.assertEqual(keys[-1], "completion_marker") + self.assertEqual(adjudication["completion_marker"], f"BUZZ-ADJUDICATION-COMPLETE:{NONCE}") + + def test_output_stages_carries_input_entries_plus_one_new_adjudication_entry(self): + input_doc = make_document() + input_doc["stages"] = [{"name": "preflight", "status": "complete", "reason": None}] + + output_doc = run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + + self.assertEqual( + output_doc["stages"], + [ + {"name": "preflight", "status": "complete", "reason": None}, + {"name": "adjudication", "status": "complete", "reason": None}, + ], + ) + + def test_output_stages_is_just_the_new_entry_when_input_has_none(self): + input_doc = make_document() + self.assertNotIn("stages", input_doc) + + output_doc = run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + + self.assertEqual( + output_doc["stages"], + [{"name": "adjudication", "status": "complete", "reason": None}], + ) + + def test_input_stages_list_is_not_mutated(self): + input_doc = make_document() + input_stages = [{"name": "preflight", "status": "complete", "reason": None}] + input_doc["stages"] = input_stages + + run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + + self.assertEqual(input_stages, [{"name": "preflight", "status": "complete", "reason": None}]) + + +class AlreadyAdjudicatedTests(unittest.TestCase): + """An input already carrying an `adjudication` entry in `stages` is a + re-run against an already-adjudicated document -- refused outright, + never silently overwritten. + """ + + def test_adjudicate_raises_and_never_calls_the_judge(self): + input_doc = make_document() + input_doc["stages"] = [{"name": "adjudication", "status": "complete", "reason": None}] + judge = CountingJudge() + + with self.assertRaises(run_adjudication.AlreadyAdjudicatedError): + run_adjudication.adjudicate(input_doc, judge) + + self.assertEqual(judge.call_count, 0, judge.calls) + + def test_main_exits_nonzero_and_prints_no_document(self): + input_doc = make_document() + input_doc["stages"] = [{"name": "adjudication", "status": "complete", "reason": None}] + stdout, stderr = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdin", io.StringIO(json.dumps(input_doc))), \ + contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exit_code = run_adjudication.main([]) + self.assertNotEqual(exit_code, 0) + self.assertEqual(stdout.getvalue(), "") + self.assertTrue(stderr.getvalue()) + + +class MalformedStagesShapeTests(unittest.TestCase): + """A `stages` value that is PRESENT but not a list was treated as absent + at both sites that read it: the re-run guard returned early, and the + manifest builder substituted `[]`. Two consequences, and the second is + the one that costs something: + + 1. The re-run guard is bypassed -- an `adjudication` entry inside an + object container adjudicates at exit 0 instead of being refused. + 2. Every prior entry is silently discarded. A `blocked` pre-flight + (#116's fork-PR-secrets-withheld case) disappears and the document + publishes as `complete`, because #119 only banners a non-`complete` + status. That is #118's fifth criterion failing through a shape + defect no verdict-side check looks at. + + "#117 never emits that shape" is not a defence available to this step: + the `stages` manifest is explicitly an output #117 does NOT produce, so + this stage cannot inherit a guarantee from it. Absent stays legal. + """ + + NON_LIST_SHAPES = ( + ({"0": {"name": "adjudication", "status": "complete", "reason": None}}, "object"), + ("adjudication", "string"), + (42, "int"), + (True, "bool"), + ) + + def test_adjudicate_raises_on_every_non_list_stages_shape(self): + for shape, label in self.NON_LIST_SHAPES: + with self.subTest(shape=label): + input_doc = make_document() + input_doc["stages"] = shape + judge = CountingJudge() + with self.assertRaises(run_adjudication.StagesShapeError): + run_adjudication.adjudicate(input_doc, judge) + # Refused before any finding is adjudicated, like every other + # input-shape refusal in this module. + self.assertEqual(judge.call_count, 0, judge.calls) + + def test_the_re_run_guard_is_not_bypassed_by_an_object_container(self): + """The bypass itself: before the fix this adjudicated at exit 0.""" + input_doc = make_document() + input_doc["stages"] = {"0": {"name": "adjudication", "status": "complete", "reason": None}} + with self.assertRaises(run_adjudication.StagesShapeError): + run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + + def test_a_blocked_preflight_is_never_silently_discarded(self): + """The expensive half. A `blocked` pre-flight inside a non-list + container used to vanish, and the document published `complete`. + """ + input_doc = make_document() + input_doc["stages"] = { + "p": {"name": "preflight", "status": "blocked", "reason": "fork PR, secrets withheld"} + } + with self.assertRaises(run_adjudication.StagesShapeError): + run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + + def test_a_stages_entry_that_is_not_an_object_is_refused(self): + input_doc = make_document() + input_doc["stages"] = ["preflight"] + with self.assertRaises(run_adjudication.StagesShapeError): + run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + + def test_a_stages_entry_with_a_non_string_name_is_refused(self): + """The Low that rides along: a non-string `name` cannot impersonate an + `adjudication` entry, so the re-run guard is not bypassed this way -- + but an off-shape entry reaching #119 is still not something to pass + through in silence. + """ + input_doc = make_document() + input_doc["stages"] = [{"name": {"nested": "adjudication"}, "status": "complete"}] + with self.assertRaises(run_adjudication.StagesShapeError): + run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + + def test_absent_stages_is_still_legal(self): + """The control. #117 emits no `stages` key at all, so absent must stay + the normal case -- a fix that refused absence would break every real + document. + """ + input_doc = make_document() + self.assertNotIn("stages", input_doc) + output_doc = run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + self.assertEqual([e["name"] for e in output_doc["stages"]], ["adjudication"]) + + def test_explicit_null_stages_is_treated_as_absent(self): + input_doc = make_document() + input_doc["stages"] = None + output_doc = run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + self.assertEqual([e["name"] for e in output_doc["stages"]], ["adjudication"]) + + def test_a_well_formed_preflight_entry_still_survives_in_order(self): + """The other control: the shape this step is meant to carry forward + must still be carried forward, in order, untouched. + """ + input_doc = make_document() + input_doc["stages"] = [ + {"name": "preflight", "status": "blocked", "reason": "fork PR, secrets withheld"} + ] + output_doc = run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + self.assertEqual([e["name"] for e in output_doc["stages"]], ["preflight", "adjudication"]) + self.assertEqual(output_doc["stages"][0]["status"], "blocked") + + def test_main_exits_nonzero_and_prints_no_document(self): + for shape, label in self.NON_LIST_SHAPES: + with self.subTest(shape=label): + input_doc = make_document() + input_doc["stages"] = shape + stdout, stderr = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdin", io.StringIO(json.dumps(input_doc))), \ + contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exit_code = run_adjudication.main([]) + self.assertNotEqual(exit_code, 0) + self.assertEqual(stdout.getvalue(), "") + self.assertTrue(stderr.getvalue()) + + def test_real_process_refuses_an_object_container(self): + """Through the real process, the way the defect was found.""" + input_doc = make_document() + input_doc["stages"] = {"0": {"name": "adjudication", "status": "complete", "reason": None}} + proc = subprocess.run( + [sys.executable, str(SCRIPT)], + input=json.dumps(input_doc), + capture_output=True, + text=True, + check=False, + ) + self.assertNotEqual(proc.returncode, 0) + self.assertEqual(proc.stdout, "") + self.assertIn("stages", proc.stderr) + + +class PublishIncompleteRuleTests(unittest.TestCase): + """#119's own rule -- "any status other than 'complete' is incomplete + and banners the whole review" -- run here as an assertion against the + output, since #119's own code does not exist to run against (STEP 4's + own done-when names this explicitly). + """ + + def test_happy_path_stage_status_is_complete_so_119_would_not_banner_it(self): + input_doc = make_document() + + output_doc = run_adjudication.adjudicate(input_doc, run_adjudication.stub_judge) + + adjudication_stage = next( + entry for entry in output_doc["stages"] if entry["name"] == "adjudication" + ) + # #119's stated rule, applied directly: only "complete" reads as + # complete: anything else -- any other string -- banners the review. + self.assertEqual(adjudication_stage["status"], "complete") + self.assertIsNone(adjudication_stage["reason"]) + + if __name__ == "__main__": unittest.main()