From e1a8158b58bd681e079962049668fdc7fb722dd2 Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Thu, 24 Sep 2026 00:23:14 +0200 Subject: [PATCH 1/4] feat: end a run stopped by the plan with a neutral comment instead of a red check When /run/start refuses a pull request run, the action skips the analysis, replaces the sticky comment with the wall's own sentence ("CodeBoarding map not drawn: m.koch has used 5 of 5 free runs this week ..."), writes it to the job summary and exits 0. The wall payload is uploaded as the codeboarding-wall artifact (wall.json, the wall.schema.json shape), which is how the web app shows the same sentence on the pull request's page. A 402 mid-run (token ceiling, no live run) ends the same way: the relay keeps the 402's wall, with-auth.sh turns the failed analysis into walled=true and exit 0, and the review steps that need a map are skipped. A sync at its ceiling skips silently. The finish step reports such a run as failed, with the wall's reason as its error. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 7 ++++ action.yml | 45 +++++++++++++++++++---- scripts/action/configure-auth.sh | 2 ++ scripts/action/run_meter.py | 60 ++++++++++++++++++++++++++++--- scripts/action/with-auth.sh | 15 +++++++- scripts/oidc_relay.py | 24 +++++++++++-- tests/test_action_auth.py | 25 +++++++++++++ tests/test_action_inputs.py | 33 +++++++++++++++++ tests/test_oidc_relay.py | 55 ++++++++++++++++++++++++++++ tests/test_run_meter.py | 61 ++++++++++++++++++++++++++++++++ 10 files changed, 314 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3fb6290..f913867 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,13 @@ ahead and how deep it may go. A final step reports whether a map was produced. checked: own-key runs still call `/run/start` and count the same, with no token ceiling, since the model bill is yours. A job without `id-token: write` skips the check with a warning. +- **At the wall the check stays green.** A run over its allowance is not analysed: the + review comment says so in one sentence, for example "CodeBoarding map not drawn: m.koch + has used 5 of 5 free runs this week (resets Monday 00:00 UTC). Pro gives 40 a week; a + Team plan covers everyone in acme-corp.", the job summary repeats it, and the job exits + 0. The same happens when a hosted run reaches its weekly token ceiling mid-analysis. The + wall payload is uploaded as the `codeboarding-wall` artifact (`wall.json`), so the web app + shows the same sentence on the pull request. A baseline sync at its ceiling skips silently. - **CodeBoarding down is never your problem.** If the proxy cannot be reached, the run goes ahead at up to 3 levels with a warning. - **Update pinned versions.** Action versions from before this release do not start runs diff --git a/action.yml b/action.yml index 0ec752b..2b08568 100644 --- a/action.yml +++ b/action.yml @@ -458,7 +458,7 @@ runs: - name: Deliver baseline id: sync_commit - if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'sync' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.sync_analyze.outputs.walled != 'true' && steps.guard.outputs.mode == 'sync' shell: bash env: ACTION_PATH: ${{ github.action_path }} @@ -583,7 +583,7 @@ runs: - name: Render review diagram id: review_render - if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.review_analyze.outputs.walled != 'true' && steps.guard.outputs.mode == 'review' shell: bash env: ACTION_PATH: ${{ github.action_path }} @@ -593,7 +593,7 @@ runs: - name: Build review artifact id: review_artifact - if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.review_analyze.outputs.walled != 'true' && steps.guard.outputs.mode == 'review' shell: bash env: ANALYSIS_PATH: ${{ steps.review_analyze.outputs.analysis_path }} @@ -614,7 +614,7 @@ runs: - name: Upload review artifact id: upload_review_artifact_dotcom - if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' && github.server_url == 'https://github.com' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.review_analyze.outputs.walled != 'true' && steps.guard.outputs.mode == 'review' && github.server_url == 'https://github.com' uses: actions/upload-artifact@v4 with: name: codeboarding-review-${{ github.run_id }}-${{ github.run_attempt }} @@ -628,7 +628,7 @@ runs: - name: Build review comment id: review_body - if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.review_analyze.outputs.walled != 'true' && steps.guard.outputs.mode == 'review' shell: bash env: DIAGRAM: ${{ steps.review_render.outputs.diagram_md }} @@ -645,7 +645,7 @@ runs: - name: Post review comment id: review_comment - if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.review_analyze.outputs.walled != 'true' && steps.guard.outputs.mode == 'review' uses: marocchino/sticky-pull-request-comment@v2 with: header: ${{ steps.guard.outputs.comment_id }} @@ -666,6 +666,39 @@ runs: BODY: ${{ steps.review_body.outputs.path }} run: cat "$BODY" >> "$GITHUB_STEP_SUMMARY" + # The plan stopped this run, at the preflight or with a 402 mid-run: say so in the same + # sticky comment and the job summary, in the plan's own words, and end green. A failed + # check would punish the author for the plan. The artifact carries the wall payload so + # the web app can show the same sentence on the pull request's page. + - name: Explain the plan's wall + id: wall + if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'review' && (steps.preflight.outputs.allowed == 'false' || steps.review_analyze.outputs.walled == 'true') + continue-on-error: true + shell: bash + env: + PR_NUMBER: ${{ steps.guard.outputs.pr_number }} + run: python3 "$GITHUB_ACTION_PATH/scripts/action/run_meter.py" wall + + - name: Post the plan's wall + if: steps.wall.outputs.path != '' && steps.guard.outputs.pr_number != '' + continue-on-error: true + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: ${{ steps.guard.outputs.comment_id }} + number: ${{ steps.guard.outputs.pr_number }} + GITHUB_TOKEN: ${{ inputs.github_token }} + path: ${{ steps.wall.outputs.path }} + + - name: Upload the plan's wall + if: steps.wall.outputs.path != '' && github.server_url == 'https://github.com' + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: codeboarding-wall + path: ${{ runner.temp }}/codeboarding-wall/wall.json + if-no-files-found: ignore + retention-days: 14 + # Skipped when the run stopped on a credential problem: that path already replaced # this same sticky comment with the input and secret to fix, and "see the workflow # logs" posted over the top of it would send the reader hunting for what they had diff --git a/scripts/action/configure-auth.sh b/scripts/action/configure-auth.sh index 649eb63..0008dcc 100755 --- a/scripts/action/configure-auth.sh +++ b/scripts/action/configure-auth.sh @@ -33,6 +33,8 @@ if [ -s "$AUTH_DIR/license.txt" ]; then fi # Written by the preflight when the proxy gave this run an id; read per request. RELAY_ARGS+=(--run-id-file "$AUTH_DIR/run-id") +# Outside the auth directory, which goes with the analysis step: the wall outlives it. +RELAY_ARGS+=(--wall-file "$RUNNER_TEMP/codeboarding-wall/wall.json") python3 "$ACTION_PATH/scripts/oidc_relay.py" "${RELAY_ARGS[@]}" > "$LOG" 2>&1 & echo $! > "$PID" diff --git a/scripts/action/run_meter.py b/scripts/action/run_meter.py index 917c499..6826142 100755 --- a/scripts/action/run_meter.py +++ b/scripts/action/run_meter.py @@ -8,6 +8,9 @@ this step's outputs, and its run id is written where the relay packs it into every hosted call. +``wall`` turns a refusal, from the preflight or from a 402 mid-run, into the neutral pull +request comment and job summary. The run exits 0: a plan is never a red check. + ``finish`` runs last, on every outcome, and reports ``POST /run/finish``: ``produced`` charges the run, anything else releases its hold. Success is the map, not the exit code. @@ -21,6 +24,7 @@ import json import os import re +import shutil import sys from pathlib import Path from urllib.request import Request, urlopen @@ -85,6 +89,8 @@ def start(environ: dict[str, str]) -> tuple[dict[str, str], int]: auth_dir = Path(environ["RUNNER_TEMP"]) / "codeboarding-auth" for stale in (Path(environ["RUNNER_TEMP"]) / "codeboarding-map", auth_dir / "run-id"): stale.unlink(missing_ok=True) + wall_file = Path(environ["RUNNER_TEMP"]) / "codeboarding-wall" / "wall.json" + shutil.rmtree(wall_file.parent, ignore_errors=True) outputs = { "allowed": "true", "depth_cap": str(depth), @@ -124,14 +130,14 @@ def start(environ: dict[str, str]) -> tuple[dict[str, str], int]: return outputs, 0 run_id = answer.get("run_id") or "" - wall = answer.get("wall") or {} + refusal = answer.get("wall") or {} outputs.update( { "allowed": str(allowed).lower(), "depth_cap": str(min(depth, cap)), "run_id": run_id, "full_analysis": str(answer.get("full_analysis") is True).lower(), - "wall_message": " ".join(str(wall.get("message", "")).split()), + "wall_message": " ".join(str(refusal.get("message", "")).split()), "mode": str(answer.get("mode") or ""), } ) @@ -145,9 +151,48 @@ def start(environ: dict[str, str]) -> tuple[dict[str, str], int]: print(f"::notice title=CodeBoarding depth::{answer['depth_reason']}") if not allowed: print(f"::notice title=CodeBoarding::{outputs['wall_message'] or 'This run is over the plan allowance.'}") + if refusal: + wall_file.parent.mkdir(parents=True) + wall_file.write_text(json.dumps(refusal), encoding="utf-8") return outputs, 0 +def wall(environ: dict[str, str]) -> dict[str, str]: + """The neutral comment for a run the plan stopped, written once for the PR and the summary. + + wall.json is the wall payload exactly as the proxy sent it; the step after this uploads + it as the `codeboarding-wall` artifact, which is how the web app shows the same sentence + on the pull request's page. + """ + runner_temp = Path(environ["RUNNER_TEMP"]) + try: + payload = json.loads((runner_temp / "codeboarding-wall" / "wall.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + payload = {} + message = payload.get("message") or "CodeBoarding map not drawn: this run is over the plan's allowance." + links = [ + f"[{label}]({payload[key]})" + for key, label in (("plans_url", "Plans"), ("upgrade_url", "Your plan")) + if key in payload + ] + run_url = f"{environ['GITHUB_SERVER_URL']}/{environ['GITHUB_REPOSITORY']}/actions/runs/{environ['GITHUB_RUN_ID']}" + platform_url = f"https://app.codeboarding.org/{environ['GITHUB_REPOSITORY']}/pull/{environ.get('PR_NUMBER', '')}" + body = "\n\n".join( + [ + "### CodeBoarding review · map not drawn", + message, + *([" · ".join(links)] if links else []), + f"run [{environ['GITHUB_RUN_ID']}]({run_url})\n" + f"", + ] + ) + path = runner_temp / "wall-comment.md" + path.write_text(body + "\n", encoding="utf-8") + with open(environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: + summary.write(body + "\n") + return {"path": str(path)} + + def map_written(environ: dict[str, str]) -> bool: """Whether the analysis this run set out to write exists. @@ -178,6 +223,13 @@ def outcome(environ: dict[str, str]) -> str: def finish(environ: dict[str, str]) -> int: """Always 0: reporting the outcome must never be what fails the job.""" body = {"run_id": environ["RUN_ID"], "outcome": outcome(environ), "error": None} + if body["outcome"] != "produced": + try: + body["error"] = json.loads( + (Path(environ["RUNNER_TEMP"]) / "codeboarding-wall" / "wall.json").read_text(encoding="utf-8") + )["reason"][:200] + except (OSError, ValueError, KeyError, TypeError): + pass try: answer = post(environ, "/run/finish", body) except Exception as exc: # noqa: BLE001 - an unreported run is released after the stale-hold timeout @@ -189,12 +241,12 @@ def finish(environ: dict[str, str]) -> int: def main(argv: list[str]) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("command", choices=["start", "finish"]) + parser.add_argument("command", choices=["start", "wall", "finish"]) args = parser.parse_args(argv) environ = dict(os.environ) if args.command == "finish": return finish(environ) - outputs, code = start(environ) + outputs, code = start(environ) if args.command == "start" else (wall(environ), 0) with open(environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as handle: handle.writelines(f"{key}={value}\n" for key, value in outputs.items()) return code diff --git a/scripts/action/with-auth.sh b/scripts/action/with-auth.sh index 1b4f1ad..0758629 100755 --- a/scripts/action/with-auth.sh +++ b/scripts/action/with-auth.sh @@ -2,6 +2,7 @@ # Runs one command with the resolved provider credentials, then removes them. set -euo pipefail AUTH_DIR="${RUNNER_TEMP}/codeboarding-auth" +# shellcheck disable=SC2329 # run by the EXIT trap, which shellcheck misses once every path exits cleanup() { if [ -s "$AUTH_DIR/relay.pid" ]; then kill "$(cat "$AUTH_DIR/relay.pid")" 2>/dev/null || true @@ -41,4 +42,16 @@ if [ -n "${MODEL:-}" ]; then fi [ -z "${AGENT_MODEL_INPUT:-}" ] || export AGENT_MODEL="$AGENT_MODEL_INPUT" [ -z "${PARSING_MODEL_INPUT:-}" ] || export PARSING_MODEL="$PARSING_MODEL_INPUT" -"$@" +if "$@"; then + exit 0 +else + status=$? +fi +# The relay kept a 402's wall: the run stopped at the plan, which is not a failure of the +# job. The action ends it neutral instead of red, and a sync skips silently. +if [ -s "$RUNNER_TEMP/codeboarding-wall/wall.json" ]; then + echo "::notice title=CodeBoarding::The map was not drawn: this run reached the plan's limit." + echo "walled=true" >> "${GITHUB_OUTPUT:-/dev/null}" + exit 0 +fi +exit "$status" diff --git a/scripts/oidc_relay.py b/scripts/oidc_relay.py index 5d128dd..f2dbace 100644 --- a/scripts/oidc_relay.py +++ b/scripts/oidc_relay.py @@ -49,6 +49,7 @@ class RelayConfig: id_token_request_token: str license_file: Path | None = None run_id_file: Path | None = None + wall_file: Path | None = None def _with_audience(url: str) -> str: @@ -146,12 +147,28 @@ def _handle(self) -> None: with urlopen(request, timeout=310) as response: # nosec B310 - configured proxy URL self._send(response.status, dict(response.headers.items()), response.read()) except HTTPError as response: - self._send(response.code, dict(response.headers.items()), response.read()) + body = response.read() + if response.code == 402: + self._keep_wall(body) + self._send(response.code, dict(response.headers.items()), body) except (RuntimeError, URLError, OSError) as exc: body = json.dumps({"error": {"message": "CodeBoarding OIDC relay request failed."}}).encode() self._send(502, {"Content-Type": "application/json"}, body) print(f"OIDC relay request failed: {exc}", file=sys.stderr) + def _keep_wall(self, body: bytes) -> None: + """A 402 mid-run (the token ceiling, no live run) is the plan's wall, not a fault. The + engine only sees an error, so the wall is kept for the action to end the run neutral.""" + if self.config.wall_file is None: + return + try: + wall = json.loads(body).get("wall") + except (ValueError, AttributeError): + return + if isinstance(wall, dict): + self.config.wall_file.parent.mkdir(parents=True, exist_ok=True) + self.config.wall_file.write_text(json.dumps(wall), encoding="utf-8") + do_GET = _handle do_POST = _handle do_PUT = _handle @@ -174,6 +191,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--ready-file", required=True, type=Path) parser.add_argument("--license-file", type=Path) parser.add_argument("--run-id-file", type=Path) + parser.add_argument("--wall-file", type=Path) args = parser.parse_args(argv) request_url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL", "") @@ -183,7 +201,9 @@ def main(argv: list[str] | None = None) -> int: return 2 server = RelayServer( - RelayConfig(args.upstream_base_url, request_url, request_token, args.license_file, args.run_id_file) + RelayConfig( + args.upstream_base_url, request_url, request_token, args.license_file, args.run_id_file, args.wall_file + ) ) args.ready_file.write_text(str(server.server_port), encoding="utf-8") try: diff --git a/tests/test_action_auth.py b/tests/test_action_auth.py index 5b74d1b..dc0f9d8 100644 --- a/tests/test_action_auth.py +++ b/tests/test_action_auth.py @@ -307,6 +307,9 @@ def test_hosted_auth_relays_to_the_codeboarding_proxy(self) -> None: self.assertEqual(upstream, "https://auduihjmm4b735zci7vyabuikq0hppqn.lambda-url.us-east-1.on.aws") self.assertIn("--license-file", args) self.assertEqual(args[args.index("--run-id-file") + 1], str(auth_dir / "run-id")) + self.assertEqual( + args[args.index("--wall-file") + 1], str(temp_dir / "runner" / "codeboarding-wall" / "wall.json") + ) self.assertEqual((auth_dir / "env" / "OPENROUTER_API_KEY").read_text(), "github-actions-oidc-relay") self.assertEqual((auth_dir / "env" / "OPENROUTER_BASE_URL").read_text(), "http://127.0.0.1:12345") @@ -395,6 +398,28 @@ def test_direct_provider_runs_start_no_relay(self) -> None: self.assertEqual(configured.returncode, 0, configured.stderr or configured.stdout) self.assertFalse(marker.exists(), "a direct-provider run contacted the hosted relay") + def test_a_run_the_relay_saw_walled_ends_neutral_and_any_other_failure_does_not(self) -> None: + """A 402 mid-run is the plan's limit: the analysis step exits 0 and says `walled`, so + the action posts the wall instead of going red. Only when the command failed.""" + wall = Path(self.temp_dir.name) / "runner" / "codeboarding-wall" / "wall.json" + output = Path(self.temp_dir.name) / "github-output" + for script, has_wall, code, walled in ( + ("exit 3", True, 0, True), + ("exit 3", False, 3, False), + ("true", True, 0, False), + ): + with self.subTest(script=script, has_wall=has_wall): + self._preflight(CB_IN_LLM="anthropic", CB_IN_ANTHROPIC_API_KEY="k") + output.write_text("", encoding="utf-8") + wall.parent.mkdir(exist_ok=True) + if has_wall: + wall.write_text('{"reason": "token_ceiling"}', encoding="utf-8") + else: + wall.unlink(missing_ok=True) + scoped = self._with_auth(script, GITHUB_OUTPUT=str(output)) + self.assertEqual(scoped.returncode, code, scoped.stderr or scoped.stdout) + self.assertEqual("walled=true" in output.read_text(encoding="utf-8"), walled) + def test_analysis_refuses_to_run_without_a_resolved_plan(self) -> None: scoped = self._with_auth("true") self.assertNotEqual(scoped.returncode, 0) diff --git a/tests/test_action_inputs.py b/tests/test_action_inputs.py index 4e2490c..992930b 100644 --- a/tests/test_action_inputs.py +++ b/tests/test_action_inputs.py @@ -110,6 +110,39 @@ def test_the_finish_runs_last_on_every_outcome_and_never_fails_the_job(self) -> self.assertIn("steps.review_analyze.outputs.analysis_path || steps.sync_analyze.outputs.analysis_path", block) self.assertIn("JOB_STATUS: ${{ job.status }}", block) + def test_a_run_stopped_by_the_plan_skips_everything_that_needs_a_map(self) -> None: + for step in ( + "Render review diagram", + "Build review artifact", + "Upload review artifact", + "Build review comment", + "Post review comment", + ): + with self.subTest(step=step): + start = ACTION.index(f"- name: {step}\n") + condition = ACTION[start : ACTION.index("\n", ACTION.index("if:", start))] + self.assertIn("steps.review_analyze.outputs.walled != 'true'", condition) + start = ACTION.index("- name: Deliver baseline\n") + self.assertIn( + "steps.sync_analyze.outputs.walled != 'true'", + ACTION[start : ACTION.index("\n", ACTION.index("if:", start))], + ) + + def test_the_wall_is_a_neutral_comment_an_artifact_and_never_a_red_check(self) -> None: + """R6: CI never goes red because of a plan. Both walls, the preflight's refusal and a + 402 mid-run, end in the same sticky comment and the `codeboarding-wall` artifact.""" + start = ACTION.index("- name: Explain the plan's wall") + block = ACTION[start : ACTION.index("- name: Post review failure", start)] + self.assertIn( + "steps.preflight.outputs.allowed == 'false' || steps.review_analyze.outputs.walled == 'true'", block + ) + self.assertIn("steps.guard.outputs.mode == 'review'", block, "a sync at the ceiling skips silently") + self.assertIn("header: ${{ steps.guard.outputs.comment_id }}", block, "it replaces the progress comment") + self.assertIn("name: codeboarding-wall", block) + self.assertIn("codeboarding-wall/wall.json", block) + self.assertEqual(block.count("continue-on-error: true"), 3, "no wall step can fail the job") + self.assertNotIn("exit 1", block) + def test_license_key_is_deprecated_but_still_wired(self) -> None: self.assertIn("Deprecated", self.inputs["license_key"]) self.assertIn("CB_IN_LICENSE_KEY: ${{ inputs.license_key }}", ACTION) diff --git a/tests/test_oidc_relay.py b/tests/test_oidc_relay.py index 95fd87d..cca74d6 100644 --- a/tests/test_oidc_relay.py +++ b/tests/test_oidc_relay.py @@ -10,6 +10,7 @@ import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from urllib.error import HTTPError from urllib.request import Request, urlopen @@ -138,6 +139,60 @@ def bearer(**files): ) self.assertEqual(bearer(license_file=license_file), "Bearer jwt~codeboarding-license~LIC") + def test_a_402_wall_is_kept_for_the_action_and_still_relayed(self): + answers = [ + { + "error": {"message": "Weekly token ceiling reached", "type": "quota"}, + "wall": {"reason": "token_ceiling"}, + }, + {"error": {"message": "Resource exhausted: token limit reached"}}, + ] + + class Issuer(BaseHTTPRequestHandler): + def do_GET(self): + self._reply(200, {"value": "jwt"}) + + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + self._reply(402, answers.pop(0)) + + def _reply(self, code, payload): + data = json.dumps(payload).encode() + self.send_response(code) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *_args): + pass + + server = _Server(Issuer) + server.start() + self.addCleanup(server.close) + with tempfile.TemporaryDirectory() as temp: + wall_file = Path(temp) / "codeboarding-wall" / "wall.json" + relay = oidc_relay.RelayServer( + oidc_relay.RelayConfig(server.url, f"{server.url}/token", "request-token", wall_file=wall_file) + ) + thread = threading.Thread(target=relay.serve_forever, daemon=True) + thread.start() + try: + statuses = [] + for _ in range(2): + wall_file.unlink(missing_ok=True) + request = Request(f"http://127.0.0.1:{relay.server_port}/chat/completions", data=b"{}") + with self.assertRaises(HTTPError) as caught: + urlopen(request) + statuses.append((caught.exception.code, wall_file.exists())) + caught.exception.close() + if wall_file.exists(): + self.assertEqual(json.loads(wall_file.read_text()), {"reason": "token_ceiling"}) + finally: + relay.shutdown() + relay.server_close() + thread.join(timeout=2) + self.assertEqual(statuses, [(402, True), (402, False)], "a 402 without a wall is today's quota, not a wall") + def test_audience_replaces_an_existing_value(self): self.assertEqual( oidc_relay._with_audience("https://issuer.example/token?audience=old&x=1"), diff --git a/tests/test_run_meter.py b/tests/test_run_meter.py index 0ce657c..a7ea208 100644 --- a/tests/test_run_meter.py +++ b/tests/test_run_meter.py @@ -221,6 +221,67 @@ def test_a_depth_that_is_not_a_positive_integer_stops_the_run(self) -> None: _, code = run_meter.start(self._environ(None, DEPTH_CAP="0")) self.assertEqual(code, 1) + # -- the wall ---------------------------------------------------------- + + @NEEDS_JSONSCHEMA + def test_a_refused_preflight_keeps_the_wall_for_the_web_app(self) -> None: + answer = example("run-start.wall-runs.json") + outputs, _, _ = self._start(answer) + self.assertEqual(outputs["allowed"], "false") + wall = json.loads((self.runner_temp / "codeboarding-wall" / "wall.json").read_text()) + self.assertEqual(wall, answer["wall"]) + self.assertEqual(validate(wall, "wall.schema.json"), []) + + def test_an_allowed_run_clears_a_wall_left_by_an_earlier_invocation(self) -> None: + stale = self.runner_temp / "codeboarding-wall" / "wall.json" + stale.parent.mkdir() + stale.write_text(json.dumps(example("wall.token-ceiling.json"))) + self._start(example("run-start.allowed.json")) + self.assertFalse(stale.exists()) + + def test_the_wall_is_a_neutral_comment_and_the_step_exits_0(self) -> None: + wall = example("run-start.wall-runs.json")["wall"] + (self.runner_temp / "codeboarding-wall").mkdir() + (self.runner_temp / "codeboarding-wall" / "wall.json").write_text(json.dumps(wall)) + output, summary = self.root / "github-output", self.root / "summary.md" + result = subprocess.run( + [sys.executable, str(SCRIPT), "wall"], + env={ + "PATH": os.environ["PATH"], + "RUNNER_TEMP": str(self.runner_temp), + "GITHUB_OUTPUT": str(output), + "GITHUB_STEP_SUMMARY": str(summary), + "GITHUB_SERVER_URL": "https://github.com", + "GITHUB_REPOSITORY": "acme-corp/billing-service", + "GITHUB_RUN_ID": "11809532110", + "PR_NUMBER": "482", + }, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + path = output.read_text().strip().removeprefix("path=") + body = Path(path).read_text() + self.assertIn("### CodeBoarding review · map not drawn", body) + self.assertIn( + "CodeBoarding map not drawn: m.koch has used 5 of 5 free runs this week (resets Monday 00:00 UTC). " + "Pro gives 40 a week; a Team plan covers everyone in acme-corp.", + body, + ) + self.assertIn("(https://app.codeboarding.org/dashboard/plan)", body) + self.assertIn("wall=runs_exhausted", body) + self.assertNotIn("failed", body.lower()) + self.assertEqual(summary.read_text(), body) + + def test_a_run_stopped_by_a_402_reports_why_it_released(self) -> None: + (self.runner_temp / "codeboarding-wall").mkdir() + (self.runner_temp / "codeboarding-wall" / "wall.json").write_text( + json.dumps(example("wall.token-ceiling.json")) + ) + body, _ = self._finish(JOB_STATUS="success") + self.assertEqual((body["outcome"], body["error"]), ("failed", "token_ceiling")) + # -- the finish -------------------------------------------------------- def _finish(self, **environ: str) -> tuple[dict, str]: From c06dd070eefd2e4755be442190aa47e6231565fd Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Thu, 24 Sep 2026 02:22:08 +0200 Subject: [PATCH 2/4] chore: refresh the vendored licensing contracts Copied from CodeBoarding/licensing-aws contracts/ at paywall/4-consume-run-key (4094c9e): adds the billing, portal, session-extension and legacy-link schemas and the optional run key on /meter/consume. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/contracts/README.md | 4 ++ .../billing-checkout.request.schema.json | 30 ++++++++++++ .../billing-checkout.response.schema.json | 17 +++++++ .../billing-portal.request.schema.json | 17 +++++++ .../billing-portal.response.schema.json | 17 +++++++ .../examples/billing-checkout.created.json | 3 ++ .../examples/billing-checkout.request.json | 5 ++ .../examples/billing-portal.created.json | 3 ++ .../examples/billing-portal.request.json | 3 ++ .../examples/legacy-link.linked.json | 6 +++ .../examples/legacy-link.not-found.json | 6 +++ .../examples/legacy-link.request.email.json | 3 ++ .../examples/legacy-link.request.key.json | 3 ++ .../meter-consume.request.run-key.json | 12 +++++ .../examples/session-extension.request.json | 3 ++ .../contracts/legacy-link.request.schema.json | 34 +++++++++++++ .../legacy-link.response.schema.json | 49 +++++++++++++++++++ .../meter-consume.request.schema.json | 5 ++ .../session-extension.request.schema.json | 14 ++++++ 19 files changed, 234 insertions(+) create mode 100644 tests/contracts/billing-checkout.request.schema.json create mode 100644 tests/contracts/billing-checkout.response.schema.json create mode 100644 tests/contracts/billing-portal.request.schema.json create mode 100644 tests/contracts/billing-portal.response.schema.json create mode 100644 tests/contracts/examples/billing-checkout.created.json create mode 100644 tests/contracts/examples/billing-checkout.request.json create mode 100644 tests/contracts/examples/billing-portal.created.json create mode 100644 tests/contracts/examples/billing-portal.request.json create mode 100644 tests/contracts/examples/legacy-link.linked.json create mode 100644 tests/contracts/examples/legacy-link.not-found.json create mode 100644 tests/contracts/examples/legacy-link.request.email.json create mode 100644 tests/contracts/examples/legacy-link.request.key.json create mode 100644 tests/contracts/examples/meter-consume.request.run-key.json create mode 100644 tests/contracts/examples/session-extension.request.json create mode 100644 tests/contracts/legacy-link.request.schema.json create mode 100644 tests/contracts/legacy-link.response.schema.json create mode 100644 tests/contracts/session-extension.request.schema.json diff --git a/tests/contracts/README.md b/tests/contracts/README.md index 5fc4122..65717a2 100644 --- a/tests/contracts/README.md +++ b/tests/contracts/README.md @@ -9,6 +9,10 @@ JSON Schemas (draft 2020-12) for the wire shapes licensing-aws answers and accep | `run-start.request.schema.json`, `run-start.response.schema.json` | `POST /run/start` on gha_proxy (OIDC) and license_proxy (extension token) | | `run-finish.request.schema.json`, `run-finish.response.schema.json` | `POST /run/finish` on both | | `meter-consume.request.schema.json`, `meter-consume.response.schema.json` | `POST /meter/consume` | +| `billing-checkout.request.schema.json`, `billing-checkout.response.schema.json` | `POST /billing/checkout` | +| `billing-portal.request.schema.json`, `billing-portal.response.schema.json` | `POST /billing/portal` | +| `legacy-link.request.schema.json`, `legacy-link.response.schema.json` | `POST /legacy/link` (always 200; only `linked: true` is a link) | +| `session-extension.request.schema.json` | body of `POST /session/extension` (its answer is `session.schema.json`) | | `wall.schema.json` | the refusal inside the answers above, and the body of a 402 from either proxy (`{"error": {"message", "type"}, "wall": …}`) | | `meter.schema.json` | one weekly allowance, used by the others | diff --git a/tests/contracts/billing-checkout.request.schema.json b/tests/contracts/billing-checkout.request.schema.json new file mode 100644 index 0000000..013088b --- /dev/null +++ b/tests/contracts/billing-checkout.request.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/billing-checkout.request.schema.json", + "title": "Body of POST /billing/checkout (the webview server, for the signed-in person)", + "type": "object", + "properties": { + "interval": { + "enum": [ + "month", + "year" + ] + }, + "success_url": { + "type": "string", + "format": "uri", + "description": "Where Stripe sends the person after paying; must be on the app origin (APP_BASE_URL, and http://localhost:3000 on the dev stack)" + }, + "cancel_url": { + "type": "string", + "format": "uri", + "description": "Where Stripe sends the person when they leave Checkout; the same origin rule" + } + }, + "required": [ + "interval", + "success_url", + "cancel_url" + ], + "additionalProperties": false +} diff --git a/tests/contracts/billing-checkout.response.schema.json b/tests/contracts/billing-checkout.response.schema.json new file mode 100644 index 0000000..5211502 --- /dev/null +++ b/tests/contracts/billing-checkout.response.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/billing-checkout.response.schema.json", + "title": "Answer of POST /billing/checkout: the Stripe Checkout page to send the person to. Refusals: 400 (bad interval or a URL off the app origin), 409 {reason: subscribed} (already paying for Pro), 502 (Stripe did not answer), 503 (prices not configured)", + "type": "object", + "properties": { + "url": { + "type": "string", + "format": "uri", + "pattern": "^https://" + } + }, + "required": [ + "url" + ], + "additionalProperties": false +} diff --git a/tests/contracts/billing-portal.request.schema.json b/tests/contracts/billing-portal.request.schema.json new file mode 100644 index 0000000..d860ea0 --- /dev/null +++ b/tests/contracts/billing-portal.request.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/billing-portal.request.schema.json", + "title": "Body of POST /billing/portal (the webview server, for the signed-in person)", + "type": "object", + "properties": { + "return_url": { + "type": "string", + "format": "uri", + "description": "Where the portal's back link goes; must be on the app origin" + } + }, + "required": [ + "return_url" + ], + "additionalProperties": false +} diff --git a/tests/contracts/billing-portal.response.schema.json b/tests/contracts/billing-portal.response.schema.json new file mode 100644 index 0000000..5e3a0bb --- /dev/null +++ b/tests/contracts/billing-portal.response.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/billing-portal.response.schema.json", + "title": "Answer of POST /billing/portal: a short-lived Stripe Customer Portal link. Refusals: 403 {reason: no_customer} (the account has never paid; /me.billing.customer is false), 400, 502", + "type": "object", + "properties": { + "url": { + "type": "string", + "format": "uri", + "pattern": "^https://" + } + }, + "required": [ + "url" + ], + "additionalProperties": false +} diff --git a/tests/contracts/examples/billing-checkout.created.json b/tests/contracts/examples/billing-checkout.created.json new file mode 100644 index 0000000..4365089 --- /dev/null +++ b/tests/contracts/examples/billing-checkout.created.json @@ -0,0 +1,3 @@ +{ + "url": "https://checkout.stripe.com/c/pay/cs_test_a1B2c3D4" +} diff --git a/tests/contracts/examples/billing-checkout.request.json b/tests/contracts/examples/billing-checkout.request.json new file mode 100644 index 0000000..2fd8b5b --- /dev/null +++ b/tests/contracts/examples/billing-checkout.request.json @@ -0,0 +1,5 @@ +{ + "interval": "year", + "success_url": "https://app.codeboarding.org/dashboard/plan?checkout=success", + "cancel_url": "https://app.codeboarding.org/dashboard/plan?checkout=cancelled" +} diff --git a/tests/contracts/examples/billing-portal.created.json b/tests/contracts/examples/billing-portal.created.json new file mode 100644 index 0000000..008b5c1 --- /dev/null +++ b/tests/contracts/examples/billing-portal.created.json @@ -0,0 +1,3 @@ +{ + "url": "https://billing.stripe.com/p/session/test_YWNjdF8x" +} diff --git a/tests/contracts/examples/billing-portal.request.json b/tests/contracts/examples/billing-portal.request.json new file mode 100644 index 0000000..c9f7d2a --- /dev/null +++ b/tests/contracts/examples/billing-portal.request.json @@ -0,0 +1,3 @@ +{ + "return_url": "https://app.codeboarding.org/dashboard/plan" +} diff --git a/tests/contracts/examples/legacy-link.linked.json b/tests/contracts/examples/legacy-link.linked.json new file mode 100644 index 0000000..2e512ac --- /dev/null +++ b/tests/contracts/examples/legacy-link.linked.json @@ -0,0 +1,6 @@ +{ + "linked": true, + "plan": "pro", + "expires_at": "2026-12-12T10:00:00+00:00", + "reason": null +} diff --git a/tests/contracts/examples/legacy-link.not-found.json b/tests/contracts/examples/legacy-link.not-found.json new file mode 100644 index 0000000..d32de2c --- /dev/null +++ b/tests/contracts/examples/legacy-link.not-found.json @@ -0,0 +1,6 @@ +{ + "linked": false, + "plan": "free", + "expires_at": null, + "reason": "not_found" +} diff --git a/tests/contracts/examples/legacy-link.request.email.json b/tests/contracts/examples/legacy-link.request.email.json new file mode 100644 index 0000000..c460ca1 --- /dev/null +++ b/tests/contracts/examples/legacy-link.request.email.json @@ -0,0 +1,3 @@ +{ + "via": "email" +} diff --git a/tests/contracts/examples/legacy-link.request.key.json b/tests/contracts/examples/legacy-link.request.key.json new file mode 100644 index 0000000..8134ef4 --- /dev/null +++ b/tests/contracts/examples/legacy-link.request.key.json @@ -0,0 +1,3 @@ +{ + "license_key": "F4AD-252F-A642-D97F" +} diff --git a/tests/contracts/examples/meter-consume.request.run-key.json b/tests/contracts/examples/meter-consume.request.run-key.json new file mode 100644 index 0000000..2ca66e5 --- /dev/null +++ b/tests/contracts/examples/meter-consume.request.run-key.json @@ -0,0 +1,12 @@ +{ + "meter": "reading", + "repository": { + "provider": "github", + "id": 1296269, + "owner": "acme-corp", + "name": "billing-service", + "private": true + }, + "pr": 482, + "run": "run11809532110" +} diff --git a/tests/contracts/examples/session-extension.request.json b/tests/contracts/examples/session-extension.request.json new file mode 100644 index 0000000..f895da4 --- /dev/null +++ b/tests/contracts/examples/session-extension.request.json @@ -0,0 +1,3 @@ +{ + "editor": "Cursor" +} diff --git a/tests/contracts/legacy-link.request.schema.json b/tests/contracts/legacy-link.request.schema.json new file mode 100644 index 0000000..95a44fe --- /dev/null +++ b/tests/contracts/legacy-link.request.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/legacy-link.request.schema.json", + "title": "Body of POST /legacy/link: the extension hands over the license key it had stored; the plan page confirms the license /me offered (offers.legacy_link)", + "oneOf": [ + { + "type": "object", + "properties": { + "license_key": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "license_key" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "via": { + "enum": [ + "email" + ] + } + }, + "required": [ + "via" + ], + "additionalProperties": false + } + ] +} diff --git a/tests/contracts/legacy-link.response.schema.json b/tests/contracts/legacy-link.response.schema.json new file mode 100644 index 0000000..4a6e526 --- /dev/null +++ b/tests/contracts/legacy-link.response.schema.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/legacy-link.response.schema.json", + "title": "Answer of POST /legacy/link, always 200 for any key: only linked true means the license is now the account's", + "type": "object", + "properties": { + "linked": { + "type": "boolean" + }, + "plan": { + "enum": [ + "free", + "pro", + "max", + "team", + "enterprise" + ], + "description": "The account's plan after the call" + }, + "expires_at": { + "type": [ + "string", + "null" + ], + "format": "date-time", + "description": "When the linked license's paid period ends; null when nothing was linked" + }, + "reason": { + "enum": [ + null, + "already_linked", + "not_found", + "no_offer", + "expired", + "linked_elsewhere", + "custom_license", + "account_has_license" + ], + "description": "null: linked now. already_linked: this account had it (linked true). not_found: no such key, or the sign-in match no longer holds. no_offer: {via: email} with nothing offered. expired: canceled or past its end. linked_elsewhere: another account or a Team plan holds it. custom_license: converted to a Team plan with us. account_has_license: this account already holds another license" + } + }, + "required": [ + "linked", + "plan", + "expires_at", + "reason" + ], + "additionalProperties": false +} diff --git a/tests/contracts/meter-consume.request.schema.json b/tests/contracts/meter-consume.request.schema.json index 46d63f1..f9a2058 100644 --- a/tests/contracts/meter-consume.request.schema.json +++ b/tests/contracts/meter-consume.request.schema.json @@ -42,6 +42,11 @@ "pr": { "type": "integer", "minimum": 1 + }, + "run": { + "type": "string", + "pattern": "^run[0-9]{1,20}$", + "description": "Set when the map's artifact names no pull request: the reading is keyed on this workflow run instead of on pr" } }, "required": [ diff --git a/tests/contracts/session-extension.request.schema.json b/tests/contracts/session-extension.request.schema.json new file mode 100644 index 0000000..2a214de --- /dev/null +++ b/tests/contracts/session-extension.request.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/session-extension.request.schema.json", + "title": "Body of POST /session/extension (the connect page, with a web session); the answer is session.schema.json", + "type": "object", + "properties": { + "editor": { + "type": "string", + "maxLength": 64, + "description": "vscode.env.appName, e.g. 'Visual Studio Code' or 'Cursor'; listed on the plan page" + } + }, + "additionalProperties": false +} From 8b42bb70fc3aa7c96bccee8396f01d83a46ad67c Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Thu, 24 Sep 2026 19:30:25 +0200 Subject: [PATCH 3/4] chore: refresh the vendored licensing contracts Copied from CodeBoarding/licensing-aws contracts/ at paywall/7-harness (55c2403): /me has no offers.legacy_link and /legacy/link takes only {license_key}, since license holders are linked by an admin backfill, not by an email match (Svilen, 24 Sep). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/legacy-link.request.email.json | 3 -- .../contracts/examples/me.free-exhausted.json | 5 +-- .../contracts/examples/me.free-one-left.json | 3 +- tests/contracts/examples/me.past-due.json | 3 +- tests/contracts/examples/me.pro.json | 3 +- tests/contracts/examples/me.team.json | 3 +- tests/contracts/examples/me.trial.json | 3 +- .../contracts/legacy-link.request.schema.json | 40 +++++-------------- .../legacy-link.response.schema.json | 3 +- tests/contracts/me.schema.json | 23 +---------- 10 files changed, 19 insertions(+), 70 deletions(-) delete mode 100644 tests/contracts/examples/legacy-link.request.email.json diff --git a/tests/contracts/examples/legacy-link.request.email.json b/tests/contracts/examples/legacy-link.request.email.json deleted file mode 100644 index c460ca1..0000000 --- a/tests/contracts/examples/legacy-link.request.email.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "via": "email" -} diff --git a/tests/contracts/examples/me.free-exhausted.json b/tests/contracts/examples/me.free-exhausted.json index 92a55a4..85b2121 100644 --- a/tests/contracts/examples/me.free-exhausted.json +++ b/tests/contracts/examples/me.free-exhausted.json @@ -72,10 +72,7 @@ } ], "offers": { - "cancel_personal_pro": false, - "legacy_link": { - "license": "Pro license from May 2026" - } + "cancel_personal_pro": false }, "billing": { "customer": false diff --git a/tests/contracts/examples/me.free-one-left.json b/tests/contracts/examples/me.free-one-left.json index 3067de9..b29781e 100644 --- a/tests/contracts/examples/me.free-one-left.json +++ b/tests/contracts/examples/me.free-one-left.json @@ -72,8 +72,7 @@ } ], "offers": { - "cancel_personal_pro": false, - "legacy_link": null + "cancel_personal_pro": false }, "billing": { "customer": false diff --git a/tests/contracts/examples/me.past-due.json b/tests/contracts/examples/me.past-due.json index 4d5a2e8..80d76d8 100644 --- a/tests/contracts/examples/me.past-due.json +++ b/tests/contracts/examples/me.past-due.json @@ -72,8 +72,7 @@ } ], "offers": { - "cancel_personal_pro": false, - "legacy_link": null + "cancel_personal_pro": false }, "billing": { "customer": true diff --git a/tests/contracts/examples/me.pro.json b/tests/contracts/examples/me.pro.json index c480e44..ca59f4f 100644 --- a/tests/contracts/examples/me.pro.json +++ b/tests/contracts/examples/me.pro.json @@ -72,8 +72,7 @@ } ], "offers": { - "cancel_personal_pro": false, - "legacy_link": null + "cancel_personal_pro": false }, "billing": { "customer": true diff --git a/tests/contracts/examples/me.team.json b/tests/contracts/examples/me.team.json index 6f2a209..4276242 100644 --- a/tests/contracts/examples/me.team.json +++ b/tests/contracts/examples/me.team.json @@ -75,8 +75,7 @@ } ], "offers": { - "cancel_personal_pro": true, - "legacy_link": null + "cancel_personal_pro": true }, "billing": { "customer": false diff --git a/tests/contracts/examples/me.trial.json b/tests/contracts/examples/me.trial.json index 68b9c7a..2f8c33a 100644 --- a/tests/contracts/examples/me.trial.json +++ b/tests/contracts/examples/me.trial.json @@ -72,8 +72,7 @@ } ], "offers": { - "cancel_personal_pro": false, - "legacy_link": null + "cancel_personal_pro": false }, "billing": { "customer": false diff --git a/tests/contracts/legacy-link.request.schema.json b/tests/contracts/legacy-link.request.schema.json index 95a44fe..a372b00 100644 --- a/tests/contracts/legacy-link.request.schema.json +++ b/tests/contracts/legacy-link.request.schema.json @@ -1,34 +1,16 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://codeboarding.org/schemas/licensing/v1/legacy-link.request.schema.json", - "title": "Body of POST /legacy/link: the extension hands over the license key it had stored; the plan page confirms the license /me offered (offers.legacy_link)", - "oneOf": [ - { - "type": "object", - "properties": { - "license_key": { - "type": "string", - "minLength": 1 - } - }, - "required": [ - "license_key" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "via": { - "enum": [ - "email" - ] - } - }, - "required": [ - "via" - ], - "additionalProperties": false + "title": "Body of POST /legacy/link: the extension hands over the license key it had stored", + "type": "object", + "properties": { + "license_key": { + "type": "string", + "minLength": 1 } - ] + }, + "required": [ + "license_key" + ], + "additionalProperties": false } diff --git a/tests/contracts/legacy-link.response.schema.json b/tests/contracts/legacy-link.response.schema.json index 4a6e526..4df1a79 100644 --- a/tests/contracts/legacy-link.response.schema.json +++ b/tests/contracts/legacy-link.response.schema.json @@ -30,13 +30,12 @@ null, "already_linked", "not_found", - "no_offer", "expired", "linked_elsewhere", "custom_license", "account_has_license" ], - "description": "null: linked now. already_linked: this account had it (linked true). not_found: no such key, or the sign-in match no longer holds. no_offer: {via: email} with nothing offered. expired: canceled or past its end. linked_elsewhere: another account or a Team plan holds it. custom_license: converted to a Team plan with us. account_has_license: this account already holds another license" + "description": "null: linked now. already_linked: this account had it (linked true). not_found: no such key. expired: canceled or past its end. linked_elsewhere: another account or a Team plan holds it. custom_license: converted to a Team plan with us. account_has_license: this account already holds another license" } }, "required": [ diff --git a/tests/contracts/me.schema.json b/tests/contracts/me.schema.json index b1d6a1d..4a6d50a 100644 --- a/tests/contracts/me.schema.json +++ b/tests/contracts/me.schema.json @@ -290,31 +290,10 @@ "properties": { "cancel_personal_pro": { "type": "boolean" - }, - "legacy_link": { - "anyOf": [ - { - "type": "object", - "properties": { - "license": { - "type": "string", - "description": "e.g. 'Pro license from May 2026'" - } - }, - "required": [ - "license" - ], - "additionalProperties": false - }, - { - "type": "null" - } - ] } }, "required": [ - "cancel_personal_pro", - "legacy_link" + "cancel_personal_pro" ], "additionalProperties": false }, From dafbb499596abf4e9c3043f9d13b4118a8a0adee Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Fri, 25 Sep 2026 13:22:28 +0200 Subject: [PATCH 4/4] test: pin the wall and red paths for an engine that stops on a mid-run 402 The engine now exits 3 on a 402 instead of drawing a folder-named map. With the relay holding the plan's wall that is still the neutral path (walled, green, released as the wall's reason); without one the run fails red, and the finish step now names it `quota_exhausted` from the engine's record rather than reporting no reason. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/action/run_meter.py | 26 ++++++++++--- tests/test_action_state.py | 73 ++++++++++++++++++++++++++++++++++++- tests/test_run_meter.py | 15 ++++++++ 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/scripts/action/run_meter.py b/scripts/action/run_meter.py index 6826142..2a77f79 100755 --- a/scripts/action/run_meter.py +++ b/scripts/action/run_meter.py @@ -220,16 +220,30 @@ def outcome(environ: dict[str, str]) -> str: return "failed" +#: The engine's own refusals, which analyze_repository.py records when it stops on one. A 402 +#: without a `wall` (a quota the plan did not explain) is still a quota, and says so here. +ENGINE_ERRORS = {"llm_quota_exhausted": "quota_exhausted", "llm_auth": "llm_auth_rejected"} + + +def failure_reason(environ: dict[str, str]) -> str | None: + """The plan's wall reason when there is one, else the engine's recorded refusal.""" + runner_temp = Path(environ["RUNNER_TEMP"]) + try: + return json.loads((runner_temp / "codeboarding-wall" / "wall.json").read_text(encoding="utf-8"))["reason"][:200] + except (OSError, ValueError, KeyError, TypeError): + pass + try: + kind = json.loads((runner_temp / "codeboarding-engine-error.json").read_text(encoding="utf-8"))["kind"] + return ENGINE_ERRORS.get(kind) + except (OSError, ValueError, KeyError, TypeError): + return None + + def finish(environ: dict[str, str]) -> int: """Always 0: reporting the outcome must never be what fails the job.""" body = {"run_id": environ["RUN_ID"], "outcome": outcome(environ), "error": None} if body["outcome"] != "produced": - try: - body["error"] = json.loads( - (Path(environ["RUNNER_TEMP"]) / "codeboarding-wall" / "wall.json").read_text(encoding="utf-8") - )["reason"][:200] - except (OSError, ValueError, KeyError, TypeError): - pass + body["error"] = failure_reason(environ) try: answer = post(environ, "/run/finish", body) except Exception as exc: # noqa: BLE001 - an unreported run is released after the stale-hold timeout diff --git a/tests/test_action_state.py b/tests/test_action_state.py index e6bf574..3c33f58 100644 --- a/tests/test_action_state.py +++ b/tests/test_action_state.py @@ -9,6 +9,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock ROOT = Path(__file__).resolve().parent.parent @@ -18,6 +19,7 @@ _SPEC.loader.exec_module(run_meter) STATE_NAMES = ROOT / "scripts" / "action" / "state-names.sh" ANALYZE = ROOT / "scripts" / "action" / "analyze.sh" +WITH_AUTH = ROOT / "scripts" / "action" / "with-auth.sh" ENGINE_STUB = '''#!/usr/bin/env python3 """CodeBoarding CLI stand-in: records each call and writes a minimal analysis.""" @@ -39,6 +41,16 @@ json.dump({"metadata": metadata, "components": []}, open(analysis, "w")) print(json.dumps({"requiresFullAnalysis": True})) sys.exit(0) +if os.environ.get("CB_ENGINE_ABORT"): + # A 402 mid-run: the relay keeps the body's `wall` when it has one, then the engine + # stops with its refusal contract (a JSON verdict, no analysis written, exit 3). + if os.environ.get("CB_RELAY_WALL"): + wall = os.path.join(os.environ["RUNNER_TEMP"], "codeboarding-wall", "wall.json") + os.makedirs(os.path.dirname(wall), exist_ok=True) + open(wall, "w").write(os.environ["CB_RELAY_WALL"]) + print(json.dumps({"mode": argv[0], "error": "LLM quota exhausted", "kind": os.environ["CB_ENGINE_ABORT"], + "statusCode": 402, "provider": "codeboarding", "requiresFullAnalysis": False})) + sys.exit(3) if argv[0] == "full": if os.environ.get("CB_FULL_CRASHES") == "before_write": sys.exit(1) @@ -186,10 +198,10 @@ def setUp(self) -> None: def tearDown(self) -> None: self.temp_dir.cleanup() - def _analyze(self, *, check: bool = True, **extra: str) -> dict[str, str]: + def _analyze(self, *, check: bool = True, with_auth: bool = False, **extra: str) -> dict[str, str]: self.output.write_text("", encoding="utf-8") result = subprocess.run( - [str(ANALYZE)], + [str(WITH_AUTH), str(ANALYZE)] if with_auth else [str(ANALYZE)], env={ "PATH": f"{self.bin_dir}:{os.environ['PATH']}", "GITHUB_OUTPUT": str(self.output), @@ -216,6 +228,7 @@ def _analyze(self, *, check: bool = True, **extra: str) -> dict[str, str]: ) if check: self.assertEqual(result.returncode, 0, result.stderr or result.stdout) + self.returncode = result.returncode values: dict[str, str] = {} for line in self.output.read_text(encoding="utf-8").splitlines(): key, _, value = line.partition("=") @@ -320,6 +333,62 @@ def test_an_incremental_map_does_not_count_when_the_full_fallback_fails(self) -> self.assertEqual([c["mode"] for c in self._engine_calls()], ["incremental", "full"]) self.assertFalse(self._map_written()) + def _finish_body(self, values: dict[str, str], job_status: str) -> dict: + """What the finish step would report for this analysis step's outputs.""" + environ = { + "RUN_ID": "github:o/r#1", + "RUNNER_TEMP": str(self.runner_temp), + "JOB_STATUS": job_status, + "ANALYSIS_PATH": values.get("analysis_path", ""), + "MAP_MARKER": str(self.runner_temp / "codeboarding-map"), + } + with mock.patch.object(run_meter, "post", return_value={}) as post, mock.patch("sys.stdout"): + run_meter.finish(environ) + return post.call_args.args[2] + + def _mid_run_402(self, kind: str, **extra: str) -> dict[str, str]: + # with-auth.sh reads the plan configure-auth.sh would have written. + auth = self.runner_temp / "codeboarding-auth" + auth.mkdir() + (auth / "tier").write_text("hosted", encoding="utf-8") + (auth / "provider-name").write_text("codeboarding", encoding="utf-8") + _state(self.base_dir, cap=4) + if kind == "sync": + _state(self.checkout / ".codeboarding", cap=4) + extra = {"ANALYSIS_KIND": "sync", "FORCE_FULL": "false", **extra} + return self._analyze(check=False, with_auth=True, DEPTH_CAP="4", CB_ENGINE_ABORT="llm_quota_exhausted", **extra) + + def test_a_mid_run_402_with_a_wall_ends_neutral_and_is_not_charged(self) -> None: + """The engine now exits 3 on a 402 instead of drawing a folder-named map. With the + relay holding the plan's wall, that is the neutral path: walled, green, no map.""" + wall = json.dumps({"reason": "token_ceiling", "message": "m"}) + for kind in ("review", "sync"): + with self.subTest(kind=kind): + self.tearDown() + self.setUp() + values = self._mid_run_402(kind, CB_RELAY_WALL=wall) + self.assertEqual(self.returncode, 0) + self.assertEqual(values.get("walled"), "true") + self.assertNotIn("analysis_path", values) + self.assertFalse(self.stage_dir.exists(), "nothing staged for delivery or publishing") + body = self._finish_body(values, job_status="success") + self.assertEqual((body["outcome"], body["error"]), ("failed", "token_ceiling")) + + def test_a_mid_run_402_without_a_wall_fails_red_and_names_the_quota(self) -> None: + """A 402 the plan did not explain (today's legacy quota) is a failure, not a wall.""" + values = self._mid_run_402("review") + self.assertNotEqual(self.returncode, 0) + self.assertNotIn("walled", values) + self.assertFalse(self.stage_dir.exists()) + self.assertEqual(self._finish_body(values, job_status="failure")["outcome"], "failed") + # analyze_repository.py records the refusal once it reads the engine's verdict (#124); + # the finish step names it from that record. + (self.runner_temp / "codeboarding-engine-error.json").write_text( + json.dumps({"kind": "llm_quota_exhausted", "statusCode": 402, "exitCode": 3}), encoding="utf-8" + ) + body = self._finish_body(values, job_status="failure") + self.assertEqual((body["outcome"], body["error"]), ("failed", "quota_exhausted")) + def test_the_head_analysis_is_named_for_the_finish_step_before_it_runs(self) -> None: _state(self.base_dir, cap=4) values = self._analyze(DEPTH_CAP="4") diff --git a/tests/test_run_meter.py b/tests/test_run_meter.py index a7ea208..c22c80f 100644 --- a/tests/test_run_meter.py +++ b/tests/test_run_meter.py @@ -282,6 +282,21 @@ def test_a_run_stopped_by_a_402_reports_why_it_released(self) -> None: body, _ = self._finish(JOB_STATUS="success") self.assertEqual((body["outcome"], body["error"]), ("failed", "token_ceiling")) + def test_a_quota_the_plan_did_not_explain_is_reported_as_a_quota(self) -> None: + """A 402 with no `wall` fails red; its release still says why, from the engine's record.""" + (self.runner_temp / "codeboarding-engine-error.json").write_text(json.dumps({"kind": "llm_quota_exhausted"})) + body, _ = self._finish(JOB_STATUS="failure") + self.assertEqual((body["outcome"], body["error"]), ("failed", "quota_exhausted")) + + def test_the_plan_wall_outranks_the_engine_record(self) -> None: + (self.runner_temp / "codeboarding-wall").mkdir() + (self.runner_temp / "codeboarding-wall" / "wall.json").write_text( + json.dumps(example("wall.token-ceiling.json")) + ) + (self.runner_temp / "codeboarding-engine-error.json").write_text(json.dumps({"kind": "llm_quota_exhausted"})) + body, _ = self._finish(JOB_STATUS="success") + self.assertEqual(body["error"], "token_ceiling") + # -- the finish -------------------------------------------------------- def _finish(self, **environ: str) -> tuple[dict, str]: