From 760f9d47b7fda1d0ea8f5068425bc02dde967aeb Mon Sep 17 00:00:00 2001 From: "jeffrey.d.leach" Date: Wed, 7 Oct 2026 18:22:52 +0000 Subject: [PATCH 1/5] Project Requirements gate: accept DEVIN_ORG_ID from a variable or a secret Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/project-requirements.yml | 2 +- docs/reference/project-requirements-gate.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/project-requirements.yml b/.github/workflows/project-requirements.yml index 2c76766..a475da2 100644 --- a/.github/workflows/project-requirements.yml +++ b/.github/workflows/project-requirements.yml @@ -55,7 +55,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_number }} DEVIN_API_KEY: ${{ secrets.DEVIN_API_KEY }} - DEVIN_ORG_ID: ${{ vars.DEVIN_ORG_ID }} + DEVIN_ORG_ID: ${{ vars.DEVIN_ORG_ID || secrets.DEVIN_ORG_ID }} DEVIN_API_URL: ${{ vars.DEVIN_API_URL }} DEVIN_MAX_ACU: ${{ vars.PROJECT_REQUIREMENTS_MAX_ACU }} WAIVER_LABEL: ${{ vars.PROJECT_REQUIREMENTS_WAIVER_LABEL }} diff --git a/docs/reference/project-requirements-gate.md b/docs/reference/project-requirements-gate.md index e3cf81c..91106ea 100644 --- a/docs/reference/project-requirements-gate.md +++ b/docs/reference/project-requirements-gate.md @@ -51,7 +51,7 @@ Repository settings (**Settings > Secrets and variables > Actions**): | Name | Kind | Required | Purpose | | --- | --- | --- | --- | | `DEVIN_API_KEY` | Secret | Yes | Devin service user API key with permission to create sessions. | -| `DEVIN_ORG_ID` | Variable | Yes | Devin organization ID (`org-...`) that sessions run in. | +| `DEVIN_ORG_ID` | Variable (or secret) | Yes | Devin organization ID (`org-...`) that sessions run in. Read from the variable first, then the secret. | | `DEVIN_API_URL` | Variable | No | Devin API base URL. Defaults to `https://api.devin.ai`; set it for dedicated deployments. | | `PROJECT_REQUIREMENTS_MAX_ACU` | Variable | No | ACU limit per session. | | `PROJECT_REQUIREMENTS_WAIVER_LABEL` | Variable | No | Waiver label name. Defaults to `requirements-waived`. | From 8654058e4be68280cebb65ce44a60086734f9f1f Mon Sep 17 00:00:00 2001 From: "jeffrey.d.leach" Date: Wed, 7 Oct 2026 18:35:11 +0000 Subject: [PATCH 2/5] Re-trigger Project Requirements gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> From c5a2b4801b86357e4784c921b4a4e25b68e98f09 Mon Sep 17 00:00:00 2001 From: "jeffrey.d.leach" Date: Wed, 7 Oct 2026 18:38:39 +0000 Subject: [PATCH 3/5] Test DEVIN_ORG_ID workflow fallback; run gate tests when the gate workflow changes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../tests/test_project_requirements_gate.py | 32 +++++++++++++++++++ .../workflows/project-requirements-tests.yml | 2 ++ 2 files changed, 34 insertions(+) diff --git a/.github/scripts/tests/test_project_requirements_gate.py b/.github/scripts/tests/test_project_requirements_gate.py index b988df3..1285b9c 100644 --- a/.github/scripts/tests/test_project_requirements_gate.py +++ b/.github/scripts/tests/test_project_requirements_gate.py @@ -1,5 +1,6 @@ import io import json +import re import sys from pathlib import Path @@ -301,3 +302,34 @@ def fake_urlopen(request, timeout): with pytest.raises(gate.GateError, match="returned HTTP .*: boom"): gate.http_json("GET", "https://x", "tok", attempts=2) assert len(seen) == calls + + +WORKFLOW = Path(__file__).resolve().parents[2] / "workflows" / "project-requirements.yml" + + +def resolve_workflow_env(name, context): + """Evaluate a workflow env entry of the form `${{ a.B || c.D }}` against `context`.""" + match = re.search(rf"^\s+{name}: \$\{{\{{(.+?)\}}\}}\s*$", WORKFLOW.read_text(), re.MULTILINE) + assert match, f"{name} not found in {WORKFLOW.name}" + for ref in match.group(1).split("||"): + scope, key = ref.strip().split(".", 1) + value = context.get(scope, {}).get(key, "") + if value: + return value + return "" + + +@pytest.mark.parametrize("context, expected", [ + ({"vars": {"DEVIN_ORG_ID": "org-var"}}, "org-var"), + ({"secrets": {"DEVIN_ORG_ID": "org-secret"}}, "org-secret"), + ({"vars": {"DEVIN_ORG_ID": "org-var"}, "secrets": {"DEVIN_ORG_ID": "org-secret"}}, "org-var"), +]) +def test_workflow_org_id_accepts_variable_or_secret(tmp_path, context, expected): + org_id = resolve_workflow_env("DEVIN_ORG_ID", context) + assert make_config(tmp_path, DEVIN_ORG_ID=org_id).devin_org_id == expected + + +def test_workflow_org_id_missing_from_both_is_a_config_error(tmp_path): + org_id = resolve_workflow_env("DEVIN_ORG_ID", {}) + with pytest.raises(gate.GateError, match="Missing configuration: DEVIN_ORG_ID"): + make_config(tmp_path, DEVIN_ORG_ID=org_id) diff --git a/.github/workflows/project-requirements-tests.yml b/.github/workflows/project-requirements-tests.yml index 84a2db6..ed8efa6 100644 --- a/.github/workflows/project-requirements-tests.yml +++ b/.github/workflows/project-requirements-tests.yml @@ -9,11 +9,13 @@ on: paths: - ".github/scripts/**" - ".github/workflows/project-requirements-tests.yml" + - ".github/workflows/project-requirements.yml" pull_request: branches: ["**"] paths: - ".github/scripts/**" - ".github/workflows/project-requirements-tests.yml" + - ".github/workflows/project-requirements.yml" permissions: contents: read From 7e20e3ee027f5de8d44e595db1b068e1d39ee4f6 Mon Sep 17 00:00:00 2001 From: "jeffrey.d.leach" Date: Wed, 7 Oct 2026 18:48:33 +0000 Subject: [PATCH 4/5] Report configuration errors as a PR comment and error status Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/scripts/project_requirements_gate.py | 35 +++++++++++++------ .../tests/test_project_requirements_gate.py | 29 +++++++++++++++ docs/reference/project-requirements-gate.md | 4 +++ 3 files changed, 58 insertions(+), 10 deletions(-) diff --git a/.github/scripts/project_requirements_gate.py b/.github/scripts/project_requirements_gate.py index 604f1cf..33764ec 100755 --- a/.github/scripts/project_requirements_gate.py +++ b/.github/scripts/project_requirements_gate.py @@ -22,6 +22,7 @@ STATUS_CONTEXT = "Project Requirements" DEFAULT_CRITERIA_PATH = ".github/project-requirements/criteria.md" +SETUP_DOC = "docs/reference/project-requirements-gate.md" COMMENT_MARKER = "" VERDICTS = ("pass", "concern", "fail") REQUIREMENTS: Tuple[Tuple[str, str], ...] = ( @@ -89,13 +90,11 @@ class Config: poll_seconds: int @classmethod - def from_env(cls, env: Dict[str, str]) -> "Config": - missing = [ - name - for name in ("GITHUB_REPOSITORY", "PR_NUMBER", "GITHUB_TOKEN", - "DEVIN_API_KEY", "DEVIN_ORG_ID") - if not env.get(name) - ] + def from_env(cls, env: Dict[str, str], require_devin: bool = True) -> "Config": + required = ("GITHUB_REPOSITORY", "PR_NUMBER", "GITHUB_TOKEN") + if require_devin: + required += ("DEVIN_API_KEY", "DEVIN_ORG_ID") + missing = [name for name in required if not env.get(name)] if missing: raise GateError("Missing configuration: " + ", ".join(missing)) server = env.get("GITHUB_SERVER_URL", "https://github.com") @@ -109,8 +108,8 @@ def from_env(cls, env: Dict[str, str]) -> "Config": github_token=env["GITHUB_TOKEN"], github_api_url=env.get("GITHUB_API_URL", "https://api.github.com").rstrip("/"), run_url=run_url, - devin_api_key=env["DEVIN_API_KEY"], - devin_org_id=env["DEVIN_ORG_ID"], + devin_api_key=env.get("DEVIN_API_KEY", ""), + devin_org_id=env.get("DEVIN_ORG_ID", ""), devin_api_url=(env.get("DEVIN_API_URL") or "https://api.devin.ai").rstrip("/"), criteria_path=Path(env.get("CRITERIA_PATH") or DEFAULT_CRITERIA_PATH), waiver_label=env.get("WAIVER_LABEL") or "requirements-waived", @@ -480,11 +479,27 @@ def run(config: Config, github: GitHub, devin: Devin) -> int: return 0 if state == "success" else 1 +def report_setup_error(env: Dict[str, str], message: str, http: Http = http_json) -> None: + """Surface a configuration error on the PR when GitHub access is still available.""" + try: + config = Config.from_env(env, require_devin=False) + github = GitHub(config, http) + sha = github.pull_request()["head"]["sha"] + github.set_status(sha, "error", f"Gate error: {message}", config.run_url) + github.upsert_comment(render_error( + f"{message}. Set it under Settings > Secrets and variables > Actions " + f"(see {SETUP_DOC}).", sha, config.run_url)) + except (GateError, OSError, ValueError, KeyError, TypeError) as err: + print(f"::warning::Could not report the configuration error on the PR: {err}") + + def main() -> int: + env = dict(os.environ) try: - config = Config.from_env(dict(os.environ)) + config = Config.from_env(env) except (GateError, ValueError) as err: print(f"::error::{err}") + report_setup_error(env, str(err)) return 1 return run(config, GitHub(config), Devin(config)) diff --git a/.github/scripts/tests/test_project_requirements_gate.py b/.github/scripts/tests/test_project_requirements_gate.py index 1285b9c..573f041 100644 --- a/.github/scripts/tests/test_project_requirements_gate.py +++ b/.github/scripts/tests/test_project_requirements_gate.py @@ -333,3 +333,32 @@ def test_workflow_org_id_missing_from_both_is_a_config_error(tmp_path): org_id = resolve_workflow_env("DEVIN_ORG_ID", {}) with pytest.raises(gate.GateError, match="Missing configuration: DEVIN_ORG_ID"): make_config(tmp_path, DEVIN_ORG_ID=org_id) + + +def test_report_setup_error_comments_and_sets_error_status(tmp_path): + env = {"GITHUB_REPOSITORY": "NASA-AMMOS/tig", "PR_NUMBER": "7", "GITHUB_TOKEN": "gh-token", + "GITHUB_RUN_ID": "99", "DEVIN_API_KEY": "devin-key"} + api = FakeApi() + gate.report_setup_error(env, "Missing configuration: DEVIN_ORG_ID", api) + status = api.statuses()[-1] + assert status["state"] == "error" + assert status["description"] == "Gate error: Missing configuration: DEVIN_ORG_ID" + comment = next(b for m, u, b in api.calls if m == "POST" and u.endswith("/issues/7/comments")) + assert "Project Requirements: ERROR" in comment["body"] + assert "Missing configuration: DEVIN_ORG_ID" in comment["body"] + assert gate.SETUP_DOC in comment["body"] + + +def test_report_setup_error_without_github_access_only_warns(capsys): + api = FakeApi() + gate.report_setup_error({"PR_NUMBER": "7"}, "Missing configuration: GITHUB_TOKEN", api) + assert api.calls == [] + assert "Could not report the configuration error" in capsys.readouterr().out + + +def test_report_setup_error_survives_github_failure(): + def failing_http(method, url, token, body=None): + raise gate.GateError("GET pulls returned HTTP 403") + + env = {"GITHUB_REPOSITORY": "NASA-AMMOS/tig", "PR_NUMBER": "7", "GITHUB_TOKEN": "gh-token"} + gate.report_setup_error(env, "Missing configuration: DEVIN_ORG_ID", failing_http) diff --git a/docs/reference/project-requirements-gate.md b/docs/reference/project-requirements-gate.md index 91106ea..b03c2c7 100644 --- a/docs/reference/project-requirements-gate.md +++ b/docs/reference/project-requirements-gate.md @@ -34,6 +34,10 @@ not apply them. They are used only by this gate. | No verdict (API error, timeout, malformed output) | error | | Waived by a maintainer | success | + The comment always states the reason. When the gate cannot evaluate the + PR at all, for example because `DEVIN_ORG_ID` or `DEVIN_API_KEY` is not + set, the comment shows the error and how to fix it. + A run takes as long as the Devin session, usually several minutes. Pushing again cancels the previous run. From e045977fadb68360778a778bdbb8803725a7cb7a Mon Sep 17 00:00:00 2001 From: "jeffrey.d.leach" Date: Wed, 7 Oct 2026 21:12:14 +0000 Subject: [PATCH 5/5] Report malformed numeric settings on the PR; test main() setup-error path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/scripts/project_requirements_gate.py | 10 ++++--- .../tests/test_project_requirements_gate.py | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/.github/scripts/project_requirements_gate.py b/.github/scripts/project_requirements_gate.py index 33764ec..a14b27c 100755 --- a/.github/scripts/project_requirements_gate.py +++ b/.github/scripts/project_requirements_gate.py @@ -23,6 +23,7 @@ STATUS_CONTEXT = "Project Requirements" DEFAULT_CRITERIA_PATH = ".github/project-requirements/criteria.md" SETUP_DOC = "docs/reference/project-requirements-gate.md" +DEVIN_NUMERIC_SETTINGS = ("DEVIN_MAX_ACU", "DEVIN_TIMEOUT_MINUTES", "DEVIN_POLL_SECONDS") COMMENT_MARKER = "" VERDICTS = ("pass", "concern", "fail") REQUIREMENTS: Tuple[Tuple[str, str], ...] = ( @@ -481,8 +482,9 @@ def run(config: Config, github: GitHub, devin: Devin) -> int: def report_setup_error(env: Dict[str, str], message: str, http: Http = http_json) -> None: """Surface a configuration error on the PR when GitHub access is still available.""" + github_env = {k: v for k, v in env.items() if k not in DEVIN_NUMERIC_SETTINGS} try: - config = Config.from_env(env, require_devin=False) + config = Config.from_env(github_env, require_devin=False) github = GitHub(config, http) sha = github.pull_request()["head"]["sha"] github.set_status(sha, "error", f"Gate error: {message}", config.run_url) @@ -493,15 +495,15 @@ def report_setup_error(env: Dict[str, str], message: str, http: Http = http_json print(f"::warning::Could not report the configuration error on the PR: {err}") -def main() -> int: +def main(http: Http = http_json) -> int: env = dict(os.environ) try: config = Config.from_env(env) except (GateError, ValueError) as err: print(f"::error::{err}") - report_setup_error(env, str(err)) + report_setup_error(env, str(err), http) return 1 - return run(config, GitHub(config), Devin(config)) + return run(config, GitHub(config, http), Devin(config, http)) if __name__ == "__main__": diff --git a/.github/scripts/tests/test_project_requirements_gate.py b/.github/scripts/tests/test_project_requirements_gate.py index 573f041..55e0966 100644 --- a/.github/scripts/tests/test_project_requirements_gate.py +++ b/.github/scripts/tests/test_project_requirements_gate.py @@ -362,3 +362,32 @@ def failing_http(method, url, token, body=None): env = {"GITHUB_REPOSITORY": "NASA-AMMOS/tig", "PR_NUMBER": "7", "GITHUB_TOKEN": "gh-token"} gate.report_setup_error(env, "Missing configuration: DEVIN_ORG_ID", failing_http) + + +@pytest.mark.parametrize("overrides, removed, error", [ + ({}, "DEVIN_ORG_ID", "Missing configuration: DEVIN_ORG_ID"), + ({}, "DEVIN_API_KEY", "Missing configuration: DEVIN_API_KEY"), + ({"DEVIN_MAX_ACU": "oops"}, None, "invalid literal for int()"), +]) +def test_main_reports_configuration_errors_on_the_pr(monkeypatch, tmp_path, overrides, removed, + error): + env = {"GITHUB_REPOSITORY": "NASA-AMMOS/tig", "PR_NUMBER": "7", "GITHUB_TOKEN": "gh-token", + "GITHUB_RUN_ID": "99", "DEVIN_API_KEY": "devin-key", "DEVIN_ORG_ID": "org-1", + "CRITERIA_PATH": str(tmp_path / "criteria.md"), **overrides} + for name in ("DEVIN_API_URL", "DEVIN_MAX_ACU", "DEVIN_TIMEOUT_MINUTES", + "DEVIN_POLL_SECONDS", "GITHUB_API_URL", "GITHUB_SERVER_URL"): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + if removed: + monkeypatch.delenv(removed) + api = FakeApi() + assert gate.main(api) == 1 + status = api.statuses()[-1] + assert status["state"] == "error" + assert status["description"].startswith("Gate error: ") + assert error in status["description"] + comment = next(b for m, u, b in api.calls if m == "POST" and u.endswith("/issues/7/comments")) + assert error in comment["body"] + assert gate.SETUP_DOC in comment["body"] + assert not any("api.devin.ai" in u for m, u, b in api.calls)