Project Requirements gate: accept DEVIN_ORG_ID from a variable or secret; report setup errors on the PR

.github/scripts/project_requirements_gate.py

@@ -22,6 +22,8 @@
 
 STATUS_CONTEXT = "Project Requirements"
 DEFAULT_CRITERIA_PATH = ".github/project-requirements/criteria.md"
+SETUP_DOC = "docs/reference/project-requirements-gate.md"
+DEVIN_NUMERIC_SETTINGS = ("DEVIN_MAX_ACU", "DEVIN_TIMEOUT_MINUTES", "DEVIN_POLL_SECONDS")
 COMMENT_MARKER = "<!-- tig-project-requirements-gate -->"
 VERDICTS = ("pass", "concern", "fail")
 REQUIREMENTS: Tuple[Tuple[str, str], ...] = (
@@ -89,13 +91,11 @@ class Config:
     poll_seconds: int
 
     @classmethod
-    def from_env(cls, env: Dict[str, str]) -> "Config":
-        missing = [
-            name
-            for name in ("GITHUB_REPOSITORY", "PR_NUMBER", "GITHUB_TOKEN",
-                         "DEVIN_API_KEY", "DEVIN_ORG_ID")
-            if not env.get(name)
-        ]
+    def from_env(cls, env: Dict[str, str], require_devin: bool = True) -> "Config":
+        required = ("GITHUB_REPOSITORY", "PR_NUMBER", "GITHUB_TOKEN")
+        if require_devin:
+            required += ("DEVIN_API_KEY", "DEVIN_ORG_ID")
+        missing = [name for name in required if not env.get(name)]
         if missing:
             raise GateError("Missing configuration: " + ", ".join(missing))
         server = env.get("GITHUB_SERVER_URL", "https://github.com")
@@ -109,8 +109,8 @@ def from_env(cls, env: Dict[str, str]) -> "Config":
             github_token=env["GITHUB_TOKEN"],
             github_api_url=env.get("GITHUB_API_URL", "https://api.github.com").rstrip("/"),
             run_url=run_url,
-            devin_api_key=env["DEVIN_API_KEY"],
-            devin_org_id=env["DEVIN_ORG_ID"],
+            devin_api_key=env.get("DEVIN_API_KEY", ""),
+            devin_org_id=env.get("DEVIN_ORG_ID", ""),
             devin_api_url=(env.get("DEVIN_API_URL") or "https://api.devin.ai").rstrip("/"),
             criteria_path=Path(env.get("CRITERIA_PATH") or DEFAULT_CRITERIA_PATH),
             waiver_label=env.get("WAIVER_LABEL") or "requirements-waived",
@@ -480,13 +480,30 @@ def run(config: Config, github: GitHub, devin: Devin) -> int:
     return 0 if state == "success" else 1
 
 
-def main() -> int:
+def report_setup_error(env: Dict[str, str], message: str, http: Http = http_json) -> None:
+    """Surface a configuration error on the PR when GitHub access is still available."""
+    github_env = {k: v for k, v in env.items() if k not in DEVIN_NUMERIC_SETTINGS}
     try:
-        config = Config.from_env(dict(os.environ))
+        config = Config.from_env(github_env, require_devin=False)
+        github = GitHub(config, http)
+        sha = github.pull_request()["head"]["sha"]
+        github.set_status(sha, "error", f"Gate error: {message}", config.run_url)
+        github.upsert_comment(render_error(
+            f"{message}. Set it under Settings > Secrets and variables > Actions "
+            f"(see {SETUP_DOC}).", sha, config.run_url))
+    except (GateError, OSError, ValueError, KeyError, TypeError) as err:
+        print(f"::warning::Could not report the configuration error on the PR: {err}")
+
+
+def main(http: Http = http_json) -> int:
+    env = dict(os.environ)
+    try:
+        config = Config.from_env(env)
     except (GateError, ValueError) as err:
         print(f"::error::{err}")
+        report_setup_error(env, str(err), http)
         return 1
-    return run(config, GitHub(config), Devin(config))
+    return run(config, GitHub(config, http), Devin(config, http))
 
 
 if __name__ == "__main__":

.github/scripts/tests/test_project_requirements_gate.py

@@ -1,5 +1,6 @@
 import io
 import json
+import re
 import sys
 from pathlib import Path
 
@@ -301,3 +302,92 @@ def fake_urlopen(request, timeout):
     with pytest.raises(gate.GateError, match="returned HTTP .*: boom"):
         gate.http_json("GET", "https://x", "tok", attempts=2)
     assert len(seen) == calls
+
+
+WORKFLOW = Path(__file__).resolve().parents[2] / "workflows" / "project-requirements.yml"
+
+
+def resolve_workflow_env(name, context):
+    """Evaluate a workflow env entry of the form `${{ a.B || c.D }}` against `context`."""
+    match = re.search(rf"^\s+{name}: \$\{{\{{(.+?)\}}\}}\s*$", WORKFLOW.read_text(), re.MULTILINE)
+    assert match, f"{name} not found in {WORKFLOW.name}"
+    for ref in match.group(1).split("||"):
+        scope, key = ref.strip().split(".", 1)
+        value = context.get(scope, {}).get(key, "")
+        if value:
+            return value
+    return ""
+
+
+@pytest.mark.parametrize("context, expected", [
+    ({"vars": {"DEVIN_ORG_ID": "org-var"}}, "org-var"),
+    ({"secrets": {"DEVIN_ORG_ID": "org-secret"}}, "org-secret"),
+    ({"vars": {"DEVIN_ORG_ID": "org-var"}, "secrets": {"DEVIN_ORG_ID": "org-secret"}}, "org-var"),
+])
+def test_workflow_org_id_accepts_variable_or_secret(tmp_path, context, expected):
+    org_id = resolve_workflow_env("DEVIN_ORG_ID", context)
+    assert make_config(tmp_path, DEVIN_ORG_ID=org_id).devin_org_id == expected
+
+
+def test_workflow_org_id_missing_from_both_is_a_config_error(tmp_path):
+    org_id = resolve_workflow_env("DEVIN_ORG_ID", {})
+    with pytest.raises(gate.GateError, match="Missing configuration: DEVIN_ORG_ID"):
+        make_config(tmp_path, DEVIN_ORG_ID=org_id)
+
+
+def test_report_setup_error_comments_and_sets_error_status(tmp_path):
+    env = {"GITHUB_REPOSITORY": "NASA-AMMOS/tig", "PR_NUMBER": "7", "GITHUB_TOKEN": "gh-token",
+           "GITHUB_RUN_ID": "99", "DEVIN_API_KEY": "devin-key"}
+    api = FakeApi()
+    gate.report_setup_error(env, "Missing configuration: DEVIN_ORG_ID", api)
+    status = api.statuses()[-1]
+    assert status["state"] == "error"
+    assert status["description"] == "Gate error: Missing configuration: DEVIN_ORG_ID"
+    comment = next(b for m, u, b in api.calls if m == "POST" and u.endswith("/issues/7/comments"))
+    assert "Project Requirements: ERROR" in comment["body"]
+    assert "Missing configuration: DEVIN_ORG_ID" in comment["body"]
+    assert gate.SETUP_DOC in comment["body"]
+
+
+def test_report_setup_error_without_github_access_only_warns(capsys):
+    api = FakeApi()
+    gate.report_setup_error({"PR_NUMBER": "7"}, "Missing configuration: GITHUB_TOKEN", api)
+    assert api.calls == []
+    assert "Could not report the configuration error" in capsys.readouterr().out
+
+
+def test_report_setup_error_survives_github_failure():
+    def failing_http(method, url, token, body=None):
+        raise gate.GateError("GET pulls returned HTTP 403")
+
+    env = {"GITHUB_REPOSITORY": "NASA-AMMOS/tig", "PR_NUMBER": "7", "GITHUB_TOKEN": "gh-token"}
+    gate.report_setup_error(env, "Missing configuration: DEVIN_ORG_ID", failing_http)
+
+
+@pytest.mark.parametrize("overrides, removed, error", [
+    ({}, "DEVIN_ORG_ID", "Missing configuration: DEVIN_ORG_ID"),
+    ({}, "DEVIN_API_KEY", "Missing configuration: DEVIN_API_KEY"),
+    ({"DEVIN_MAX_ACU": "oops"}, None, "invalid literal for int()"),
+])
+def test_main_reports_configuration_errors_on_the_pr(monkeypatch, tmp_path, overrides, removed,
+                                                      error):
+    env = {"GITHUB_REPOSITORY": "NASA-AMMOS/tig", "PR_NUMBER": "7", "GITHUB_TOKEN": "gh-token",
+           "GITHUB_RUN_ID": "99", "DEVIN_API_KEY": "devin-key", "DEVIN_ORG_ID": "org-1",
+           "CRITERIA_PATH": str(tmp_path / "criteria.md"), **overrides}
+    for name in ("DEVIN_API_URL", "DEVIN_MAX_ACU", "DEVIN_TIMEOUT_MINUTES",
+                 "DEVIN_POLL_SECONDS", "GITHUB_API_URL", "GITHUB_SERVER_URL"):
+        monkeypatch.delenv(name, raising=False)
+    for name, value in env.items():
+        monkeypatch.setenv(name, value)
+    if removed:
+        monkeypatch.delenv(removed)
+    api = FakeApi()
+    assert gate.main(api) == 1
+    status = api.statuses()[-1]
+    assert status["state"] == "error"
+    assert status["description"].startswith("Gate error: ")
+    assert error in status["description"]
+    comment = next(b for m, u, b in api.calls if m == "POST" and u.endswith("/issues/7/comments"))
+    assert error in comment["body"]
+    assert gate.SETUP_DOC in comment["body"]
+    assert not any("api.devin.ai" in u for m, u, b in api.calls)

.github/workflows/project-requirements-tests.yml

@@ -9,11 +9,13 @@ on:
     paths:
       - ".github/scripts/**"
       - ".github/workflows/project-requirements-tests.yml"
+      - ".github/workflows/project-requirements.yml"
   pull_request:
     branches: ["**"]
     paths:
       - ".github/scripts/**"
       - ".github/workflows/project-requirements-tests.yml"
+      - ".github/workflows/project-requirements.yml"
 
 permissions:
   contents: read

.github/workflows/project-requirements.yml

@@ -55,7 +55,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_number }}
           DEVIN_API_KEY: ${{ secrets.DEVIN_API_KEY }}
-          DEVIN_ORG_ID: ${{ vars.DEVIN_ORG_ID }}
+          DEVIN_ORG_ID: ${{ vars.DEVIN_ORG_ID || secrets.DEVIN_ORG_ID }}
           DEVIN_API_URL: ${{ vars.DEVIN_API_URL }}
           DEVIN_MAX_ACU: ${{ vars.PROJECT_REQUIREMENTS_MAX_ACU }}
           WAIVER_LABEL: ${{ vars.PROJECT_REQUIREMENTS_WAIVER_LABEL }}

docs/reference/project-requirements-gate.md

@@ -34,6 +34,10 @@ not apply them. They are used only by this gate.
    | No verdict (API error, timeout, malformed output) | error |
    | Waived by a maintainer | success |
 
+   The comment always states the reason. When the gate cannot evaluate the
+   PR at all, for example because `DEVIN_ORG_ID` or `DEVIN_API_KEY` is not
+   set, the comment shows the error and how to fix it.
+
 A run takes as long as the Devin session, usually several minutes. Pushing
 again cancels the previous run.
 
@@ -51,7 +55,7 @@ Repository settings (**Settings > Secrets and variables > Actions**):
 | Name | Kind | Required | Purpose |
 | --- | --- | --- | --- |
 | `DEVIN_API_KEY` | Secret | Yes | Devin service user API key with permission to create sessions. |
-| `DEVIN_ORG_ID` | Variable | Yes | Devin organization ID (`org-...`) that sessions run in. |
+| `DEVIN_ORG_ID` | Variable (or secret) | Yes | Devin organization ID (`org-...`) that sessions run in. Read from the variable first, then the secret. |
 | `DEVIN_API_URL` | Variable | No | Devin API base URL. Defaults to `https://api.devin.ai`; set it for dedicated deployments. |
 | `PROJECT_REQUIREMENTS_MAX_ACU` | Variable | No | ACU limit per session. |
 | `PROJECT_REQUIREMENTS_WAIVER_LABEL` | Variable | No | Waiver label name. Defaults to `requirements-waived`. |