ci(pitot): surface per-agent E2E status

.github/scripts/pitot_e2e_report.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""Publish a trusted, sticky PR summary for Pitot's per-agent E2E workflows."""
+
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import os
+from pathlib import Path
+import re
+import urllib.parse
+import urllib.request
+import zipfile
+
+
+MARKER = "<!-- pitot-e2e-report -->"
+WORKFLOWS = {
+    "claude": "Pitot E2E — Claude",
+    "cursor": "Pitot E2E — Cursor",
+    "codex": "Pitot E2E — Codex",
+    "gemini": "Pitot E2E — Gemini",
+    "opencode": "Pitot E2E — OpenCode",
+}
+PLATFORMS = ("ubuntu", "macos")
+MAX_ARTIFACT_BYTES = 1_000_000
+
+
+def validate_result(
+    value: object,
+    *,
+    agent: str,
+    platform: str,
+    expected_sha: str | None = None,
+    expected_run_url: str | None = None,
+) -> dict[str, object]:
+    if not isinstance(value, dict):
+        raise ValueError("result must be an object")
+    required = {
+        "schema_version",
+        "agent",
+        "platform",
+        "status",
+        "verification_mode",
+        "evidence",
+        "commit_sha",
+        "run_url",
+    }
+    if set(value) != required:
+        raise ValueError("result fields do not match schema")
+    if value["schema_version"] != 1 or value["agent"] != agent or value["platform"] != platform:
+        raise ValueError("result identity does not match artifact")
+    if value["status"] not in {"pass", "fail"}:
+        raise ValueError("invalid result status")
+    mode = value["verification_mode"]
+    if value["status"] == "pass" and mode not in {"real_cli", "hook_subprocess"}:
+        raise ValueError("passing result requires a verification mode")
+    if value["status"] == "fail" and mode is not None:
+        raise ValueError("failed result cannot claim a verification mode")
+    allowed_evidence = {
+        "real host CLI completed the Pitot integration path",
+        "active hook subprocess produced a normalized Pitot action",
+        "E2E command failed",
+        "E2E command returned without one valid result marker",
+    }
+    if value["evidence"] not in allowed_evidence:
+        raise ValueError("invalid evidence summary")
+    if not isinstance(value["commit_sha"], str) or not re.fullmatch(r"[0-9a-f]{40}", value["commit_sha"]):
+        raise ValueError("invalid commit_sha")
+    if not isinstance(value["run_url"], str) or not re.fullmatch(
+        r"https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/[0-9]+",
+        value["run_url"],
+    ):
+        raise ValueError("invalid run_url")
+    if expected_sha is not None and value["commit_sha"] != expected_sha:
+        raise ValueError("result commit does not match workflow run")
+    if expected_run_url is not None and value["run_url"] != expected_run_url:
+        raise ValueError("result URL does not match workflow run")
+    return value
+
+
+def aggregate(platform_results: dict[str, dict[str, object] | None]) -> str:
+    if any(platform not in platform_results or platform_results[platform] is None for platform in PLATFORMS):
+        return "pending"
+    return "passing" if all(platform_results[p]["status"] == "pass" for p in PLATFORMS) else "failing"
+
+
+def result_cell(result: dict[str, object] | None) -> str:
+    if result is None:
+        return "⏳ Pending"
+    if result["status"] == "fail":
+        return "❌ Failed"
+    mode = "real CLI" if result["verification_mode"] == "real_cli" else "hook verified"
+    return f"✅ Pass · {mode}"
+
+
+def failed_result(agent: str, platform: str, head_sha: str, run_url: str) -> dict[str, object]:
+    return {
+        "schema_version": 1,
+        "agent": agent,
+        "platform": platform,
+        "status": "fail",
+        "verification_mode": None,
+        "evidence": "E2E command failed",
+        "commit_sha": head_sha,
+        "run_url": run_url,
+    }
+
+
+def render_comment(
+    results: dict[str, dict[str, dict[str, object] | None]],
+    run_urls: dict[str, str],
+    head_sha: str,
+) -> str:
+    lines = [
+        MARKER,
+        "## Pitot coding-agent E2E",
+        "",
+        "Intelligence Flow is the verification source; Pitot's public README carries the latest `main` status.",
+        "",
+        "| Agent | Ubuntu | macOS | Result | Evidence |",
+        "|---|---|---|---|---|",
+    ]
+    icons = {"passing": "✅ Passing", "failing": "❌ Failing", "pending": "⏳ Pending"}
+    for agent in WORKFLOWS:
+        platforms = results.get(agent, {})
+        status = aggregate(platforms)
+        url = run_urls.get(agent)
+        label = agent.capitalize() if agent != "opencode" else "OpenCode"
+        linked_label = f"[{label}]({url})" if url else label
+        evidence = "Both platforms are required" if status != "pending" else "Waiting for both platform artifacts"
+        lines.append(
+            f"| {linked_label} | {result_cell(platforms.get('ubuntu'))} | "
+            f"{result_cell(platforms.get('macos'))} | {icons[status]} | {evidence} |"
+        )
+    lines.extend(["", f"Source commit: `{head_sha[:12]}`", ""])
+    return "\n".join(lines)
+
+
+class GitHub:
+    def __init__(self, repository: str, token: str) -> None:
+        self.repository = repository
+        self.token = token
+        self.base = f"https://api.github.com/repos/{repository}"
+
+    def request(self, method: str, path: str, payload: object | None = None) -> object:
+        data = json.dumps(payload).encode() if payload is not None else None
+        request = urllib.request.Request(
+            self.base + path,
+            data=data,
+            method=method,
+            headers={
+                "Accept": "application/vnd.github+json",
+                "Authorization": f"Bearer {self.token}",
+                "Content-Type": "application/json",
+                "X-GitHub-Api-Version": "2022-11-28",
+                "User-Agent": "pitot-e2e-reporter",
+            },
+        )
+        with urllib.request.urlopen(request, timeout=30) as response:
+            body = response.read(MAX_ARTIFACT_BYTES)
+        return json.loads(body) if body else {}
+
+    def download(self, url: str) -> bytes:
+        request = urllib.request.Request(
+            url,
+            headers={"Authorization": f"Bearer {self.token}", "User-Agent": "pitot-e2e-reporter"},
+        )
+        with urllib.request.urlopen(request, timeout=30) as response:
+            data = response.read(MAX_ARTIFACT_BYTES + 1)
+        if len(data) > MAX_ARTIFACT_BYTES:
+            raise ValueError("artifact exceeds size limit")
+        return data
+
+
+def load_artifact(
+    github: GitHub,
+    artifact: dict[str, object],
+    *,
+    agent: str,
+    platform: str,
+    expected_sha: str,
+    expected_run_url: str,
+) -> dict[str, object]:
+    archive = github.download(str(artifact["archive_download_url"]))
+    with zipfile.ZipFile(io.BytesIO(archive)) as bundle:
+        names = bundle.namelist()
+        if names != ["result.json"] or bundle.getinfo("result.json").file_size > MAX_ARTIFACT_BYTES:
+            raise ValueError("artifact must contain one bounded result.json")
+        value = json.loads(bundle.read("result.json"))
+    return validate_result(
+        value,
+        agent=agent,
+        platform=platform,
+        expected_sha=expected_sha,
+        expected_run_url=expected_run_url,
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--event", type=Path, required=True)
+    parser.add_argument("--repository", required=True)
+    args = parser.parse_args()
+    event = json.loads(args.event.read_text(encoding="utf-8"))
+    workflow_run = event["workflow_run"]
+    token = os.environ.get("GITHUB_TOKEN")
+    if not token:
+        raise SystemExit("GITHUB_TOKEN is required")
+    github = GitHub(args.repository, token)
+    pull_requests = workflow_run.get("pull_requests", [])
+    if not pull_requests:
+        owner = workflow_run.get("head_repository", {}).get("owner", {}).get("login")
+        branch = workflow_run.get("head_branch")
+        if owner and branch:
+            head = urllib.parse.quote(f"{owner}:{branch}", safe=":")
+            candidates = github.request("GET", f"/pulls?state=open&head={head}&per_page=20")
+            pull_requests = [pr for pr in candidates if pr.get("head", {}).get("sha") == workflow_run["head_sha"]]
+        if not pull_requests:
+            print("No pull request is associated with this workflow run; nothing to report.")
+            return 0
+    head_sha = workflow_run["head_sha"]
+    query = urllib.parse.urlencode({"head_sha": head_sha, "event": "pull_request", "per_page": 100})
+    runs = github.request("GET", f"/actions/runs?{query}")["workflow_runs"]
+
+    results: dict[str, dict[str, dict[str, object] | None]] = {
+        agent: {platform: None for platform in PLATFORMS} for agent in WORKFLOWS
+    }
+    run_urls: dict[str, str] = {}
+    for agent, workflow_name in WORKFLOWS.items():
+        candidates = [run for run in runs if run.get("name") == workflow_name]
+        if not candidates:
+            continue
+        run = max(candidates, key=lambda value: value["id"])
+        run_urls[agent] = run["html_url"]
+        if run["status"] != "completed":
+            continue
+        artifacts = github.request("GET", f"/actions/runs/{run['id']}/artifacts")["artifacts"]
+        by_name = {artifact["name"]: artifact for artifact in artifacts if not artifact.get("expired")}
+        for platform in PLATFORMS:
+            artifact = by_name.get(f"pitot-e2e-{agent}-{platform}")
+            if artifact is not None:
+                try:
+                    results[agent][platform] = load_artifact(
+                        github,
+                        artifact,
+                        agent=agent,
+                        platform=platform,
+                        expected_sha=run["head_sha"],
+                        expected_run_url=run["html_url"],
+                    )
+                except (ValueError, json.JSONDecodeError, zipfile.BadZipFile):
+                    results[agent][platform] = failed_result(agent, platform, head_sha, run["html_url"])
+            elif run.get("conclusion") != "success":
+                results[agent][platform] = failed_result(agent, platform, head_sha, run["html_url"])
+
+    body = render_comment(results, run_urls, head_sha)
+    issue_number = pull_requests[0]["number"]
+    comments = github.request("GET", f"/issues/{issue_number}/comments?per_page=100")
+    existing = next(
+        (
+            comment
+            for comment in comments
+            if MARKER in comment.get("body", "")
+            and comment.get("user", {}).get("login") == "github-actions[bot]"
+        ),
+        None,
+    )
+    if existing:
+        github.request("PATCH", f"/issues/comments/{existing['id']}", {"body": body})
+        print(f"Updated Pitot E2E report on PR #{issue_number}")
+    else:
+        github.request("POST", f"/issues/{issue_number}/comments", {"body": body})
+        print(f"Created Pitot E2E report on PR #{issue_number}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/pitot-e2e-agent.yml

@@ -0,0 +1,79 @@
+name: Pitot E2E reusable agent check
+
+on:
+  workflow_call:
+    inputs:
+      agent:
+        description: Supported Pitot coding-agent host
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  CLAUDE_CLI_VERSION: 2.1.217
+
+jobs:
+  e2e:
+    name: ${{ inputs.agent }} (${{ matrix.platform }})
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            platform: ubuntu
+          - runner: macos-latest
+            platform: macos
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: labs/15-pitot/pitot/go.mod
+          cache-dependency-path: labs/15-pitot/pitot/go.mod
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+      - name: Install Claude CLI
+        if: inputs.agent == 'claude'
+        shell: bash
+        run: |
+          npm install -g "@anthropic-ai/claude-code@$CLAUDE_CLI_VERSION"
+          claude_bin="$(npm prefix -g)/bin/claude"
+          installed_version="$("$claude_bin" --version)"
+          echo "Using $claude_bin ($installed_version)"
+          [[ "$installed_version" == "$CLAUDE_CLI_VERSION"* ]]
+          echo "CLAUDE_PATH=$claude_bin" >> "$GITHUB_ENV"
+      - name: Install Codex CLI
+        if: inputs.agent == 'codex'
+        run: npm install -g @openai/codex
+      - name: Install Cursor CLI
+        if: inputs.agent == 'cursor'
+        run: curl https://cursor.com/install -fsS | bash
+      - name: Run ${{ inputs.agent }} E2E
+        id: e2e
+        continue-on-error: true
+        timeout-minutes: 10
+        shell: bash
+        env:
+          PITOT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: >-
+          python3 labs/15-pitot/tests/run_e2e_report.py
+          --agent "${{ inputs.agent }}"
+          --platform "${{ matrix.platform }}"
+          --output "${{ runner.temp }}/pitot-e2e/result.json"
+          -- bash "labs/15-pitot/tests/e2e_${{ inputs.agent }}_cli_test.sh"
+          < /dev/null
+      - name: Upload structured E2E result
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pitot-e2e-${{ inputs.agent }}-${{ matrix.platform }}
+          path: ${{ runner.temp }}/pitot-e2e/result.json
+          if-no-files-found: error
+          retention-days: 14
+      - name: Enforce E2E result
+        if: always() && steps.e2e.outcome != 'success'
+        shell: bash
+        run: exit 1

.github/workflows/pitot-e2e-claude.yml

@@ -0,0 +1,19 @@
+name: Pitot E2E — Claude
+
+on:
+  pull_request:
+    paths: &pitot_paths
+      - "labs/15-pitot/**"
+      - "go.work"
+      - ".github/workflows/pitot-e2e-*.yml"
+      - ".github/scripts/pitot_e2e_report.py"
+  push:
+    branches: [main]
+    paths: *pitot_paths
+  workflow_dispatch:
+
+jobs:
+  verify:
+    uses: ./.github/workflows/pitot-e2e-agent.yml
+    with:
+      agent: claude

.github/workflows/pitot-e2e-codex.yml

@@ -0,0 +1,19 @@
+name: Pitot E2E — Codex
+
+on:
+  pull_request:
+    paths: &pitot_paths
+      - "labs/15-pitot/**"
+      - "go.work"
+      - ".github/workflows/pitot-e2e-*.yml"
+      - ".github/scripts/pitot_e2e_report.py"
+  push:
+    branches: [main]
+    paths: *pitot_paths
+  workflow_dispatch:
+
+jobs:
+  verify:
+    uses: ./.github/workflows/pitot-e2e-agent.yml
+    with:
+      agent: codex