From ef14e6a09c157044b4bbdaddb1451afb81effb2b Mon Sep 17 00:00:00 2001 From: bigboateng Date: Wed, 22 Jul 2026 13:19:03 +0100 Subject: [PATCH 1/3] ci(pitot): surface per-agent e2e status --- .github/scripts/pitot_e2e_report.py | 279 ++++++++++++++++++ .github/workflows/pitot-e2e-agent.yml | 69 +++++ .github/workflows/pitot-e2e-claude.yml | 19 ++ .github/workflows/pitot-e2e-codex.yml | 19 ++ .github/workflows/pitot-e2e-cursor.yml | 19 ++ .github/workflows/pitot-e2e-gemini.yml | 19 ++ .github/workflows/pitot-e2e-opencode.yml | 19 ++ .github/workflows/pitot-e2e-report.yml | 34 +++ .github/workflows/pitot-lab.yml | 15 - .../15-pitot/pitot-distribution/UPSTREAM.json | 2 +- .../2026-07-22-agent-e2e-status.md | 3 + labs/15-pitot/public-readme-preview/README.md | 12 +- labs/15-pitot/tests/e2e_unified_runner.sh | 11 +- labs/15-pitot/tests/run_e2e_report.py | 78 +++++ labs/15-pitot/tests/test_e2e_reporting.py | 129 ++++++++ 15 files changed, 708 insertions(+), 19 deletions(-) create mode 100644 .github/scripts/pitot_e2e_report.py create mode 100644 .github/workflows/pitot-e2e-agent.yml create mode 100644 .github/workflows/pitot-e2e-claude.yml create mode 100644 .github/workflows/pitot-e2e-codex.yml create mode 100644 .github/workflows/pitot-e2e-cursor.yml create mode 100644 .github/workflows/pitot-e2e-gemini.yml create mode 100644 .github/workflows/pitot-e2e-opencode.yml create mode 100644 .github/workflows/pitot-e2e-report.yml create mode 100644 labs/15-pitot/pitot-distribution/release-notes/2026-07-22-agent-e2e-status.md create mode 100644 labs/15-pitot/tests/run_e2e_report.py create mode 100644 labs/15-pitot/tests/test_e2e_reporting.py diff --git a/.github/scripts/pitot_e2e_report.py b/.github/scripts/pitot_e2e_report.py new file mode 100644 index 000000000..a9a2dc9b7 --- /dev/null +++ b/.github/scripts/pitot_e2e_report.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +"""Publish a trusted, sticky PR summary for Pitot's per-agent E2E workflows.""" + +from __future__ import annotations + +import argparse +import io +import json +import os +from pathlib import Path +import re +import urllib.parse +import urllib.request +import zipfile + + +MARKER = "" +WORKFLOWS = { + "claude": "Pitot E2E — Claude", + "cursor": "Pitot E2E — Cursor", + "codex": "Pitot E2E — Codex", + "gemini": "Pitot E2E — Gemini", + "opencode": "Pitot E2E — OpenCode", +} +PLATFORMS = ("ubuntu", "macos") +MAX_ARTIFACT_BYTES = 1_000_000 + + +def validate_result( + value: object, + *, + agent: str, + platform: str, + expected_sha: str | None = None, + expected_run_url: str | None = None, +) -> dict[str, object]: + if not isinstance(value, dict): + raise ValueError("result must be an object") + required = { + "schema_version", + "agent", + "platform", + "status", + "verification_mode", + "evidence", + "commit_sha", + "run_url", + } + if set(value) != required: + raise ValueError("result fields do not match schema") + if value["schema_version"] != 1 or value["agent"] != agent or value["platform"] != platform: + raise ValueError("result identity does not match artifact") + if value["status"] not in {"pass", "fail"}: + raise ValueError("invalid result status") + mode = value["verification_mode"] + if value["status"] == "pass" and mode not in {"real_cli", "hook_subprocess"}: + raise ValueError("passing result requires a verification mode") + if value["status"] == "fail" and mode is not None: + raise ValueError("failed result cannot claim a verification mode") + allowed_evidence = { + "real host CLI completed the Pitot integration path", + "active hook subprocess produced a normalized Pitot action", + "E2E command failed", + "E2E command returned without one valid result marker", + } + if value["evidence"] not in allowed_evidence: + raise ValueError("invalid evidence summary") + if not isinstance(value["commit_sha"], str) or not re.fullmatch(r"[0-9a-f]{40}", value["commit_sha"]): + raise ValueError("invalid commit_sha") + if not isinstance(value["run_url"], str) or not re.fullmatch( + r"https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/[0-9]+", + value["run_url"], + ): + raise ValueError("invalid run_url") + if expected_sha is not None and value["commit_sha"] != expected_sha: + raise ValueError("result commit does not match workflow run") + if expected_run_url is not None and value["run_url"] != expected_run_url: + raise ValueError("result URL does not match workflow run") + return value + + +def aggregate(platform_results: dict[str, dict[str, object] | None]) -> str: + if any(platform not in platform_results or platform_results[platform] is None for platform in PLATFORMS): + return "pending" + return "passing" if all(platform_results[p]["status"] == "pass" for p in PLATFORMS) else "failing" + + +def result_cell(result: dict[str, object] | None) -> str: + if result is None: + return "⏳ Pending" + if result["status"] == "fail": + return "❌ Failed" + mode = "real CLI" if result["verification_mode"] == "real_cli" else "hook verified" + return f"✅ Pass · {mode}" + + +def failed_result(agent: str, platform: str, head_sha: str, run_url: str) -> dict[str, object]: + return { + "schema_version": 1, + "agent": agent, + "platform": platform, + "status": "fail", + "verification_mode": None, + "evidence": "E2E command failed", + "commit_sha": head_sha, + "run_url": run_url, + } + + +def render_comment( + results: dict[str, dict[str, dict[str, object] | None]], + run_urls: dict[str, str], + head_sha: str, +) -> str: + lines = [ + MARKER, + "## Pitot coding-agent E2E", + "", + "Intelligence Flow is the verification source; Pitot's public README carries the latest `main` status.", + "", + "| Agent | Ubuntu | macOS | Result | Evidence |", + "|---|---|---|---|---|", + ] + icons = {"passing": "✅ Passing", "failing": "❌ Failing", "pending": "⏳ Pending"} + for agent in WORKFLOWS: + platforms = results.get(agent, {}) + status = aggregate(platforms) + url = run_urls.get(agent) + label = agent.capitalize() if agent != "opencode" else "OpenCode" + linked_label = f"[{label}]({url})" if url else label + evidence = "Both platforms are required" if status != "pending" else "Waiting for both platform artifacts" + lines.append( + f"| {linked_label} | {result_cell(platforms.get('ubuntu'))} | " + f"{result_cell(platforms.get('macos'))} | {icons[status]} | {evidence} |" + ) + lines.extend(["", f"Source commit: `{head_sha[:12]}`", ""]) + return "\n".join(lines) + + +class GitHub: + def __init__(self, repository: str, token: str) -> None: + self.repository = repository + self.token = token + self.base = f"https://api.github.com/repos/{repository}" + + def request(self, method: str, path: str, payload: object | None = None) -> object: + data = json.dumps(payload).encode() if payload is not None else None + request = urllib.request.Request( + self.base + path, + data=data, + method=method, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "pitot-e2e-reporter", + }, + ) + with urllib.request.urlopen(request, timeout=30) as response: + body = response.read(MAX_ARTIFACT_BYTES) + return json.loads(body) if body else {} + + def download(self, url: str) -> bytes: + request = urllib.request.Request( + url, + headers={"Authorization": f"Bearer {self.token}", "User-Agent": "pitot-e2e-reporter"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + data = response.read(MAX_ARTIFACT_BYTES + 1) + if len(data) > MAX_ARTIFACT_BYTES: + raise ValueError("artifact exceeds size limit") + return data + + +def load_artifact( + github: GitHub, + artifact: dict[str, object], + *, + agent: str, + platform: str, + expected_sha: str, + expected_run_url: str, +) -> dict[str, object]: + archive = github.download(str(artifact["archive_download_url"])) + with zipfile.ZipFile(io.BytesIO(archive)) as bundle: + names = bundle.namelist() + if names != ["result.json"] or bundle.getinfo("result.json").file_size > MAX_ARTIFACT_BYTES: + raise ValueError("artifact must contain one bounded result.json") + value = json.loads(bundle.read("result.json")) + return validate_result( + value, + agent=agent, + platform=platform, + expected_sha=expected_sha, + expected_run_url=expected_run_url, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--event", type=Path, required=True) + parser.add_argument("--repository", required=True) + args = parser.parse_args() + event = json.loads(args.event.read_text(encoding="utf-8")) + workflow_run = event["workflow_run"] + token = os.environ.get("GITHUB_TOKEN") + if not token: + raise SystemExit("GITHUB_TOKEN is required") + github = GitHub(args.repository, token) + pull_requests = workflow_run.get("pull_requests", []) + if not pull_requests: + owner = workflow_run.get("head_repository", {}).get("owner", {}).get("login") + branch = workflow_run.get("head_branch") + if owner and branch: + head = urllib.parse.quote(f"{owner}:{branch}", safe=":") + candidates = github.request("GET", f"/pulls?state=open&head={head}&per_page=20") + pull_requests = [pr for pr in candidates if pr.get("head", {}).get("sha") == workflow_run["head_sha"]] + if not pull_requests: + print("No pull request is associated with this workflow run; nothing to report.") + return 0 + head_sha = workflow_run["head_sha"] + query = urllib.parse.urlencode({"head_sha": head_sha, "event": "pull_request", "per_page": 100}) + runs = github.request("GET", f"/actions/runs?{query}")["workflow_runs"] + + results: dict[str, dict[str, dict[str, object] | None]] = { + agent: {platform: None for platform in PLATFORMS} for agent in WORKFLOWS + } + run_urls: dict[str, str] = {} + for agent, workflow_name in WORKFLOWS.items(): + candidates = [run for run in runs if run.get("name") == workflow_name] + if not candidates: + continue + run = max(candidates, key=lambda value: value["id"]) + run_urls[agent] = run["html_url"] + if run["status"] != "completed": + continue + artifacts = github.request("GET", f"/actions/runs/{run['id']}/artifacts")["artifacts"] + by_name = {artifact["name"]: artifact for artifact in artifacts if not artifact.get("expired")} + for platform in PLATFORMS: + artifact = by_name.get(f"pitot-e2e-{agent}-{platform}") + if artifact is not None: + try: + results[agent][platform] = load_artifact( + github, + artifact, + agent=agent, + platform=platform, + expected_sha=run["head_sha"], + expected_run_url=run["html_url"], + ) + except (ValueError, json.JSONDecodeError, zipfile.BadZipFile): + results[agent][platform] = failed_result(agent, platform, head_sha, run["html_url"]) + elif run.get("conclusion") != "success": + results[agent][platform] = failed_result(agent, platform, head_sha, run["html_url"]) + + body = render_comment(results, run_urls, head_sha) + issue_number = pull_requests[0]["number"] + comments = github.request("GET", f"/issues/{issue_number}/comments?per_page=100") + existing = next( + ( + comment + for comment in comments + if MARKER in comment.get("body", "") + and comment.get("user", {}).get("login") == "github-actions[bot]" + ), + None, + ) + if existing: + github.request("PATCH", f"/issues/comments/{existing['id']}", {"body": body}) + print(f"Updated Pitot E2E report on PR #{issue_number}") + else: + github.request("POST", f"/issues/{issue_number}/comments", {"body": body}) + print(f"Created Pitot E2E report on PR #{issue_number}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/pitot-e2e-agent.yml b/.github/workflows/pitot-e2e-agent.yml new file mode 100644 index 000000000..066b100a0 --- /dev/null +++ b/.github/workflows/pitot-e2e-agent.yml @@ -0,0 +1,69 @@ +name: Pitot E2E reusable agent check + +on: + workflow_call: + inputs: + agent: + description: Supported Pitot coding-agent host + required: true + type: string + +permissions: + contents: read + +jobs: + e2e: + name: ${{ inputs.agent }} (${{ matrix.platform }}) + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-latest + platform: ubuntu + - runner: macos-latest + platform: macos + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: labs/15-pitot/pitot/go.mod + cache-dependency-path: labs/15-pitot/pitot/go.mod + - uses: actions/setup-node@v4 + with: + node-version: "20" + - name: Install Claude CLI + if: inputs.agent == 'claude' + run: npm install -g @anthropic-ai/claude-code + - name: Install Codex CLI + if: inputs.agent == 'codex' + run: npm install -g @openai/codex + - name: Install Cursor CLI + if: inputs.agent == 'cursor' + run: curl https://cursor.com/install -fsS | bash + - name: Run ${{ inputs.agent }} E2E + id: e2e + continue-on-error: true + timeout-minutes: 10 + shell: bash + env: + PITOT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python3 labs/15-pitot/tests/run_e2e_report.py + --agent "${{ inputs.agent }}" + --platform "${{ matrix.platform }}" + --output "${{ runner.temp }}/pitot-e2e/result.json" + -- bash "labs/15-pitot/tests/e2e_${{ inputs.agent }}_cli_test.sh" + < /dev/null + - name: Upload structured E2E result + if: always() + uses: actions/upload-artifact@v4 + with: + name: pitot-e2e-${{ inputs.agent }}-${{ matrix.platform }} + path: ${{ runner.temp }}/pitot-e2e/result.json + if-no-files-found: error + retention-days: 14 + - name: Enforce E2E result + if: always() && steps.e2e.outcome != 'success' + shell: bash + run: exit 1 diff --git a/.github/workflows/pitot-e2e-claude.yml b/.github/workflows/pitot-e2e-claude.yml new file mode 100644 index 000000000..bb8f9989b --- /dev/null +++ b/.github/workflows/pitot-e2e-claude.yml @@ -0,0 +1,19 @@ +name: Pitot E2E — Claude + +on: + pull_request: + paths: &pitot_paths + - "labs/15-pitot/**" + - "go.work" + - ".github/workflows/pitot-e2e-*.yml" + - ".github/scripts/pitot_e2e_report.py" + push: + branches: [main] + paths: *pitot_paths + workflow_dispatch: + +jobs: + verify: + uses: ./.github/workflows/pitot-e2e-agent.yml + with: + agent: claude diff --git a/.github/workflows/pitot-e2e-codex.yml b/.github/workflows/pitot-e2e-codex.yml new file mode 100644 index 000000000..c38752f50 --- /dev/null +++ b/.github/workflows/pitot-e2e-codex.yml @@ -0,0 +1,19 @@ +name: Pitot E2E — Codex + +on: + pull_request: + paths: &pitot_paths + - "labs/15-pitot/**" + - "go.work" + - ".github/workflows/pitot-e2e-*.yml" + - ".github/scripts/pitot_e2e_report.py" + push: + branches: [main] + paths: *pitot_paths + workflow_dispatch: + +jobs: + verify: + uses: ./.github/workflows/pitot-e2e-agent.yml + with: + agent: codex diff --git a/.github/workflows/pitot-e2e-cursor.yml b/.github/workflows/pitot-e2e-cursor.yml new file mode 100644 index 000000000..b5db90d1a --- /dev/null +++ b/.github/workflows/pitot-e2e-cursor.yml @@ -0,0 +1,19 @@ +name: Pitot E2E — Cursor + +on: + pull_request: + paths: &pitot_paths + - "labs/15-pitot/**" + - "go.work" + - ".github/workflows/pitot-e2e-*.yml" + - ".github/scripts/pitot_e2e_report.py" + push: + branches: [main] + paths: *pitot_paths + workflow_dispatch: + +jobs: + verify: + uses: ./.github/workflows/pitot-e2e-agent.yml + with: + agent: cursor diff --git a/.github/workflows/pitot-e2e-gemini.yml b/.github/workflows/pitot-e2e-gemini.yml new file mode 100644 index 000000000..4afd38039 --- /dev/null +++ b/.github/workflows/pitot-e2e-gemini.yml @@ -0,0 +1,19 @@ +name: Pitot E2E — Gemini + +on: + pull_request: + paths: &pitot_paths + - "labs/15-pitot/**" + - "go.work" + - ".github/workflows/pitot-e2e-*.yml" + - ".github/scripts/pitot_e2e_report.py" + push: + branches: [main] + paths: *pitot_paths + workflow_dispatch: + +jobs: + verify: + uses: ./.github/workflows/pitot-e2e-agent.yml + with: + agent: gemini diff --git a/.github/workflows/pitot-e2e-opencode.yml b/.github/workflows/pitot-e2e-opencode.yml new file mode 100644 index 000000000..cf9f7ea96 --- /dev/null +++ b/.github/workflows/pitot-e2e-opencode.yml @@ -0,0 +1,19 @@ +name: Pitot E2E — OpenCode + +on: + pull_request: + paths: &pitot_paths + - "labs/15-pitot/**" + - "go.work" + - ".github/workflows/pitot-e2e-*.yml" + - ".github/scripts/pitot_e2e_report.py" + push: + branches: [main] + paths: *pitot_paths + workflow_dispatch: + +jobs: + verify: + uses: ./.github/workflows/pitot-e2e-agent.yml + with: + agent: opencode diff --git a/.github/workflows/pitot-e2e-report.yml b/.github/workflows/pitot-e2e-report.yml new file mode 100644 index 000000000..371269581 --- /dev/null +++ b/.github/workflows/pitot-e2e-report.yml @@ -0,0 +1,34 @@ +name: Report Pitot E2E results + +on: + workflow_run: + workflows: + - Pitot E2E — Claude + - Pitot E2E — Cursor + - Pitot E2E — Codex + - Pitot E2E — Gemini + - Pitot E2E — OpenCode + types: [completed] + +permissions: + actions: read + contents: read + pull-requests: write + +concurrency: + group: pitot-e2e-report-${{ github.event.workflow_run.head_sha }} + cancel-in-progress: true + +jobs: + report: + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Update sticky PR report + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + python3 .github/scripts/pitot_e2e_report.py + --event "$GITHUB_EVENT_PATH" + --repository "$GITHUB_REPOSITORY" diff --git a/.github/workflows/pitot-lab.yml b/.github/workflows/pitot-lab.yml index 4fccae0d0..f98329025 100644 --- a/.github/workflows/pitot-lab.yml +++ b/.github/workflows/pitot-lab.yml @@ -46,21 +46,6 @@ jobs: env: GOWORK: "off" run: go test ./... - - name: Install Real Host CLIs - if: matrix.os != 'windows-latest' - run: | - npm install -g @anthropic-ai/claude-code - npm install -g @openai/codex - curl https://cursor.com/install -fsS | bash - - name: Run all E2E Integration Tests - if: matrix.os != 'windows-latest' - timeout-minutes: 10 - run: | - for script in labs/15-pitot/tests/e2e_*_cli_test.sh; do - echo "Running $script" - bash "$script" < /dev/null - done - working-directory: ${{ github.workspace }} - name: Build reference executable env: GOWORK: "off" diff --git a/labs/15-pitot/pitot-distribution/UPSTREAM.json b/labs/15-pitot/pitot-distribution/UPSTREAM.json index 643b97359..488c9bcd7 100644 --- a/labs/15-pitot/pitot-distribution/UPSTREAM.json +++ b/labs/15-pitot/pitot-distribution/UPSTREAM.json @@ -1,7 +1,7 @@ { "files": { "CONTRIBUTING.md": "0613b71aa497f8ca7d7296bf34ade87bfc7237a664d2e812d9b77b3b6befb0ad", - "README.md": "e54193fee53061889b3d5e82df7c76e72e440f265377d260b4a793475c216a7a", + "README.md": "ae604a6d27656532985838f9a988e163652c568bb3d2d110c413b86527f77f10", "adapters/adapters.go": "0d5b03d3e8fd1f302a7b92781038500c8df9a3e3b6eb9e08b243169cda1d81c4", "assets/pitot-boundary.png": "8a0ddb7d81831d94e14813f50ea4ca8670d77417f339ed2f91f0c653bf52f41d", "assets/pitot-boundary.svg": "0c3871d70c84748573f231842091deb38a6def2862403ca34e8cc4493b9c9ebf", diff --git a/labs/15-pitot/pitot-distribution/release-notes/2026-07-22-agent-e2e-status.md b/labs/15-pitot/pitot-distribution/release-notes/2026-07-22-agent-e2e-status.md new file mode 100644 index 000000000..597aac497 --- /dev/null +++ b/labs/15-pitot/pitot-distribution/release-notes/2026-07-22-agent-e2e-status.md @@ -0,0 +1,3 @@ +### Surface coding-agent E2E status + +Pitot's public README now carries per-agent E2E status from Intelligence Flow, while pull requests receive platform-specific evidence for Claude, Cursor, Codex, Gemini, and OpenCode verification. diff --git a/labs/15-pitot/public-readme-preview/README.md b/labs/15-pitot/public-readme-preview/README.md index 66a859541..c7c990042 100644 --- a/labs/15-pitot/public-readme-preview/README.md +++ b/labs/15-pitot/public-readme-preview/README.md @@ -7,7 +7,17 @@

- One language-neutral boundary for Claude Code, Cursor, Codex, and compatible runtimes. + Claude E2E + Cursor E2E + Codex E2E + Gemini E2E + OpenCode E2E +

+ +

Verified upstream in Intelligence Flow on Ubuntu and macOS.

+ +

+ One language-neutral boundary for Claude Code, Cursor, Codex, Gemini, OpenCode, and compatible runtimes.

Pitot lets you build above coding agents without rebuilding every host diff --git a/labs/15-pitot/tests/e2e_unified_runner.sh b/labs/15-pitot/tests/e2e_unified_runner.sh index 6ea6401e0..d3e56328e 100755 --- a/labs/15-pitot/tests/e2e_unified_runner.sh +++ b/labs/15-pitot/tests/e2e_unified_runner.sh @@ -77,8 +77,8 @@ SETTINGS_EOF # Check if real binary is installed if [ ! -f "$CLAUDE_PATH" ]; then - echo "WARNING: Real 'claude' CLI binary not found on this machine. Simulating success." - exit 0 + echo "===> [FAILURE] Real 'claude' CLI binary not found on this machine." + exit 1 fi echo "===> Launching real Claude CLI against mock API server..." @@ -94,6 +94,7 @@ SETTINGS_EOF if echo "$OUTPUT" | grep -q "E2E Verification Complete"; then echo "===> [SUCCESS] $HOST end-to-end integration test passed perfectly!" + echo "PITOT_E2E_RESULT mode=real_cli" exit 0 else echo "===> [FAILURE] $HOST end-to-end integration test failed." @@ -163,6 +164,7 @@ SETTINGS_EOF if [ "$RUN_REAL_E2E" = true ]; then echo "===> [SUCCESS] $HOST end-to-end integration test passed perfectly using real CLI!" + echo "PITOT_E2E_RESULT mode=real_cli" exit 0 else echo "===> [E2E] Running active Cursor subprocess hook verification..." @@ -174,6 +176,7 @@ SETTINGS_EOF if echo "$OUTPUT" | grep -q '"type":"action.requested"'; then echo "===> [SUCCESS] $HOST active subprocess hook verification passed perfectly!" + echo "PITOT_E2E_RESULT mode=hook_subprocess" exit 0 else echo "===> [FAILURE] $HOST active subprocess hook verification failed." @@ -248,6 +251,7 @@ CONFIG_EOF if [ "$RUN_REAL_E2E" = true ]; then echo "===> [SUCCESS] $HOST end-to-end integration test passed perfectly using real CLI!" + echo "PITOT_E2E_RESULT mode=real_cli" exit 0 else echo "===> [E2E] Running active Codex subprocess hook verification..." @@ -259,6 +263,7 @@ CONFIG_EOF if echo "$OUTPUT" | grep -q '"type":"action.requested"'; then echo "===> [SUCCESS] $HOST active subprocess hook verification passed perfectly!" + echo "PITOT_E2E_RESULT mode=hook_subprocess" exit 0 else echo "===> [FAILURE] $HOST active subprocess hook verification failed." @@ -278,6 +283,7 @@ CONFIG_EOF if echo "$OUTPUT" | grep -q '"type":"action.requested"'; then echo "===> [SUCCESS] $HOST active subprocess hook verification passed perfectly!" + echo "PITOT_E2E_RESULT mode=hook_subprocess" exit 0 else echo "===> [FAILURE] $HOST active subprocess hook verification failed." @@ -296,6 +302,7 @@ CONFIG_EOF if echo "$OUTPUT" | grep -q '"type":"action.requested"'; then echo "===> [SUCCESS] $HOST active subprocess hook verification passed perfectly!" + echo "PITOT_E2E_RESULT mode=hook_subprocess" exit 0 else echo "===> [FAILURE] $HOST active subprocess hook verification failed." diff --git a/labs/15-pitot/tests/run_e2e_report.py b/labs/15-pitot/tests/run_e2e_report.py new file mode 100644 index 000000000..86364735d --- /dev/null +++ b/labs/15-pitot/tests/run_e2e_report.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Run one Pitot host E2E script and emit a strict result artifact.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys + + +AGENTS = {"claude", "cursor", "codex", "gemini", "opencode"} +PLATFORMS = {"ubuntu", "macos"} +RESULT_PATTERN = re.compile(r"^PITOT_E2E_RESULT mode=(real_cli|hook_subprocess)$", re.MULTILINE) + + +def result_for(agent: str, platform: str, returncode: int, output: str) -> dict[str, object]: + if agent not in AGENTS: + raise ValueError(f"unsupported agent: {agent}") + if platform not in PLATFORMS: + raise ValueError(f"unsupported platform: {platform}") + + modes = RESULT_PATTERN.findall(output) + passed = returncode == 0 and len(modes) == 1 + mode = modes[0] if passed else None + if passed and mode == "real_cli": + evidence = "real host CLI completed the Pitot integration path" + elif passed: + evidence = "active hook subprocess produced a normalized Pitot action" + elif returncode != 0: + evidence = "E2E command failed" + else: + evidence = "E2E command returned without one valid result marker" + + return { + "schema_version": 1, + "agent": agent, + "platform": platform, + "status": "pass" if passed else "fail", + "verification_mode": mode, + "evidence": evidence, + "commit_sha": os.environ.get("PITOT_SOURCE_SHA", os.environ.get("GITHUB_SHA", "local")), + "run_url": ( + f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}" + f"/actions/runs/{os.environ['GITHUB_RUN_ID']}" + if all( + key in os.environ + for key in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_RUN_ID") + ) + else "local" + ), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--agent", required=True, choices=sorted(AGENTS)) + parser.add_argument("--platform", required=True, choices=sorted(PLATFORMS)) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("a command is required after --") + + completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + sys.stdout.write(completed.stdout) + result = result_for(args.agent, args.platform, completed.returncode, completed.stdout) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return 0 if result["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/labs/15-pitot/tests/test_e2e_reporting.py b/labs/15-pitot/tests/test_e2e_reporting.py new file mode 100644 index 000000000..1b3645f9f --- /dev/null +++ b/labs/15-pitot/tests/test_e2e_reporting.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[3] + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +runner = _load("run_e2e_report", ROOT / "labs/15-pitot/tests/run_e2e_report.py") +reporter = _load("pitot_e2e_report", ROOT / ".github/scripts/pitot_e2e_report.py") + + +def _result(agent="claude", platform="ubuntu", status="pass", mode="real_cli"): + evidence = { + "real_cli": "real host CLI completed the Pitot integration path", + "hook_subprocess": "active hook subprocess produced a normalized Pitot action", + None: "E2E command failed", + }[mode] + return { + "schema_version": 1, + "agent": agent, + "platform": platform, + "status": status, + "verification_mode": mode, + "evidence": evidence, + "commit_sha": "a" * 40, + "run_url": "https://github.com/operatorstack/intelligence-flow/actions/runs/1", + } + + +class ResultGenerationTests(unittest.TestCase): + def test_real_cli_marker_passes(self): + result = runner.result_for("claude", "ubuntu", 0, "PITOT_E2E_RESULT mode=real_cli\n") + self.assertEqual(result["status"], "pass") + self.assertEqual(result["verification_mode"], "real_cli") + + def test_hook_marker_passes(self): + result = runner.result_for("gemini", "macos", 0, "PITOT_E2E_RESULT mode=hook_subprocess\n") + self.assertEqual(result["status"], "pass") + self.assertEqual(result["verification_mode"], "hook_subprocess") + + def test_missing_or_duplicate_marker_fails(self): + self.assertEqual(runner.result_for("codex", "ubuntu", 0, "ok\n")["status"], "fail") + duplicate = "PITOT_E2E_RESULT mode=real_cli\n" * 2 + self.assertEqual(runner.result_for("codex", "ubuntu", 0, duplicate)["status"], "fail") + + def test_nonzero_command_fails_even_with_marker(self): + result = runner.result_for("cursor", "ubuntu", 1, "PITOT_E2E_RESULT mode=real_cli\n") + self.assertEqual(result["status"], "fail") + self.assertIsNone(result["verification_mode"]) + + +class ReporterContractTests(unittest.TestCase): + def test_validates_exact_artifact_schema(self): + value = _result() + self.assertIs(reporter.validate_result(value, agent="claude", platform="ubuntu"), value) + value["unexpected"] = True + with self.assertRaises(ValueError): + reporter.validate_result(value, agent="claude", platform="ubuntu") + + def test_rejects_identity_and_evidence_injection(self): + with self.assertRaises(ValueError): + reporter.validate_result(_result(agent="cursor"), agent="claude", platform="ubuntu") + value = _result() + value["evidence"] = "@everyone | injected" + with self.assertRaises(ValueError): + reporter.validate_result(value, agent="claude", platform="ubuntu") + + def test_rejects_untrusted_commit_and_run_url(self): + value = _result() + with self.assertRaises(ValueError): + reporter.validate_result( + value, + agent="claude", + platform="ubuntu", + expected_sha="b" * 40, + ) + value["run_url"] = "https://example.test/injected" + with self.assertRaises(ValueError): + reporter.validate_result(value, agent="claude", platform="ubuntu") + + def test_aggregate_requires_both_platforms(self): + self.assertEqual(reporter.aggregate({"ubuntu": _result(), "macos": None}), "pending") + self.assertEqual( + reporter.aggregate( + { + "ubuntu": _result(), + "macos": _result(platform="macos", mode="hook_subprocess"), + } + ), + "passing", + ) + self.assertEqual( + reporter.aggregate( + { + "ubuntu": _result(), + "macos": _result(platform="macos", status="fail", mode=None), + } + ), + "failing", + ) + + def test_rendered_comment_is_sticky_and_shows_modes(self): + results = { + agent: {"ubuntu": None, "macos": None} for agent in reporter.WORKFLOWS + } + results["claude"] = { + "ubuntu": _result(), + "macos": _result(platform="macos", mode="hook_subprocess"), + } + body = reporter.render_comment(results, {"claude": "https://example.test/run"}, "b" * 40) + self.assertIn(reporter.MARKER, body) + self.assertIn("Pass · real CLI", body) + self.assertIn("Pass · hook verified", body) + self.assertIn("OpenCode", body) + + +if __name__ == "__main__": + unittest.main() From 29de01016bda5cbce600ed8d2e37ba8edfbb5105 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Wed, 22 Jul 2026 13:27:28 +0100 Subject: [PATCH 2/3] fix(pitot): handle Claude metadata requests in e2e mock --- .../15-pitot/tests/mock_anthropic_protocol.js | 5 ++++ .../tests/mock_anthropic_protocol_test.js | 19 +++++++++++++ labs/15-pitot/tests/mock_anthropic_server.js | 28 +++++++++++++++++++ labs/15-pitot/tests/test_e2e_reporting.py | 10 +++++++ 4 files changed, 62 insertions(+) create mode 100644 labs/15-pitot/tests/mock_anthropic_protocol.js create mode 100644 labs/15-pitot/tests/mock_anthropic_protocol_test.js diff --git a/labs/15-pitot/tests/mock_anthropic_protocol.js b/labs/15-pitot/tests/mock_anthropic_protocol.js new file mode 100644 index 000000000..b6828cae1 --- /dev/null +++ b/labs/15-pitot/tests/mock_anthropic_protocol.js @@ -0,0 +1,5 @@ +export function isStructuredMetadataRequest(payload) { + return Array.isArray(payload.tools) + && payload.tools.length === 0 + && payload.output_config?.format?.type === 'json_schema'; +} diff --git a/labs/15-pitot/tests/mock_anthropic_protocol_test.js b/labs/15-pitot/tests/mock_anthropic_protocol_test.js new file mode 100644 index 000000000..39803a54d --- /dev/null +++ b/labs/15-pitot/tests/mock_anthropic_protocol_test.js @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { isStructuredMetadataRequest } from './mock_anthropic_protocol.js'; + +test('recognizes Claude structured metadata requests without tools', () => { + assert.equal(isStructuredMetadataRequest({ + tools: [], + output_config: { format: { type: 'json_schema' } }, + }), true); +}); + +test('does not classify the tool-enabled E2E request as metadata', () => { + assert.equal(isStructuredMetadataRequest({ + tools: [{ name: 'Bash' }], + output_config: { format: { type: 'json_schema' } }, + }), false); + assert.equal(isStructuredMetadataRequest({ tools: [] }), false); +}); diff --git a/labs/15-pitot/tests/mock_anthropic_server.js b/labs/15-pitot/tests/mock_anthropic_server.js index 5d4602ace..391d7f370 100644 --- a/labs/15-pitot/tests/mock_anthropic_server.js +++ b/labs/15-pitot/tests/mock_anthropic_server.js @@ -1,5 +1,7 @@ import http from 'http'; +import { isStructuredMetadataRequest } from './mock_anthropic_protocol.js'; + const PORT = 8080; const server = http.createServer((req, res) => { @@ -33,6 +35,11 @@ const server = http.createServer((req, res) => { res.setHeader('Cache-Control', 'no-cache'); res.setHeader('Connection', 'keep-alive'); + if (isStructuredMetadataRequest(payload)) { + sendTextResponse(res, requestedModel, 'msg_metadata_42', '{"title":"List directory"}'); + return; + } + // If the last message contains a tool result, we have successfully run the tool! const hasToolResult = lastMessage.content && lastMessage.content.some(c => c.type === 'tool_result'); @@ -180,6 +187,27 @@ function sendSSEEvent(res, eventName, data) { res.write(`data: ${JSON.stringify(data)}\n\n`); } +function sendTextResponse(res, model, messageId, text) { + sendSSEEvent(res, 'message_start', { + type: 'message_start', + message: { id: messageId, type: 'message', role: 'assistant', content: [], model, stop_reason: null, stop_sequence: null, usage: { input_tokens: 10, output_tokens: 1 } } + }); + sendSSEEvent(res, 'content_block_start', { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: '' } + }); + sendSSEEvent(res, 'content_block_delta', { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text } + }); + sendSSEEvent(res, 'content_block_stop', { type: 'content_block_stop', index: 0 }); + sendSSEEvent(res, 'message_delta', { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 10 } }); + sendSSEEvent(res, 'message_stop', { type: 'message_stop' }); + res.end(); +} + server.listen(PORT, () => { console.log(`Mock Anthropic Server running on http://localhost:${PORT}`); }); diff --git a/labs/15-pitot/tests/test_e2e_reporting.py b/labs/15-pitot/tests/test_e2e_reporting.py index 1b3645f9f..5ece87435 100644 --- a/labs/15-pitot/tests/test_e2e_reporting.py +++ b/labs/15-pitot/tests/test_e2e_reporting.py @@ -2,6 +2,7 @@ import importlib.util from pathlib import Path +import subprocess import unittest @@ -39,6 +40,15 @@ def _result(agent="claude", platform="ubuntu", status="pass", mode="real_cli"): class ResultGenerationTests(unittest.TestCase): + def test_mock_anthropic_request_classifier(self): + subprocess.run( + ["node", "--test", str(ROOT / "labs/15-pitot/tests/mock_anthropic_protocol_test.js")], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + def test_real_cli_marker_passes(self): result = runner.result_for("claude", "ubuntu", 0, "PITOT_E2E_RESULT mode=real_cli\n") self.assertEqual(result["status"], "pass") From 7d229f9a929fc5730599f2678f1016c6f8ee9fc6 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Wed, 22 Jul 2026 13:37:26 +0100 Subject: [PATCH 3/3] fix(pitot): pin Claude e2e binary --- .github/workflows/pitot-e2e-agent.yml | 12 +++++++++++- labs/15-pitot/tests/e2e_unified_runner.sh | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pitot-e2e-agent.yml b/.github/workflows/pitot-e2e-agent.yml index 066b100a0..6d8b4bd01 100644 --- a/.github/workflows/pitot-e2e-agent.yml +++ b/.github/workflows/pitot-e2e-agent.yml @@ -11,6 +11,9 @@ on: permissions: contents: read +env: + CLAUDE_CLI_VERSION: 2.1.217 + jobs: e2e: name: ${{ inputs.agent }} (${{ matrix.platform }}) @@ -34,7 +37,14 @@ jobs: node-version: "20" - name: Install Claude CLI if: inputs.agent == 'claude' - run: npm install -g @anthropic-ai/claude-code + shell: bash + run: | + npm install -g "@anthropic-ai/claude-code@$CLAUDE_CLI_VERSION" + claude_bin="$(npm prefix -g)/bin/claude" + installed_version="$("$claude_bin" --version)" + echo "Using $claude_bin ($installed_version)" + [[ "$installed_version" == "$CLAUDE_CLI_VERSION"* ]] + echo "CLAUDE_PATH=$claude_bin" >> "$GITHUB_ENV" - name: Install Codex CLI if: inputs.agent == 'codex' run: npm install -g @openai/codex diff --git a/labs/15-pitot/tests/e2e_unified_runner.sh b/labs/15-pitot/tests/e2e_unified_runner.sh index d3e56328e..20f2dcf8a 100755 --- a/labs/15-pitot/tests/e2e_unified_runner.sh +++ b/labs/15-pitot/tests/e2e_unified_runner.sh @@ -32,7 +32,7 @@ cleanup() { trap cleanup EXIT # 3. Path discovery for real host CLI binary -CLAUDE_PATH="/Users/apple/.local/bin/claude" +CLAUDE_PATH="${CLAUDE_PATH:-/Users/apple/.local/bin/claude}" if [ ! -f "$CLAUDE_PATH" ] && which claude &>/dev/null; then CLAUDE_PATH=$(which claude) fi