Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 22 additions & 6 deletions .github/scripts/pitot_e2e_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,24 @@
"gemini": "Pitot E2E — Gemini",
"opencode": "Pitot E2E — OpenCode",
}
PLATFORMS = ("ubuntu", "macos")
PLATFORMS = ("ubuntu", "macos", "windows")
MAX_ARTIFACT_BYTES = 1_000_000


class ArtifactRedirectHandler(urllib.request.HTTPRedirectHandler):
def redirect_request(self, request, file_pointer, code, message, headers, new_url):
redirected = super().redirect_request(request, file_pointer, code, message, headers, new_url)
if redirected is None:
return None
source = urllib.parse.urlparse(request.full_url)
target = urllib.parse.urlparse(new_url)
if target.scheme != "https":
raise ValueError("artifact redirect must use HTTPS")
if source.netloc != target.netloc:
redirected.remove_header("Authorization")
return redirected


def validate_result(
value: object,
*,
Expand Down Expand Up @@ -118,8 +132,8 @@ def render_comment(
"",
"Intelligence Flow is the verification source; Pitot's public README carries the latest `main` status.",
"",
"| Agent | Ubuntu | macOS | Result | Evidence |",
"|---|---|---|---|---|",
"| Agent | Ubuntu | macOS | Windows | Result | Evidence |",
"|---|---|---|---|---|---|",
]
icons = {"passing": "✅ Passing", "failing": "❌ Failing", "pending": "⏳ Pending"}
for agent in WORKFLOWS:
Expand All @@ -128,10 +142,11 @@ def render_comment(
url = run_urls.get(agent)
label = agent.capitalize() if agent != "opencode" else "OpenCode"
linked_label = f"[{label}]({url})" if url else label
evidence = "Both platforms are required" if status != "pending" else "Waiting for both platform artifacts"
evidence = "All platforms are required" if status != "pending" else "Waiting for all platform artifacts"
lines.append(
f"| {linked_label} | {result_cell(platforms.get('ubuntu'))} | "
f"{result_cell(platforms.get('macos'))} | {icons[status]} | {evidence} |"
f"{result_cell(platforms.get('macos'))} | {result_cell(platforms.get('windows'))} | "
f"{icons[status]} | {evidence} |"
)
lines.extend(["", f"Source commit: `{head_sha[:12]}`", ""])
return "\n".join(lines)
Expand Down Expand Up @@ -166,7 +181,8 @@ def download(self, url: str) -> bytes:
url,
headers={"Authorization": f"Bearer {self.token}", "User-Agent": "pitot-e2e-reporter"},
)
with urllib.request.urlopen(request, timeout=30) as response:
opener = urllib.request.build_opener(ArtifactRedirectHandler())
with opener.open(request, timeout=30) as response:
data = response.read(MAX_ARTIFACT_BYTES + 1)
if len(data) > MAX_ARTIFACT_BYTES:
raise ValueError("artifact exceeds size limit")
Expand Down
22 changes: 18 additions & 4 deletions .github/workflows/pitot-e2e-agent.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ jobs:
platform: ubuntu
- runner: macos-latest
platform: macos
- runner: windows-latest
platform: windows
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v7
Expand All @@ -35,12 +37,24 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: "20"
- uses: actions/setup-python@v6
with:
python-version: "3.11"
- name: Resolve Bash runtime
shell: bash
run: |
bash_path="$(command -v bash)"
if [[ "${{ runner.os }}" == "Windows" ]]; then
bash_path="$(cygpath -m "$bash_path")"
fi
echo "PITOT_BASH=$bash_path" >> "$GITHUB_ENV"
- name: Install Claude CLI
if: inputs.agent == 'claude'
shell: bash
run: |
npm install -g "@anthropic-ai/claude-code@$CLAUDE_CLI_VERSION"
claude_bin="$(npm prefix -g)/bin/claude"
claude_bin="$(command -v claude)"
[[ -n "$claude_bin" ]]
installed_version="$("$claude_bin" --version)"
echo "Using $claude_bin ($installed_version)"
[[ "$installed_version" == "$CLAUDE_CLI_VERSION"* ]]
Expand All @@ -49,7 +63,7 @@ jobs:
if: inputs.agent == 'codex'
run: npm install -g @openai/codex
- name: Install Cursor CLI
if: inputs.agent == 'cursor'
if: inputs.agent == 'cursor' && runner.os != 'Windows'
run: curl https://cursor.com/install -fsS | bash
- name: Run ${{ inputs.agent }} E2E
id: e2e
Expand All @@ -59,11 +73,11 @@ jobs:
env:
PITOT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
run: >-
python3 labs/15-pitot/tests/run_e2e_report.py
python labs/15-pitot/tests/run_e2e_report.py
--agent "${{ inputs.agent }}"
--platform "${{ matrix.platform }}"
--output "${{ runner.temp }}/pitot-e2e/result.json"
-- bash "labs/15-pitot/tests/e2e_${{ inputs.agent }}_cli_test.sh"
-- "$PITOT_BASH" "labs/15-pitot/tests/e2e_${{ inputs.agent }}_cli_test.sh"
< /dev/null
- name: Upload structured E2E result
if: always()
Expand Down
2 changes: 1 addition & 1 deletion labs/15-pitot/pitot-distribution/UPSTREAM.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"files": {
"CONTRIBUTING.md": "0613b71aa497f8ca7d7296bf34ade87bfc7237a664d2e812d9b77b3b6befb0ad",
"README.md": "ae604a6d27656532985838f9a988e163652c568bb3d2d110c413b86527f77f10",
"README.md": "20cacdc35bd17d34c8451f8eb87b8bb7a788ca3d3c6c1283e36bfc901f0a4479",
"adapters/adapters.go": "0d5b03d3e8fd1f302a7b92781038500c8df9a3e3b6eb9e08b243169cda1d81c4",
"assets/pitot-boundary.png": "8a0ddb7d81831d94e14813f50ea4ca8670d77417f339ed2f91f0c653bf52f41d",
"assets/pitot-boundary.svg": "0c3871d70c84748573f231842091deb38a6def2862403ca34e8cc4493b9c9ebf",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Verify every coding-agent integration on Windows

Pitot's per-agent E2E matrix and pull-request report now require Windows alongside Ubuntu and macOS. Windows results use the same structured, commit-bound evidence artifacts, while host setup preserves real CLI verification where supported and the existing active-hook subprocess boundary where a Windows CLI installer is unavailable.
2 changes: 1 addition & 1 deletion labs/15-pitot/public-readme-preview/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<a href="https://github.com/operatorstack/intelligence-flow/actions/workflows/pitot-e2e-opencode.yml"><img alt="OpenCode E2E" src="https://github.com/operatorstack/intelligence-flow/actions/workflows/pitot-e2e-opencode.yml/badge.svg?branch=main"></a>
</p>

<p align="center"><sub>Verified upstream in <a href="https://github.com/operatorstack/intelligence-flow">Intelligence Flow</a> on Ubuntu and macOS.</sub></p>
<p align="center"><sub>Verified upstream in <a href="https://github.com/operatorstack/intelligence-flow">Intelligence Flow</a> on Ubuntu, macOS, and Windows.</sub></p>

<p align="center">
One language-neutral boundary for Claude Code, Cursor, Codex, Gemini, OpenCode, and compatible runtimes.
Expand Down
15 changes: 11 additions & 4 deletions labs/15-pitot/tests/e2e_unified_runner.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@ echo "===> [E2E] Starting $HOST + Pitot Hook Integration Test"

# 1. Compile the local Go 'pitot' binary
echo "===> Compiling pitot binary..."
go build -o labs/15-pitot/tests/pitot labs/15-pitot/pitot/cmd/pitot/main.go
PITOT_BINARY="labs/15-pitot/tests/pitot"
if [ "${RUNNER_OS:-}" = "Windows" ]; then
PITOT_BINARY="${PITOT_BINARY}.exe"
fi
go build -o "$PITOT_BINARY" labs/15-pitot/pitot/cmd/pitot/main.go

# 2. Host-specific setup and mocking
SERVER_PID=""
Expand All @@ -24,7 +28,7 @@ cleanup() {
if [ -n "$SERVER_PID" ]; then
kill "$SERVER_PID" 2>/dev/null || true
fi
rm -f labs/15-pitot/tests/pitot
rm -f "$PITOT_BINARY"
if [ -n "${MOCK_HOME:-}" ]; then
rm -rf "$MOCK_HOME"
fi
Expand All @@ -44,8 +48,11 @@ fi

CODEX_PATH="" # Codex CLI path placeholder

# Get absolute path of the compiled pitot binary
PITOT_ABS_PATH=$(pwd)/labs/15-pitot/tests/pitot
# Get an absolute path that the host process can execute on each runner OS.
PITOT_ABS_PATH="$(pwd)/$PITOT_BINARY"
if [ "${RUNNER_OS:-}" = "Windows" ] && command -v cygpath &>/dev/null; then
PITOT_ABS_PATH="$(cygpath -m "$PITOT_ABS_PATH")"
fi

# Spin up mock API server on port 8080 for all tests
echo "===> Starting local mock API server..."
Expand Down
13 changes: 10 additions & 3 deletions labs/15-pitot/tests/run_e2e_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@


AGENTS = {"claude", "cursor", "codex", "gemini", "opencode"}
PLATFORMS = {"ubuntu", "macos"}
PLATFORMS = {"ubuntu", "macos", "windows"}
RESULT_PATTERN = re.compile(r"^PITOT_E2E_RESULT mode=(real_cli|hook_subprocess)$", re.MULTILINE)


Expand Down Expand Up @@ -66,8 +66,15 @@ def main() -> int:
if not command:
parser.error("a command is required after --")

completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
sys.stdout.write(completed.stdout)
completed = subprocess.run(
command,
text=True,
encoding="utf-8",
errors="replace",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
sys.stdout.buffer.write(completed.stdout.encode("utf-8", errors="replace"))
result = result_for(args.agent, args.platform, completed.returncode, completed.stdout)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
Expand Down
86 changes: 84 additions & 2 deletions labs/15-pitot/tests/test_e2e_reporting.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
import importlib.util
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
import urllib.request


ROOT = Path(__file__).resolve().parents[3]
Expand Down Expand Up @@ -59,6 +62,11 @@ def test_hook_marker_passes(self):
self.assertEqual(result["status"], "pass")
self.assertEqual(result["verification_mode"], "hook_subprocess")

def test_windows_result_is_supported(self):
result = runner.result_for("opencode", "windows", 0, "PITOT_E2E_RESULT mode=hook_subprocess\n")
self.assertEqual(result["platform"], "windows")
self.assertEqual(result["status"], "pass")

def test_missing_or_duplicate_marker_fails(self):
self.assertEqual(runner.result_for("codex", "ubuntu", 0, "ok\n")["status"], "fail")
duplicate = "PITOT_E2E_RESULT mode=real_cli\n" * 2
Expand All @@ -69,8 +77,66 @@ def test_nonzero_command_fails_even_with_marker(self):
self.assertEqual(result["status"], "fail")
self.assertIsNone(result["verification_mode"])

def test_runner_replaces_non_utf8_command_output(self):
with tempfile.TemporaryDirectory() as temp:
output = Path(temp) / "result.json"
completed = subprocess.run(
[
sys.executable,
str(ROOT / "labs/15-pitot/tests/run_e2e_report.py"),
"--agent", "claude",
"--platform", "windows",
"--output", str(output),
"--",
sys.executable,
"-c", "import sys; sys.stdout.buffer.write(b'\\x96')",
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
self.assertNotEqual(completed.returncode, 0)
self.assertTrue(output.is_file())


class ReporterContractTests(unittest.TestCase):
def test_artifact_redirect_strips_cross_host_credentials(self):
request = urllib.request.Request(
"https://api.github.com/repos/operatorstack/intelligence-flow/actions/artifacts/1/zip",
headers={"Authorization": "Bearer secret"},
)
redirected = reporter.ArtifactRedirectHandler().redirect_request(
request,
None,
302,
"Found",
{},
"https://artifact.example.test/signed.zip",
)
self.assertIsNotNone(redirected)
self.assertIsNone(redirected.get_header("Authorization"))

def test_artifact_redirect_rejects_http_downgrade(self):
request = urllib.request.Request("https://api.github.com/artifact")
with self.assertRaisesRegex(ValueError, "must use HTTPS"):
reporter.ArtifactRedirectHandler().redirect_request(
request,
None,
302,
"Found",
{},
"http://artifact.example.test/signed.zip",
)

def test_workflow_matrix_covers_every_reported_platform(self):
workflow = (ROOT / ".github/workflows/pitot-e2e-agent.yml").read_text()
runners = {"ubuntu": "ubuntu-latest", "macos": "macos-latest", "windows": "windows-latest"}
self.assertEqual(set(reporter.PLATFORMS), set(runner.PLATFORMS))
for platform, image in runners.items():
self.assertIn(f"runner: {image}\n platform: {platform}", workflow)
self.assertIn('bash_path="$(cygpath -m "$bash_path")"', workflow)
self.assertIn('-- "$PITOT_BASH"', workflow)
self.assertNotIn('-- bash "labs/15-pitot/tests/e2e_', workflow)

def test_validates_exact_artifact_schema(self):
value = _result()
self.assertIs(reporter.validate_result(value, agent="claude", platform="ubuntu"), value)
Expand Down Expand Up @@ -99,13 +165,24 @@ def test_rejects_untrusted_commit_and_run_url(self):
with self.assertRaises(ValueError):
reporter.validate_result(value, agent="claude", platform="ubuntu")

def test_aggregate_requires_both_platforms(self):
def test_aggregate_requires_all_platforms(self):
self.assertEqual(reporter.aggregate({"ubuntu": _result(), "macos": None}), "pending")
self.assertEqual(
reporter.aggregate(
{
"ubuntu": _result(),
"macos": _result(platform="macos", mode="hook_subprocess"),
"windows": None,
}
),
"pending",
)
self.assertEqual(
reporter.aggregate(
{
"ubuntu": _result(),
"macos": _result(platform="macos", mode="hook_subprocess"),
"windows": _result(platform="windows", mode="hook_subprocess"),
}
),
"passing",
Expand All @@ -115,23 +192,28 @@ def test_aggregate_requires_both_platforms(self):
{
"ubuntu": _result(),
"macos": _result(platform="macos", status="fail", mode=None),
"windows": _result(platform="windows", mode="hook_subprocess"),
}
),
"failing",
)

def test_rendered_comment_is_sticky_and_shows_modes(self):
results = {
agent: {"ubuntu": None, "macos": None} for agent in reporter.WORKFLOWS
agent: {platform: None for platform in reporter.PLATFORMS}
for agent in reporter.WORKFLOWS
}
results["claude"] = {
"ubuntu": _result(),
"macos": _result(platform="macos", mode="hook_subprocess"),
"windows": _result(platform="windows", mode="hook_subprocess"),
}
body = reporter.render_comment(results, {"claude": "https://example.test/run"}, "b" * 40)
self.assertIn(reporter.MARKER, body)
self.assertIn("Pass · real CLI", body)
self.assertIn("Pass · hook verified", body)
self.assertIn("Windows", body)
self.assertIn("All platforms are required", body)
self.assertIn("OpenCode", body)


Expand Down
Loading