- 
- 
- 
- 
+ 
-
-
-
-
-
+
The badged hosts are verified upstream in Intelligence Flow on Ubuntu, macOS, and Windows.
+Every supervised adapter is required on Ubuntu, macOS, and Windows.
+ +Supervised adapters: Claude · Cursor · Codex · Gemini · Kimi Code · OpenCode
+One language-neutral boundary for Claude Code, Cursor, Codex, Gemini, Kimi Code, OpenCode, and compatible runtimes. diff --git a/labs/15-pitot/scripts/pitot_adapter_inventory.go b/labs/15-pitot/scripts/pitot_adapter_inventory.go new file mode 100644 index 000000000..9e43f6c6e --- /dev/null +++ b/labs/15-pitot/scripts/pitot_adapter_inventory.go @@ -0,0 +1,22 @@ +// Command pitot_adapter_inventory prints the shipped Pitot adapter IDs for the +// repository supervisor. It is internal verification machinery, not a public +// Pitot command. +package main + +import ( + "encoding/json" + "os" + + "github.com/operatorstack/pitot/adapters" +) + +func main() { + hosts := adapters.Supported() + values := make([]string, len(hosts)) + for index, host := range hosts { + values[index] = string(host) + } + if err := json.NewEncoder(os.Stdout).Encode(values); err != nil { + panic(err) + } +} diff --git a/labs/15-pitot/scripts/pitot_adapter_supervisor.py b/labs/15-pitot/scripts/pitot_adapter_supervisor.py new file mode 100644 index 000000000..d34d4b157 --- /dev/null +++ b/labs/15-pitot/scripts/pitot_adapter_supervisor.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Supervise Pitot's complete built-in adapter verification boundary.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[3] +MANIFEST = Path("labs/15-pitot/adapter-verification.json") +README = Path("labs/15-pitot/public-readme-preview/README.md") +UNIFIED_WORKFLOW = Path(".github/workflows/pitot-e2e.yml") +REPORT_WORKFLOW = Path(".github/workflows/pitot-e2e-report.yml") +REUSABLE_WORKFLOW = Path(".github/workflows/pitot-e2e-agent.yml") +INVENTORY_HELPER = Path("labs/15-pitot/scripts/pitot_adapter_inventory.go") +README_START = "" +README_END = "" +ID_PATTERN = re.compile(r"^[a-z][a-z0-9-]*$") +LABEL_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 .+_-]{0,39}$") + + +class ContractError(ValueError): + pass + + +def load_manifest(root: Path = ROOT) -> dict[str, object]: + try: + value = json.loads((root / MANIFEST).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ContractError(f"cannot load {MANIFEST}: {error}") from error + validate_manifest(value) + return value + + +def validate_manifest(value: object) -> None: + if not isinstance(value, dict) or set(value) != {"schema_version", "platforms", "agents"}: + raise ContractError("manifest must contain schema_version, platforms, and agents") + if value["schema_version"] != 1: + raise ContractError("unsupported manifest schema_version") + platforms = value["platforms"] + agents = value["agents"] + if not isinstance(platforms, list) or not isinstance(agents, list) or not agents: + raise ContractError("platforms and non-empty agents must be lists") + expected_platforms = [ + {"id": "ubuntu", "runner": "ubuntu-latest"}, + {"id": "macos", "runner": "macos-latest"}, + {"id": "windows", "runner": "windows-latest"}, + ] + if platforms != expected_platforms: + raise ContractError("platforms must be the required Ubuntu, macOS, and Windows matrix") + ids: list[str] = [] + labels: list[str] = [] + for agent in agents: + if not isinstance(agent, dict) or set(agent) != {"id", "label"}: + raise ContractError("each agent must contain exactly id and label") + agent_id, label = agent["id"], agent["label"] + if not isinstance(agent_id, str) or not ID_PATTERN.fullmatch(agent_id): + raise ContractError(f"invalid agent id: {agent_id!r}") + if not isinstance(label, str) or not LABEL_PATTERN.fullmatch(label): + raise ContractError(f"agent {agent_id} has an invalid label") + ids.append(agent_id) + labels.append(label) + if len(ids) != len(set(ids)): + raise ContractError("agent ids must be unique") + if len(labels) != len(set(labels)): + raise ContractError("agent labels must be unique") + + +def built_in_adapters(root: Path = ROOT) -> list[str]: + completed = subprocess.run( + ["go", "run", str(root / INVENTORY_HELPER)], + cwd=root, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + value = json.loads(completed.stdout) + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ContractError("Go adapter inventory returned an invalid result") + return value + + +def matrix(manifest: dict[str, object]) -> list[dict[str, str]]: + return [ + {"agent": agent["id"], "platform": platform["id"], "runner": platform["runner"]} + for agent in manifest["agents"] + for platform in manifest["platforms"] + ] + + +def readme_block(manifest: dict[str, object]) -> str: + labels = " · ".join(agent["label"] for agent in manifest["agents"]) + return "\n".join( + [ + README_START, + '
',
+ ' ',
+ "
Every supervised adapter is required on Ubuntu, macOS, and Windows.
', + "", + f'Supervised adapters: {labels}
', + README_END, + ] + ) + + +def replace_readme_block(text: str, manifest: dict[str, object]) -> str: + pattern = re.compile(re.escape(README_START) + r".*?" + re.escape(README_END), re.DOTALL) + if len(pattern.findall(text)) != 1: + raise ContractError("README must contain exactly one supervisor marker block") + return pattern.sub(readme_block(manifest), text) + + +def unified_workflow() -> str: + return """# Generated by pitot_adapter_supervisor.py. Do not edit directly. +name: Pitot E2E + +on: + pull_request: + paths: &pitot_paths + - "labs/15-pitot/**" + - "go.work" + - ".github/workflows/pitot-e2e*.yml" + - ".github/scripts/pitot_e2e_report.py" + push: + branches: [main] + paths: *pitot_paths + workflow_dispatch: + +permissions: + contents: read + +jobs: + inventory: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.supervisor.outputs.matrix }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: labs/15-pitot/pitot/go.mod + cache-dependency-path: labs/15-pitot/pitot/go.mod + - id: supervisor + shell: bash + run: | + matrix="$(python3 labs/15-pitot/scripts/pitot_adapter_supervisor.py matrix)" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + - name: Upload supervised adapter inventory + uses: actions/upload-artifact@v4 + with: + name: pitot-e2e-inventory + path: labs/15-pitot/adapter-verification.json + if-no-files-found: error + retention-days: 14 + + verify: + needs: inventory + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.inventory.outputs.matrix) }} + uses: ./.github/workflows/pitot-e2e-agent.yml + with: + agent: ${{ matrix.agent }} + platform: ${{ matrix.platform }} + runner: ${{ matrix.runner }} +""" + + +def report_workflow() -> str: + return """# Generated by pitot_adapter_supervisor.py. Do not edit directly. +name: Report Pitot E2E results + +on: + workflow_run: + workflows: [Pitot E2E] + types: [completed] + +permissions: + actions: read + contents: read + pull-requests: write + +concurrency: + group: pitot-e2e-report-${{ github.event.workflow_run.head_sha }} + cancel-in-progress: true + +jobs: + report: + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Update sticky PR report + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + python3 .github/scripts/pitot_e2e_report.py + --event "$GITHUB_EVENT_PATH" + --repository "$GITHUB_REPOSITORY" +""" + + +def contract_errors(root: Path, adapters: list[str] | None = None) -> list[str]: + try: + manifest = load_manifest(root) + except ContractError as error: + return [str(error)] + errors: list[str] = [] + inventory = adapters if adapters is not None else built_in_adapters(root) + declared = [agent["id"] for agent in manifest["agents"]] + missing = sorted(set(inventory) - set(declared)) + extra = sorted(set(declared) - set(inventory)) + if missing: + errors.append(f"built-in adapters missing from manifest: {', '.join(missing)}") + if extra: + errors.append(f"manifest agents are not built-in adapters: {', '.join(extra)}") + for agent_id in declared: + script = root / f"labs/15-pitot/tests/e2e_{agent_id}_cli_test.sh" + if not script.is_file(): + errors.append(f"adapter {agent_id} is missing {script.relative_to(root)}") + expected_files = { + UNIFIED_WORKFLOW: unified_workflow(), + REPORT_WORKFLOW: report_workflow(), + } + for relative, expected in expected_files.items(): + path = root / relative + actual = path.read_text(encoding="utf-8") if path.is_file() else "" + if actual != expected: + errors.append(f"generated surface drifted: {relative}") + reusable_path = root / REUSABLE_WORKFLOW + reusable = reusable_path.read_text(encoding="utf-8") if reusable_path.is_file() else "" + required_reusable_contract = ( + "platform:\n description: Canonical Pitot platform identifier", + "runner:\n description: GitHub-hosted runner selected by the supervisor", + "runs-on: ${{ inputs.runner }}", + '--agent "${{ inputs.agent }}"', + '--platform "${{ inputs.platform }}"', + "name: pitot-e2e-${{ inputs.agent }}-${{ inputs.platform }}", + ) + if any(fragment not in reusable for fragment in required_reusable_contract) or "matrix.platform" in reusable: + errors.append(f"reusable workflow escaped supervisor inputs: {REUSABLE_WORKFLOW}") + readme_path = root / README + if readme_path.is_file(): + actual = readme_path.read_text(encoding="utf-8") + try: + expected = replace_readme_block(actual, manifest) + if actual != expected: + errors.append(f"generated surface drifted: {README}") + except ContractError as error: + errors.append(str(error)) + else: + errors.append(f"missing {README}") + return errors + + +def render(root: Path = ROOT) -> None: + manifest = load_manifest(root) + readme_path = root / README + readme_path.write_text( + replace_readme_block(readme_path.read_text(encoding="utf-8"), manifest), + encoding="utf-8", + ) + (root / UNIFIED_WORKFLOW).write_text(unified_workflow(), encoding="utf-8") + (root / REPORT_WORKFLOW).write_text(report_workflow(), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("check", "matrix", "render")) + parser.add_argument("--repo", type=Path, default=ROOT) + args = parser.parse_args() + root = args.repo.resolve() + try: + if args.operation == "render": + render(root) + print("PASS: rendered Pitot adapter verification surfaces") + return 0 + errors = contract_errors(root) + if errors: + raise ContractError("; ".join(errors)) + if args.operation == "matrix": + manifest = load_manifest(root) + print(json.dumps({"include": matrix(manifest)}, separators=(",", ":"))) + else: + print(f"PASS: supervised {len(load_manifest(root)['agents'])} Pitot adapters") + return 0 + except (ContractError, subprocess.CalledProcessError) as error: + print(f"BLOCKED: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/labs/15-pitot/tests/run_e2e_report.py b/labs/15-pitot/tests/run_e2e_report.py index 157d7901b..80ca513f3 100644 --- a/labs/15-pitot/tests/run_e2e_report.py +++ b/labs/15-pitot/tests/run_e2e_report.py @@ -12,8 +12,10 @@ import sys -AGENTS = {"claude", "cursor", "codex", "gemini", "opencode"} -PLATFORMS = {"ubuntu", "macos", "windows"} +ROOT = Path(__file__).resolve().parents[3] +MANIFEST = json.loads((ROOT / "labs/15-pitot/adapter-verification.json").read_text(encoding="utf-8")) +AGENTS = {agent["id"] for agent in MANIFEST["agents"]} +PLATFORMS = {platform["id"] for platform in MANIFEST["platforms"]} RESULT_PATTERN = re.compile(r"^PITOT_E2E_RESULT mode=(real_cli|hook_subprocess)$", re.MULTILINE) diff --git a/labs/15-pitot/tests/test_adapter_supervisor.py b/labs/15-pitot/tests/test_adapter_supervisor.py new file mode 100644 index 000000000..5d4d6baef --- /dev/null +++ b/labs/15-pitot/tests/test_adapter_supervisor.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import copy +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[3] +SPEC = importlib.util.spec_from_file_location( + "pitot_adapter_supervisor", + ROOT / "labs/15-pitot/scripts/pitot_adapter_supervisor.py", +) +supervisor = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(supervisor) + + +def _fixture(root: Path) -> tuple[Path, list[str]]: + manifest = copy.deepcopy(supervisor.load_manifest(ROOT)) + manifest_path = root / supervisor.MANIFEST + manifest_path.parent.mkdir(parents=True) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + tests = root / "labs/15-pitot/tests" + tests.mkdir(parents=True) + ids = [agent["id"] for agent in manifest["agents"]] + for agent_id in ids: + (tests / f"e2e_{agent_id}_cli_test.sh").write_text("#!/usr/bin/env bash\n") + readme = root / supervisor.README + readme.parent.mkdir(parents=True) + readme.write_text( + f"# Pitot\n\n{supervisor.README_START}\nplaceholder\n{supervisor.README_END}\n", + encoding="utf-8", + ) + (root / ".github/workflows").mkdir(parents=True) + reusable = root / supervisor.REUSABLE_WORKFLOW + reusable.write_text( + "platform:\n description: Canonical Pitot platform identifier\n" + "runner:\n description: GitHub-hosted runner selected by the supervisor\n" + "runs-on: ${{ inputs.runner }}\n" + '--agent "${{ inputs.agent }}"\n' + '--platform "${{ inputs.platform }}"\n' + "name: pitot-e2e-${{ inputs.agent }}-${{ inputs.platform }}\n" + ) + supervisor.render(root) + return manifest_path, ids + + +class AdapterSupervisorTests(unittest.TestCase): + def test_real_repository_contract_is_complete(self): + self.assertEqual(supervisor.contract_errors(ROOT), []) + + def test_matrix_contains_every_agent_on_every_platform(self): + manifest = supervisor.load_manifest(ROOT) + matrix = supervisor.matrix(manifest) + self.assertEqual(len(matrix), 18) + self.assertEqual( + {(item["agent"], item["platform"]) for item in matrix if item["agent"] == "kimi"}, + {("kimi", "ubuntu"), ("kimi", "macos"), ("kimi", "windows")}, + ) + + def test_missing_and_extra_inventory_entries_fail(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + manifest_path, ids = _fixture(root) + value = json.loads(manifest_path.read_text()) + value["agents"] = value["agents"][1:] + manifest_path.write_text(json.dumps(value)) + errors = supervisor.contract_errors(root, ids) + self.assertTrue(any("missing from manifest" in error for error in errors)) + + value["agents"].append({"id": "phantom", "label": "Phantom"}) + manifest_path.write_text(json.dumps(value)) + errors = supervisor.contract_errors(root, ids) + self.assertTrue(any("not built-in adapters" in error for error in errors)) + + def test_duplicate_id_and_label_fail(self): + manifest = copy.deepcopy(supervisor.load_manifest(ROOT)) + manifest["agents"].append(copy.deepcopy(manifest["agents"][0])) + with self.assertRaisesRegex(supervisor.ContractError, "ids must be unique"): + supervisor.validate_manifest(manifest) + manifest = copy.deepcopy(supervisor.load_manifest(ROOT)) + manifest["agents"][1]["label"] = manifest["agents"][0]["label"] + with self.assertRaisesRegex(supervisor.ContractError, "labels must be unique"): + supervisor.validate_manifest(manifest) + + def test_missing_script_fails(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + _, ids = _fixture(root) + (root / "labs/15-pitot/tests/e2e_kimi_cli_test.sh").unlink() + self.assertTrue(any("adapter kimi is missing" in error for error in supervisor.contract_errors(root, ids))) + + def test_readme_and_workflow_drift_fail(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + _, ids = _fixture(root) + readme = root / supervisor.README + readme.write_text(readme.read_text().replace("Every supervised", "Some supervised")) + self.assertTrue(any("README.md" in error for error in supervisor.contract_errors(root, ids))) + + supervisor.render(root) + workflow = root / supervisor.UNIFIED_WORKFLOW + workflow.write_text(workflow.read_text() + "# drift\n") + self.assertTrue(any("pitot-e2e.yml" in error for error in supervisor.contract_errors(root, ids))) + + def test_reusable_workflow_cannot_select_its_own_platforms(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + _, ids = _fixture(root) + workflow = root / supervisor.REUSABLE_WORKFLOW + workflow.write_text(workflow.read_text() + "matrix.platform\n") + self.assertTrue(any("escaped supervisor" in error for error in supervisor.contract_errors(root, ids))) + + +if __name__ == "__main__": + unittest.main() diff --git a/labs/15-pitot/tests/test_e2e_reporting.py b/labs/15-pitot/tests/test_e2e_reporting.py index f64d204b0..2a2dad60b 100644 --- a/labs/15-pitot/tests/test_e2e_reporting.py +++ b/labs/15-pitot/tests/test_e2e_reporting.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import json from pathlib import Path import subprocess import sys @@ -22,6 +23,10 @@ def _load(name: str, path: Path): runner = _load("run_e2e_report", ROOT / "labs/15-pitot/tests/run_e2e_report.py") reporter = _load("pitot_e2e_report", ROOT / ".github/scripts/pitot_e2e_report.py") +supervisor = _load( + "pitot_adapter_supervisor", + ROOT / "labs/15-pitot/scripts/pitot_adapter_supervisor.py", +) def _result(agent="claude", platform="ubuntu", status="pass", mode="real_cli"): @@ -129,13 +134,48 @@ def test_artifact_redirect_rejects_http_downgrade(self): def test_workflow_matrix_covers_every_reported_platform(self): workflow = (ROOT / ".github/workflows/pitot-e2e-agent.yml").read_text() - runners = {"ubuntu": "ubuntu-latest", "macos": "macos-latest", "windows": "windows-latest"} + unified = (ROOT / ".github/workflows/pitot-e2e.yml").read_text() self.assertEqual(set(reporter.PLATFORMS), set(runner.PLATFORMS)) - for platform, image in runners.items(): - self.assertIn(f"runner: {image}\n platform: {platform}", workflow) + generated = supervisor.matrix(supervisor.load_manifest(ROOT)) + self.assertEqual(len(generated), len(reporter.AGENTS) * len(reporter.PLATFORMS)) + self.assertEqual( + {(item["agent"], item["platform"]) for item in generated}, + {(agent, platform) for agent in reporter.AGENTS for platform in reporter.PLATFORMS}, + ) + self.assertIn("runs-on: ${{ inputs.runner }}", workflow) + self.assertIn('--platform "${{ inputs.platform }}"', workflow) self.assertIn('bash_path="$(cygpath -m "$bash_path")"', workflow) self.assertIn('-- "$PITOT_BASH"', workflow) self.assertNotIn('-- bash "labs/15-pitot/tests/e2e_', workflow) + self.assertIn("fromJSON(needs.inventory.outputs.matrix)", unified) + self.assertIn("name: pitot-e2e-inventory", unified) + + def test_inventory_schema_rejects_identity_injection(self): + inventory = json.loads((ROOT / "labs/15-pitot/adapter-verification.json").read_text()) + self.assertIs(reporter.validate_inventory(inventory), inventory) + inventory["agents"][0]["label"] = "Claude | @everyone" + with self.assertRaisesRegex(ValueError, "label"): + reporter.validate_inventory(inventory) + + def test_missing_artifacts_fail_every_inventory_cell(self): + inventory = reporter.validate_inventory( + json.loads((ROOT / "labs/15-pitot/adapter-verification.json").read_text()) + ) + results = reporter.collect_results( + None, + {}, + inventory, + "a" * 40, + "https://github.com/operatorstack/intelligence-flow/actions/runs/1", + ) + self.assertEqual(len(results), 6) + self.assertTrue( + all( + result["status"] == "fail" + for platforms in results.values() + for result in platforms.values() + ) + ) def test_validates_exact_artifact_schema(self): value = _result() @@ -201,7 +241,7 @@ def test_aggregate_requires_all_platforms(self): def test_rendered_comment_is_sticky_and_shows_modes(self): results = { agent: {platform: None for platform in reporter.PLATFORMS} - for agent in reporter.WORKFLOWS + for agent in reporter.AGENTS } results["claude"] = { "ubuntu": _result(), @@ -215,6 +255,7 @@ def test_rendered_comment_is_sticky_and_shows_modes(self): self.assertIn("Windows", body) self.assertIn("All platforms are required", body) self.assertIn("OpenCode", body) + self.assertIn("Kimi Code", body) if __name__ == "__main__":