diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index a96e047..ddcc434 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -88,6 +88,7 @@ jobs: id: catalog env: CATALOG_SOURCE_COMMIT: ${{ steps.source.outputs.commit }} + GH_TOKEN: ${{ github.token }} run: python3 scripts/plugin_catalog_publication.py --expected-head "$CATALOG_SOURCE_COMMIT" - name: Merge catalog proposal diff --git a/docs/plugin-catalog.md b/docs/plugin-catalog.md index 0bab59b..5f91537 100644 --- a/docs/plugin-catalog.md +++ b/docs/plugin-catalog.md @@ -161,8 +161,11 @@ verifies its seed before requesting release metadata. It always scans every admi missed notifications and replaced pending runs are repaired by the next successful reconciliation. After generation, the workflow validates and tests the complete site. The publication helper checks -the catalog successor, exact built index bytes and unchanged source, then commits only the new index -as a child of the checked-out commit. It pushes that commit to `automation/plugin-catalog`, +the catalog successor, exact built index bytes and unchanged source, then computes a commit tree +that changes only the new index on top of the checked-out commit. Because `master` accepts only +signed commits, the helper has GitHub create that blob, tree and commit through the REST API with +the job token, which GitHub signs. It requires GitHub's objects to match the locally computed tree +and parent and the commit to be verified. It then points `automation/plugin-catalog` at that commit, replacing any earlier unmerged proposal, and `master` changes only through the resulting pull request. The workflow opens or updates the pull request, enables squash auto-merge and dispatches Website CI on the branch, because pull requests opened with the job token do not trigger workflows. @@ -178,8 +181,8 @@ and can deploy it without incrementing generation. A local generated file alone anything. Publisher tests also run in read-only website CI, without contacting release sources. The repository allows auto-merge and lets GitHub Actions create pull requests. The central build job -uses its short-lived GitHub job token with contents, pull-request and Actions write access to push -the proposal branch, open and auto-merge its pull request and dispatch Website CI; the deployment +uses its short-lived GitHub job token with contents, pull-request and Actions write access to create +the signed proposal, open and auto-merge its pull request and dispatch Website CI; the deployment job has Pages and identity-token permissions. Plugin notification senders should invoke `pages.yml` on `master` using `workflow_dispatch`, with Actions write access scoped to this receiving repository. They do not need website Contents write access. Notification payloads provide no @@ -249,6 +252,6 @@ are generation failure bounds, not permission to truncate the catalog or omit ol Tests cover provenance rejection, immutable identities, explicit withdrawal, failed writes, concurrent publishers, duplicate events, invalid/oversized/truncated input, bounded retries, manifest/archive agreement, safe manifest reads, credential-free redirects, committed generation -continuity, proposal replacement, merge verification, concurrent source pushes and deployment retry. -Network discovery in the host remains a separate consumer; installation must retain its exact GitHub -revalidation. +continuity, signed proposal creation, proposal replacement, merge verification, concurrent source +pushes and deployment retry. Network discovery in the host remains a separate consumer; installation +must retain its exact GitHub revalidation. diff --git a/scripts/plugin_catalog_publication.py b/scripts/plugin_catalog_publication.py index c5e4016..872ca68 100644 --- a/scripts/plugin_catalog_publication.py +++ b/scripts/plugin_catalog_publication.py @@ -2,12 +2,16 @@ """Propose one verified catalog for merge and confirm the merged source before Pages upload.""" import argparse +import base64 +import json import os from pathlib import Path import re import subprocess import sys import tempfile +import urllib.error +import urllib.request import plugin_catalog as catalog @@ -18,6 +22,8 @@ BRANCH = "refs/heads/master" PROPOSAL = "refs/heads/automation/plugin-catalog" REMOTE = "https://github.com/computer-mcp/computer-mcp.github.io" +API = "https://api.github.com/repos/computer-mcp/computer-mcp.github.io" +MAX_RESPONSE = 1024 * 1024 def git(root, *arguments, data=None, environment=None): @@ -28,6 +34,30 @@ def git(root, *arguments, data=None, environment=None): return result.stdout +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *arguments): + return None + + +def rest(method, path, body): + token = os.environ.get("GH_TOKEN") + catalog.require(bool(token), "Catalog proposal requires the workflow token") + request = urllib.request.Request(API + path, method=method, data=json.dumps(body).encode(), headers={ + "Accept": "application/vnd.github+json", "Authorization": "Bearer " + token, + "Content-Type": "application/json", "User-Agent": "computer-mcp-plugin-catalog/1", + "X-GitHub-Api-Version": "2022-11-28"}) + try: + with urllib.request.build_opener(NoRedirect()).open(request, timeout=60) as response: + data = response.read(MAX_RESPONSE + 1) + except urllib.error.HTTPError as error: + error.close() + raise catalog.CatalogError(f"GitHub HTTP {error.code}; Pages must not deploy") from None + catalog.require(len(data) <= MAX_RESPONSE, "GitHub response exceeds catalog budget") + value = catalog.decode_json(data, MAX_RESPONSE) + catalog.require(isinstance(value, dict), "Invalid GitHub response") + return value + + def remote_head(root, ref=BRANCH): fields = git(root, "ls-remote", "--exit-code", "origin", ref).decode().split() catalog.require(len(fields) == 2 and fields[1] == ref @@ -76,23 +106,40 @@ def propose_commit(root, expected_head): "Remote master changed; reconcile from the current source before deployment") if current == previous: return expected_head - # Construct the tree from the verified bytes and parent, without touching the caller's index - # or committing unrelated working-tree files. Each run replaces any earlier unmerged proposal. + # Compute the expected tree from the verified bytes and parent without touching the caller's + # index. GitHub then creates the same objects, because master accepts only signed commits and + # GitHub signs commits that the job token creates through its API. Each run replaces any + # earlier unmerged proposal. blob = git(root, "hash-object", "-w", "--stdin", data=data).decode().strip() with tempfile.TemporaryDirectory(prefix="computer-mcp-catalog-index-") as directory: environment = {**os.environ, "GIT_INDEX_FILE": str(Path(directory) / "index")} git(root, "read-tree", expected_head, environment=environment) git(root, "update-index", "--add", "--cacheinfo", f"100644,{blob},{INDEX}", environment=environment) tree = git(root, "write-tree", environment=environment).decode().strip() - identity = {**os.environ, "GIT_AUTHOR_NAME": "github-actions[bot]", - "GIT_COMMITTER_NAME": "github-actions[bot]", - "GIT_AUTHOR_EMAIL": "41898282+github-actions[bot]@users.noreply.github.com", - "GIT_COMMITTER_EMAIL": "41898282+github-actions[bot]@users.noreply.github.com"} - commit = git(root, "-c", "commit.gpgsign=false", "commit-tree", tree, "-p", expected_head, - "-m", f"Publish plugin catalog generation {current['generation']}", - environment=identity).decode().strip() - git(root, "push", "--porcelain", "--force", "origin", commit + ":" + PROPOSAL) - catalog.require(remote_head(root, PROPOSAL) == commit, "Catalog proposal changed before merge") + base = git(root, "rev-parse", expected_head + "^{tree}").decode().strip() + created = rest("POST", "/git/blobs", {"content": base64.b64encode(data).decode(), "encoding": "base64"}) + catalog.require(created.get("sha") == blob, "GitHub stored different catalog bytes") + created = rest("POST", "/git/trees", {"base_tree": base, "tree": [ + {"path": INDEX, "mode": "100644", "type": "blob", "sha": blob}]}) + catalog.require(created.get("sha") == tree, "GitHub built a different catalog tree") + created = rest("POST", "/git/commits", {"message": f"Publish plugin catalog generation {current['generation']}", + "tree": tree, "parents": [expected_head]}) + commit = created.get("sha") + catalog.require(isinstance(commit, str) and re.fullmatch(r"[0-9a-f]{40}", commit) is not None, + "Invalid catalog proposal identity") + verification = created.get("verification") + catalog.require(isinstance(verification, dict) and verification.get("verified") is True, + "GitHub did not sign the catalog proposal") + if git(root, "ls-remote", "origin", PROPOSAL).strip(): + rest("PATCH", "/git/" + PROPOSAL, {"sha": commit, "force": True}) + else: + rest("POST", "/git/refs", {"ref": PROPOSAL, "sha": commit}) + git(root, "fetch", "--no-tags", "origin", PROPOSAL) + catalog.require(git(root, "rev-parse", "FETCH_HEAD").decode().strip() == commit, + "Catalog proposal changed before merge") + catalog.require(git(root, "rev-list", "--parents", "-n", "1", commit).decode().split() == [commit, expected_head] + and git(root, "rev-parse", commit + "^{tree}").decode().strip() == tree, + "GitHub created a different catalog proposal") return commit diff --git a/scripts/test_plugin_catalog_publication.py b/scripts/test_plugin_catalog_publication.py index 7f35071..91c2b67 100644 --- a/scripts/test_plugin_catalog_publication.py +++ b/scripts/test_plugin_catalog_publication.py @@ -1,14 +1,56 @@ +import base64 import hashlib import json +import os from pathlib import Path import subprocess import tempfile import unittest +from unittest.mock import patch import plugin_catalog as catalog import plugin_catalog_publication as publication from test_plugin_catalog import NOW, Source +REST = publication.rest + + +class RemoteAPI: + """Serve the repository's Git data API from the bare fixture remote.""" + + def __init__(self, remote): + self.remote, self.calls, self.verified = remote, [], True + + def git(self, *arguments, data=None, environment=None): + return subprocess.run(["git", "-C", str(self.remote), *arguments], input=data, check=True, + capture_output=True, timeout=10, env=environment).stdout.decode().strip() + + def __call__(self, method, path, body): + self.calls.append((method, path)) + if (method, path) == ("POST", "/git/blobs"): + return {"sha": self.git("hash-object", "-w", "--stdin", data=base64.b64decode(body["content"]))} + if (method, path) == ("POST", "/git/trees"): + with tempfile.TemporaryDirectory() as directory: + environment = {**os.environ, "GIT_INDEX_FILE": str(Path(directory) / "index")} + self.git("read-tree", body["base_tree"], environment=environment) + for entry in body["tree"]: + self.git("update-index", "--add", "--cacheinfo", f"{entry['mode']},{entry['sha']},{entry['path']}", + environment=environment) + return {"sha": self.git("write-tree", environment=environment)} + if (method, path) == ("POST", "/git/commits"): + parents = [argument for parent in body["parents"] for argument in ("-p", parent)] + commit = self.git("-c", "user.name=GitHub", "-c", "user.email=github@example.invalid", + "-c", "commit.gpgsign=false", "commit-tree", body["tree"], *parents, + "-m", body["message"]) + return {"sha": commit, "verification": {"verified": self.verified}} + if (method, path) == ("POST", "/git/refs"): + self.git("update-ref", body["ref"], body["sha"], "0" * 40) + return {} + if method == "PATCH" and path.startswith("/git/refs/heads/") and body["force"] is True: + self.git("update-ref", path.removeprefix("/git/"), body["sha"]) + return {} + raise AssertionError(f"Unexpected API request {method} {path}") + class PublicationTests(unittest.TestCase): def setUp(self): @@ -17,6 +59,10 @@ def setUp(self): base = Path(self.temporary.name) self.root, self.remote = base / "source", base / "remote.git" self.root.mkdir() + self.api = RemoteAPI(self.remote) + rest = patch.object(publication, "rest", self.api) + rest.start() + self.addCleanup(rest.stop) self.command(base, "init", "--bare", str(self.remote)) self.command(self.root, "init", "-b", "master") self.command(self.root, "remote", "add", "origin", str(self.remote)) @@ -98,6 +144,7 @@ def test_duplicate_reconciliation_keeps_commit_and_generation(self): self.assertEqual(publication.propose_commit(self.root, self.head), self.head) self.assertEqual(publication.remote_head(self.root), self.head) self.assertEqual(self.proposals(), []) + self.assertEqual(self.api.calls, []) def test_new_proposal_replaces_unmerged_proposal(self): stale = self.advance_remote() @@ -105,6 +152,18 @@ def test_new_proposal_replaces_unmerged_proposal(self): self.command(self.remote, "update-ref", publication.BRANCH, self.head) commit = publication.propose_commit(self.root, self.head) self.assertEqual(self.proposals(), [commit]) + self.assertIn(("PATCH", "/git/" + publication.PROPOSAL), self.api.calls) + + def test_unsigned_proposal_is_not_published(self): + self.api.verified = False + with self.assertRaisesRegex(catalog.CatalogError, "sign"): + publication.propose_commit(self.root, self.head) + self.assertEqual(self.proposals(), []) + self.assertEqual(publication.remote_head(self.root), self.head) + + def test_proposal_requires_the_workflow_token(self): + with patch.dict(os.environ, {"GH_TOKEN": ""}), self.assertRaisesRegex(catalog.CatalogError, "token"): + REST("POST", "/git/blobs", {}) def test_squash_merged_proposal_is_the_deployable_source(self): commit = publication.propose_commit(self.root, self.head)