Verify official GitHub targets during page service errors

.github/workflows/ci.yml

@@ -49,6 +49,8 @@ jobs:
         run: npm run build
 
       - name: Validate links
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: npm run check:links
 
       - name: Install browser

.github/workflows/pages.yml

@@ -64,6 +64,8 @@ jobs:
         run: npm ci
 
       - name: Verify website
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: |
           npm run release:check
           npm run catalog:check-policy

README.md

@@ -36,6 +36,14 @@ npm test
 The production build is written to `dist/`. GitHub Pages deploys only that artifact from the
 official repository's `main` branch after validation and catalog reconciliation.
 
+Link validation retries transient HTTP errors with bounded concurrency. When an official GitHub
+repository, file, directory, release or security-policy link returns a server error, it verifies the
+corresponding public target through the GitHub API. Client errors, missing targets and API failures
+still fail validation. `GH_TOKEN` or `GITHUB_TOKEN` can provide API authentication; the token is
+sent only to the GitHub API. Workflows use their repository token for read-only API requests.
+Content verification covers `main`, `master` and full commit hashes; other content refs retain HTTP
+validation.
+
 `public/release.json` binds the deployed site to the product's delivered commit and release tag. The
 private website package version describes this build project, not the product version. After the
 main repository publishes an accepted release, run `npm run release:update -- vX.Y.Z` to import its

package.json

@@ -13,8 +13,8 @@
     "format": "prettier --write .",
     "format:check": "prettier --check .",
     "check:html": "html-validate index.html",
-    "check:links": "linkinator dist --recurse --skip '^https://computer-mcp.github.io'",
-    "test": "npm run catalog:test && node --test scripts/release.test.mjs && playwright test",
+    "check:links": "node scripts/check-links.mjs",
+    "test": "npm run catalog:test && node --test scripts/release.test.mjs scripts/check-links.test.mjs && playwright test",
     "catalog:test": "python3 -m unittest discover -s scripts -p 'test_plugin_catalog*.py'",
     "catalog:check": "python3 scripts/plugin_catalog.py check",
     "catalog:check-policy": "python3 scripts/plugin_catalog.py check-policy",

scripts/check-links.mjs

@@ -0,0 +1,138 @@
+import { LinkChecker, LinkState } from "linkinator";
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+
+export function githubTarget(link) {
+  const url = new URL(link);
+  if (url.origin !== "https://github.com" || url.search || url.username || url.password)
+    return null;
+  const parts = url.pathname.replace(/\/$/, "").split("/").slice(1);
+  if (parts[0] !== "computer-mcp" || !/^[\w.-]+$/.test(parts[1] ?? "")) return null;
+  const repository = `${parts[0]}/${parts[1]}`;
+  if (parts.length === 2) return { repository, kind: "repository" };
+  if (parts.length === 3 && parts[2] === "releases") return { repository, kind: "releases" };
+  if (parts.length === 4 && parts[2] === "releases" && parts[3] === "latest")
+    return { repository, kind: "latest" };
+  if (parts.length === 4 && parts[2] === "security" && parts[3] === "policy")
+    return { repository, kind: "policy" };
+  if (
+    parts.length < 5 ||
+    !["blob", "tree"].includes(parts[2]) ||
+    !/^(main|master|[0-9a-f]{40})$/.test(parts[3])
+  )
+    return null;
+  const path = parts.slice(4).map((part) => decodeURIComponent(part));
+  if (path.some((part) => !part || part === "." || part === ".." || /[\\/]/.test(part)))
+    return null;
+  return {
+    repository,
+    kind: parts[2] === "blob" ? "file" : "directory",
+    ref: parts[3],
+    path: path.map(encodeURIComponent).join("/"),
+  };
+}
+
+export async function githubJSON(endpoint) {
+  const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN;
+  const response = await fetch(`https://api.github.com/${endpoint}`, {
+    method: "GET",
+    headers: {
+      Accept: "application/vnd.github+json",
+      "User-Agent": "computer-mcp-website-link-check",
+      ...(token ? { Authorization: `Bearer ${token}` } : {}),
+    },
+    redirect: "error",
+    signal: AbortSignal.timeout(30_000),
+  });
+  if (!response.ok) throw new Error(`GitHub API returned HTTP ${response.status}`);
+  return response.json();
+}
+
+export async function resolveFailures(links, request = githubJSON) {
+  const requests = new Map();
+  const get = (endpoint) => {
+    if (!requests.has(endpoint))
+      requests.set(
+        endpoint,
+        Promise.resolve().then(() => request(endpoint)),
+      );
+    return requests.get(endpoint);
+  };
+  const failures = [];
+  const verified = [];
+  for (const link of links.filter((item) => item.state === LinkState.BROKEN)) {
+    try {
+      const target = link.status >= 500 && link.status <= 599 ? githubTarget(link.url) : null;
+      if (!target) {
+        failures.push(link);
+        continue;
+      }
+      const base = `repos/${target.repository}`;
+      const repository = await get(base);
+      if (
+        repository.private !== false ||
+        repository.full_name?.toLowerCase() !== target.repository.toLowerCase()
+      )
+        throw new Error("The target is not the expected public repository");
+      if (target.kind !== "repository") {
+        const endpoint = {
+          releases: `${base}/releases`,
+          latest: `${base}/releases/latest`,
+          policy: `${base}/contents/SECURITY.md?ref=${encodeURIComponent(repository.default_branch)}`,
+          file: `${base}/contents/${target.path}?ref=${target.ref}`,
+          directory: `${base}/contents/${target.path}?ref=${target.ref}`,
+        }[target.kind];
+        const content = await get(endpoint);
+        if (target.kind === "directory" || target.kind === "releases") {
+          if (!Array.isArray(content)) throw new Error("The target is not the expected listing");
+        } else if (target.kind === "latest") {
+          if (
+            content.draft !== false ||
+            content.prerelease !== false ||
+            !content.html_url?.startsWith(`https://github.com/${target.repository}/releases/tag/`)
+          )
+            throw new Error("The target has no public stable release");
+        } else if (content.type !== "file") {
+          throw new Error("The target is not a file");
+        }
+      }
+      verified.push(link.url);
+    } catch (error) {
+      failures.push({ ...link, verificationError: error.message });
+    }
+  }
+  return { failures, verified };
+}
+
+async function main() {
+  const checker = new LinkChecker();
+  checker.on("retry", ({ url, secondsUntilRetry }) =>
+    console.log(`Retrying ${url} in ${secondsUntilRetry} seconds`),
+  );
+  const result = await checker.check({
+    path: "dist",
+    recurse: true,
+    linksToSkip: ["^https://computer-mcp.github.io"],
+    concurrency: 5,
+    retryErrors: true,
+    retryErrorsCount: 3,
+    timeout: 30_000,
+  });
+  const { failures, verified } = await resolveFailures(result.links);
+  for (const url of verified) console.log(`Verified through GitHub API: ${url}`);
+  for (const link of failures)
+    console.error(
+      `[${link.status ?? "failed"}] ${link.url}: ${link.verificationError ?? "link failed"}`,
+    );
+  if (failures.length || (!result.passed && !verified.length)) {
+    process.exitCode = 1;
+    return;
+  }
+  console.log(`All ${result.links.length} links verified (${verified.length} through GitHub API).`);
+}
+
+if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url)
+  main().catch((error) => {
+    console.error(error.message);
+    process.exitCode = 1;
+  });

scripts/check-links.test.mjs

@@ -0,0 +1,156 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { githubJSON, githubTarget, resolveFailures } from "./check-links.mjs";
+
+const repository = {
+  private: false,
+  full_name: "computer-mcp/computer-mcp",
+  default_branch: "master",
+};
+const fileURL =
+  "https://github.com/computer-mcp/computer-mcp/blob/master/Documentation/Reference/QuickStart.md";
+const broken = (url, status = 503) => ({ url, status, state: "BROKEN" });
+
+test("a GitHub service error requires a public repository and the exact file", async () => {
+  const endpoints = [];
+  const result = await resolveFailures([broken(fileURL)], async (endpoint) => {
+    endpoints.push(endpoint);
+    return endpoint === "repos/computer-mcp/computer-mcp" ? repository : { type: "file" };
+  });
+  assert.deepEqual(endpoints, [
+    "repos/computer-mcp/computer-mcp",
+    "repos/computer-mcp/computer-mcp/contents/Documentation/Reference/QuickStart.md?ref=master",
+  ]);
+  assert.deepEqual(result, { failures: [], verified: [fileURL] });
+});
+
+test("a missing file or failed API leaves the link failed", async () => {
+  const result = await resolveFailures([broken(fileURL)], async (endpoint) => {
+    if (endpoint === "repos/computer-mcp/computer-mcp") return repository;
+    throw new Error("HTTP 404");
+  });
+  assert.equal(result.failures.length, 1);
+  assert.deepEqual(result.verified, []);
+});
+
+test("file and directory routes retain their target type", async () => {
+  const directoryURL = "https://github.com/computer-mcp/computer-mcp/tree/master/Documentation";
+  const result = await resolveFailures(
+    [broken(fileURL), broken(directoryURL)],
+    async (endpoint) => {
+      return endpoint === "repos/computer-mcp/computer-mcp" ? repository : [];
+    },
+  );
+  assert.equal(result.failures[0].url, fileURL);
+  assert.deepEqual(result.verified, [directoryURL]);
+});
+
+test("private or different repositories cannot satisfy public links", async () => {
+  for (const value of [
+    { ...repository, private: true },
+    { ...repository, full_name: "computer-mcp/another-repository" },
+  ]) {
+    let calls = 0;
+    const result = await resolveFailures([broken(fileURL)], async () => {
+      calls += 1;
+      return value;
+    });
+    assert.equal(calls, 1);
+    assert.equal(result.failures.length, 1);
+  }
+});
+
+test("client errors, other hosts and unsupported GitHub routes use their HTTP result", async () => {
+  const links = [
+    broken(fileURL, 404),
+    broken(fileURL, 403),
+    broken(fileURL, 0),
+    broken("https://example.com/document"),
+    broken("https://github.com/another-owner/repository/blob/master/document.md"),
+    broken("https://github.com/computer-mcp/computer-mcp/issues/1"),
+  ];
+  const result = await resolveFailures(links, () => assert.fail("Unexpected GitHub API request"));
+  assert.deepEqual(result.failures, links);
+  assert.deepEqual(result.verified, []);
+});
+
+test("encoded path separators and ambiguous branch routes retain HTTP validation", () => {
+  assert.equal(
+    githubTarget("https://github.com/computer-mcp/computer-mcp/blob/master/a%2Fb.md"),
+    null,
+  );
+  assert.equal(
+    githubTarget("https://github.com/computer-mcp/computer-mcp/blob/feature/branch/a.md"),
+    null,
+  );
+  assert.equal(
+    githubTarget("https://github.com/computer-mcp/computer-mcp/blob/master/a.md?raw=true"),
+    null,
+  );
+});
+
+test("repository metadata is reused and latest releases must be public and stable", async () => {
+  const latest = "https://github.com/computer-mcp/computer-mcp/releases/latest";
+  let metadataCalls = 0;
+  const result = await resolveFailures([broken(latest), broken(fileURL)], async (endpoint) => {
+    if (endpoint === "repos/computer-mcp/computer-mcp") {
+      metadataCalls += 1;
+      return repository;
+    }
+    if (endpoint.endsWith("/latest")) return { draft: true, prerelease: false };
+    return { type: "file" };
+  });
+  assert.equal(metadataCalls, 1);
+  assert.equal(result.failures[0].url, latest);
+  assert.deepEqual(result.verified, [fileURL]);
+});
+
+test("public repository, release and policy routes require their corresponding API target", async () => {
+  const urls = [
+    "https://github.com/computer-mcp/computer-mcp",
+    "https://github.com/computer-mcp/computer-mcp/releases",
+    "https://github.com/computer-mcp/computer-mcp/releases/latest",
+    "https://github.com/computer-mcp/computer-mcp/security/policy",
+  ];
+  const responses = new Map([
+    ["repos/computer-mcp/computer-mcp", repository],
+    ["repos/computer-mcp/computer-mcp/releases", []],
+    [
+      "repos/computer-mcp/computer-mcp/releases/latest",
+      {
+        draft: false,
+        prerelease: false,
+        html_url: "https://github.com/computer-mcp/computer-mcp/releases/tag/example-tag",
+      },
+    ],
+    ["repos/computer-mcp/computer-mcp/contents/SECURITY.md?ref=master", { type: "file" }],
+  ]);
+  const result = await resolveFailures(
+    urls.map((url) => broken(url)),
+    async (endpoint) => {
+      assert.ok(responses.has(endpoint));
+      return responses.get(endpoint);
+    },
+  );
+  assert.deepEqual(result, { failures: [], verified: urls });
+});
+
+test("API credentials are confined to the fixed GitHub API origin", async () => {
+  const originalToken = process.env.GH_TOKEN;
+  const originalFetch = globalThis.fetch;
+  process.env.GH_TOKEN = "test-credential";
+  globalThis.fetch = async (url, options) => {
+    assert.equal(new URL(url).origin, "https://api.github.com");
+    assert.equal(options.method, "GET");
+    assert.equal(options.headers.Authorization, "Bearer test-credential");
+    assert.equal(options.redirect, "error");
+    return { ok: true, json: async () => repository };
+  };
+  try {
+    assert.deepEqual(await githubJSON("repos/computer-mcp/computer-mcp"), repository);
+  } finally {
+    globalThis.fetch = originalFetch;
+    if (originalToken === undefined) delete process.env.GH_TOKEN;
+    else process.env.GH_TOKEN = originalToken;
+  }
+});