diff --git a/.agents/resume b/.agents/resume new file mode 100755 index 000000000..24e302916 --- /dev/null +++ b/.agents/resume @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Runs on every orb wake. Nothing to authenticate or reconnect; just confirm the +# toolchain from .agents/setup still resolves so a broken snapshot fails loudly. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +eval "$("$HOME/.local/bin/mise" -C "$repo_root" env -s bash)" +node --version +pnpm --version diff --git a/.agents/setup b/.agents/setup new file mode 100755 index 000000000..5ca196193 --- /dev/null +++ b/.agents/setup @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Prepares a fresh Amp orb: the Node and pnpm versions pinned in .tool-versions, +# then the pnpm dependencies. Idempotent: Amp reruns it on stale snapshots. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +step() { printf '\n==> %s (%s)\n' "$1" "$(date +%T)"; } + +mise_bin="$HOME/.local/bin/mise" +step "Install mise" +if [ ! -x "$mise_bin" ]; then + curl -fsSL https://mise.run | sh +fi + +step "Install pinned toolchain from .tool-versions" +"$mise_bin" install --yes +# The installer cannot change this running script's environment. +eval "$("$mise_bin" env -s bash)" + +step "Expose the toolchain to login shells" +profile_marker="# sourcegraph/docs: pinned Node and pnpm from mise" +if ! grep -Fqx "$profile_marker" "$HOME/.bash_profile" 2>/dev/null; then + cat >>"$HOME/.bash_profile" <"$HOME/.config/amp/AGENTS.md" <<'EOF' +# Inside the Orb + +- Node and pnpm come from mise (`.tool-versions`); `.agents/setup` puts them on PATH for login shells. Use `pnpm`, not `npm`. +- Dev server: `amp orb services ensure` starts `pnpm dev` from `.amp/services.yaml` and returns a portal link. +EOF + +step "Done: node $(node --version), pnpm $(pnpm --version)" diff --git a/.agents/skills/publishing-docs-versions/SKILL.md b/.agents/skills/publishing-docs-versions/SKILL.md new file mode 100644 index 000000000..55a5a7ac4 --- /dev/null +++ b/.agents/skills/publishing-docs-versions/SKILL.md @@ -0,0 +1,142 @@ +--- +name: publishing-docs-versions +description: "Archives Sourcegraph docs release branches and updates latest-version metadata. Use when a new Sourcegraph docs version is released, when creating docs legacy branches, or when updating DOCS_LATEST_VERSION and previous-version lists." +--- + +# Publishing Sourcegraph Docs Versions + +Use this skill for the Sourcegraph docs repo release-version workflow: cutting legacy branches in the `legacy` remote and updating `origin` so the newest version becomes `latest`. + +## Repository conventions + +- Remotes: + - `origin` = `sourcegraph/docs` + - `legacy` = `sourcegraph/docs-legacy-versions` +- 7.x legacy branches use underscores: `v7_0`, `v7_1`, `v7_2`, etc. +- Legacy branches are pushed directly to the `legacy` remote. +- `origin/main` is protected; direct pushes are rejected. Make a branch on `origin` and open a PR. +- The files that control latest/previous versions are: + - `docs.config.js` + - `src/data/versions.ts` + - `docs/legacy.mdx` +- `src/data/versions.ts` on `origin/main` is the canonical dropdown list. The + current site exposes it through `/docs/api/versions`, and legacy selectors + load that manifest at runtime. +- A legacy branch's bundled `src/data/versions.ts` is only a fallback. Its first + entry identifies the archived site and must not have the `latest` label. + +## Standard workflow for a new release + +For a new release `X.Y`: + +1. Archive the previous docs version `P.Q` in the `legacy` remote. +2. Ensure the legacy branch’s own config says it is version `P.Q`, marks it as + selected rather than latest, and lists older fallback versions only. +3. Update `origin` so `X.Y` is latest and `P.Q` appears as a previous version. + +Example: when 7.4 is released, archive 7.3 as `legacy/v7_3`, then update `origin` with `DOCS_LATEST_VERSION: '7.4'` and add 7.3 to previous-version lists. + +## Finding the cut point for a legacy branch + +Always inspect history instead of guessing: + +```bash +git fetch --all --prune +git log --date=short --pretty=format:'%h %ad %s' origin/main --since='' --until='' --reverse +git for-each-ref refs/remotes/legacy --format='%(refname:short) %(objectname:short) %(committerdate:short) %(subject)' | sort -V | tail -50 +``` + +Pick the commit the user considers the correct snapshot for the previous version. If the user gives a release date but not a commit, use the repo’s existing pattern: + +- Choose a hand-picked `origin/main` commit near the release cut. +- Prefer the last relevant docs commit on the release date when obvious. +- If no version-bump commit exists yet and the user wants the just-current docs archived, use the current `origin/main` tip. + +For reference, recent historical decisions: + +- `legacy/v7_0` was cut from `a5d80e5e` and then got empty no-op marker commits. +- `legacy/v7_1` was cut from `6b8ae421` for the Apr 24 release-date snapshot. +- `legacy/v7_2` was cut from `fccceac6`, then updated with a version metadata commit. + +## Creating and pushing the legacy branch + +Create the branch from the selected cut commit, add an empty marker commit, and push to `legacy`: + +```bash +git switch --detach +git switch -c vX_Y +git commit --allow-empty -m "Branch for docs version X.Y" +git push -u legacy vX_Y +``` + +Then update the legacy branch so it identifies itself as `X.Y` and lists only +older fallback versions. The runtime manifest supplies the current canonical +list when the current docs site is available. + +For `v7_3`, for example: + +- `docs.config.js`: `DOCS_LATEST_VERSION: '7.3'` +- `src/data/versions.ts`: first entry identifies `v7.3` without a `latest` + label; fallback entries should include `v7.2`, `v7.1`, `v7.0`, then 6.x. +- `docs/legacy.mdx`: `Sourcegraph 7.X` should include `7.2`, `7.1`, `7.0` (not 7.3 itself). + +Commit and push: + +```bash +git add docs.config.js src/data/versions.ts docs/legacy.mdx +git commit -m "Update docs latest version to X.Y" +git push legacy vX_Y +``` + +## Updating origin for the newly released version + +Work on a branch from `origin/main`: + +```bash +git switch main +git pull --ff-only origin main +git switch -c eg-update-docs-to-X-Y +``` + +Apply the same version metadata pattern, but for the new latest version. + +For `7.4`, for example: + +- `docs.config.js`: `DOCS_LATEST_VERSION: '7.4'` +- `src/data/versions.ts`: add previous versions in descending order: `v7.3`, `v7.2`, `v7.1`, `v7.0`, then 6.x. +- `docs/legacy.mdx`: add/update `Sourcegraph 7.X` with `7.3`, `7.2`, `7.1`, `7.0`. + +Commit, push the branch, and open a PR: + +```bash +git add docs.config.js src/data/versions.ts docs/legacy.mdx +git commit -m "Update docs latest version to X.Y" +git push -u origin eg-update-docs-to-X-Y +gh pr create --base main --head eg-update-docs-to-X-Y --title "Update docs latest version to X.Y" --body "## Summary +- Set DOCS_LATEST_VERSION to X.Y +- Add previous 7.x versions to the version selector +- Update the legacy versions page + +## Test plan +- Not run (config/navigation content change only)" +``` + +## Verification + +Use lightweight verification for this content/config change: + +```bash +git diff -- docs.config.js src/data/versions.ts docs/legacy.mdx +git status --short --branch +git ls-remote --heads legacy 'v7_*' +``` + +Confirm: + +- The legacy branch points to the pushed commit. +- The legacy branch identifies itself without claiming to be latest and lists + only older fallback versions. +- The origin PR branch sets the new latest version and includes the archived version in previous-version lists. +- The current manifest lists the new latest version first, and its URL points to + `https://sourcegraph.com/docs`. +- Return the local workspace to clean `main` unless the user asked to stay on a release branch. diff --git a/.amp/services.yaml b/.amp/services.yaml new file mode 100644 index 000000000..87e947eb5 --- /dev/null +++ b/.amp/services.yaml @@ -0,0 +1,10 @@ +services: + docs: + command: pnpm dev --hostname 0.0.0.0 + port: 31420 + env: + NODE_OPTIONS: --max-old-space-size=3072 + health: /agentic-batch-changes + portal: + url: /agentic-batch-changes + title: Sourcegraph Docs diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml new file mode 100644 index 000000000..c5e4d7503 --- /dev/null +++ b/.github/workflows/check-links.yml @@ -0,0 +1,110 @@ +name: Check links + +# Reports internal links and #anchors that this PR breaks, compared with the +# merge base, absolute links to this site, and external links on added lines +# that 404. Pre-existing broken links on the base branch are ignored. + +on: + pull_request: + +# A new push supersedes the run for the previous one +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + check-links: + name: Broken links introduced by this PR + runs-on: ubuntu-latest + steps: + - name: Check out pull request head + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Install github-slugger, the only dependency of dev/check-links.mjs + # Into a scratch prefix, not the repo: `npm install ` next to + # package.json would install every dependency of the site + run: | + npm install --prefix "$RUNNER_TEMP/deps" --no-package-lock --no-audit --no-fund \ + "github-slugger@$(node -p 'require("./package.json").dependencies["github-slugger"]')" + ln -s "$RUNNER_TEMP/deps/node_modules" node_modules + + - name: Check out merge base + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + merge_base=$(git merge-base "$BASE_SHA" HEAD) + git worktree add "$RUNNER_TEMP/base" "$merge_base" + git diff -U0 "$merge_base" HEAD > "$RUNNER_TEMP/changes.diff" + + - name: Record broken links already present on the base branch + # Exit 1 means findings, which is expected here + run: | + node dev/check-links.mjs --check-anchors --check-self-links --format json \ + --root "$RUNNER_TEMP/base" > "$RUNNER_TEMP/base-links.json" \ + || [ $? -eq 1 ] + + - name: Find broken links introduced by this PR + id: check + env: + # File links in the report open the file on the PR branch + LINK_BASE: ${{ github.event.pull_request.head.repo.html_url }}/blob/${{ github.event.pull_request.head.ref }} + run: | + if node dev/check-links.mjs --check-anchors --check-self-links --check-external --format markdown \ + --baseline "$RUNNER_TEMP/base-links.json" \ + --diff "$RUNNER_TEMP/changes.diff" \ + --review "$RUNNER_TEMP/review.json" \ + --link-base "$LINK_BASE" > "$RUNNER_TEMP/report.md"; then + echo "broken=false" >> "$GITHUB_OUTPUT" + else + echo "broken=true" >> "$GITHUB_OUTPUT" + fi + cat "$RUNNER_TEMP/report.md" + + - name: Comment on the pull request + # Fork PRs get a read-only token; the report is still in the job log + if: github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + BROKEN: ${{ steps.check.outputs.broken }} + run: | + marker='' + existing_comment=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --paginate --jq ".[] | select(.body | startswith(\"$marker\")) | .id" | head -n 1) + + # Comment only when there is something to report, or an earlier report to resolve + if [ "$BROKEN" = true ]; then + { echo "$marker"; cat "$RUNNER_TEMP/report.md"; } > "$RUNNER_TEMP/comment.md" + elif [ -n "$existing_comment" ]; then + printf '%s\n### ✅ The broken links an earlier revision of this PR introduced are fixed\n' \ + "$marker" > "$RUNNER_TEMP/comment.md" + else + exit 0 + fi + + if [ -n "$existing_comment" ]; then + gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing_comment" \ + --field body=@"$RUNNER_TEMP/comment.md" + else + gh pr comment "$PR_NUMBER" --body-file "$RUNNER_TEMP/comment.md" + fi + + - name: Suggest fixes as review comments + # One suggested change per finding with a fix, kept in sync with the + # findings; see dev/sync-review-comments.sh + if: github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: dev/sync-review-comments.sh ''; + const max = 40; + const lines = pages.slice(0, max).map(p => `- [/${p.path}](${p.url})`); + if (pages.length > max) lines.push(`- …and ${pages.length - max} more`); + + const body = [ + marker, + `Direct preview links to pages changed in this PR:`, + ...lines + ].join('\n'); + + // Update our previous comment instead of adding a new one per push. + const comments = await github.paginate(github.rest.issues.listComments, { + owner, repo, issue_number: pr.number, per_page: 100 + }); + const existing = comments.find(c => c.body?.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({owner, repo, comment_id: existing.id, body}); + } else { + await github.rest.issues.createComment({owner, repo, issue_number: pr.number, body}); + } diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 000000000..5e67db872 --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,54 @@ +name: Spell check + +# Reports spelling errors on lines added by a pull request, as a summary comment +# plus inline review comments on the flagged lines. Existing spelling errors and +# errors on unchanged lines are not included. + +on: + pull_request: + +permissions: + contents: read + pull-requests: write + +jobs: + spellcheck: + name: CSpell (advisory) + runs-on: ubuntu-latest + steps: + - name: Check out pull request head + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + # Node 24 is pre-cached on ubuntu-latest, avoiding the download + # incurred by node-version: latest. + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 24 + + # Install only CSpell instead of the site's full dependency tree. + - name: Install CSpell + run: npm install --global cspell@10 + + - name: Find spelling errors introduced by this PR + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + base=$(git merge-base "$BASE_SHA" HEAD) + # Exit 1 means findings; anything else is an operational error + node dev/check-spelling.mjs --base "$base" --format json \ + > "$RUNNER_TEMP/spelling.json" || [ "$?" -eq 1 ] + cat "$RUNNER_TEMP/spelling.json" + + - name: Report on the pull request + # Fork PRs get a read-only token; the findings are still in the job log + if: github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + run: node dev/post-spelling-review.mjs --findings "$RUNNER_TEMP/spelling.json" diff --git a/.github/workflows/vercel-build-report.yml b/.github/workflows/vercel-build-report.yml new file mode 100644 index 000000000..624bec91f --- /dev/null +++ b/.github/workflows/vercel-build-report.yml @@ -0,0 +1,87 @@ +name: Vercel build report + +# Vercel only shows build logs to members of its team. When a PR's Vercel +# build fails, this attaches the build log to the Vercel Slack app's "failed +# to deploy" post (SLACK_BOT_TOKEN secret and SLACK_CHANNEL_ID variable; see +# dev/slack-app-vercel-build-report.json) and comments a link to it on the +# PR; when a later revision builds, the comment is updated to say so. The log +# never goes on the PR itself, so anything sensitive a build prints stays in +# Slack instead of a public repository. +# +# GitHub only delivers repository_dispatch (and finds workflow_dispatch +# workflows) once the workflow file is on the default branch, so before merge +# run dev/report-vercel-build.mjs locally instead. After merge, re-run on a PR +# by hand with the same payload fields as inputs: +# gh workflow run vercel-build-report.yml \ +# -f id=dpl_... -f state=failed -f sha= +on: + repository_dispatch: + # A build that exits non-zero is `failed`; `error` is only sent for + # deleted deployments, which have no log + types: [vercel.deployment.failed, vercel.deployment.success] + workflow_dispatch: + inputs: + id: + description: Vercel deployment ID (client_payload.id) + required: true + state: + description: Deployment state (client_payload.state.type) + required: true + type: choice + options: [failed, success] + sha: + description: Full commit SHA of the PR head (client_payload.git.sha) + required: true + +permissions: + contents: read + pull-requests: write + +env: + DEPLOYMENT_ID: ${{ github.event.client_payload.id || inputs.id }} + DEPLOYMENT_STATE: ${{ github.event.client_payload.state.type || inputs.state }} + COMMIT_SHA: ${{ github.event.client_payload.git.sha || inputs.sha }} + GH_TOKEN: ${{ github.token }} + LOG_FILE: ${{ github.workspace }}/vercel-build.log + +jobs: + report: + # `failed` is also sent for checks_failed, aliasing_failed and account + # problems, where the build log shows a build that passed + if: >- + github.event.client_payload.environment != 'production' + && (github.event.client_payload.state.type != 'failed' + || github.event.client_payload.state.detail == 'deployment_failed') + runs-on: ubuntu-latest + steps: + - name: Check out dev/report-vercel-build.mjs + uses: actions/checkout@v4 + with: + sparse-checkout: dev/report-vercel-build.mjs + sparse-checkout-cone-mode: false + + - name: Fetch the build log from Vercel + # Vercel is only contacted when the build failed + if: env.DEPLOYMENT_STATE == 'failed' + id: log + env: + # Scoped to the sourcegraph-docs project, so it needs no team ID + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + run: node dev/report-vercel-build.mjs fetch-log "$LOG_FILE" + + - name: Attach the log to the Vercel app's Slack post + if: env.DEPLOYMENT_STATE == 'failed' + id: slack + # The PR should still hear about the failure when Slack is down + continue-on-error: true + env: + PR_NUMBER: ${{ steps.log.outputs.pull_request }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + SLACK_CHANNEL_ID: ${{ vars.SLACK_CHANNEL_ID }} + run: node dev/report-vercel-build.mjs slack "$LOG_FILE" + + - name: Comment on the pull request + env: + PR_NUMBER: ${{ steps.log.outputs.pull_request }} + SLACK_PERMALINK: ${{ steps.slack.outputs.permalink }} + run: node dev/report-vercel-build.mjs comment diff --git a/.gitignore b/.gitignore index 968a8330e..4fb487043 100644 --- a/.gitignore +++ b/.gitignore @@ -46,10 +46,13 @@ next-env.d.ts # We ignore the generated file as it should always be generated public/changelog.rss -# baseai -**/.baseai/ - # env file .env +# amp orb portal state +.amp/portals/ + public/technical-changelog.rss + +# script output +logs/ diff --git a/.tool-versions b/.tool-versions index ab17fd7e5..791fcadda 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -nodejs 20.19.6 +nodejs 24.21.0 pnpm 10.25.0 diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..cd59864b7 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": ["streetsidesoftware.code-spell-checker"] +} diff --git a/AGENTS.md b/AGENTS.md index 9a5e9b001..ddb33dcd9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,10 @@ - **Build**: `npm run build` - **Dev**: `npm run dev` - **Lint**: `npm run lint` +- **Checks**: `npm run check` runs every `dev/check-*.mjs` (links, filenames, images); `npm run build` runs them first, so any finding fails a deploy +- **Check links**: `npm run check -- links --check-anchors --check-self-links` (CI comments on PRs that break links; see `dev/check-links.mjs`; the build runs it without flags, so only dead page links fail a deploy). When moving a page or renaming a heading, update every link to it; a redirect in `src/data/redirects.ts` does not satisfy the check. Link to this site with relative paths (`/admin/config/site-config`), never `https://sourcegraph.com/docs/…` or `https://docs.sourcegraph.com/…`. To also probe the external links you added: `npm run check -- links --check-anchors --check-self-links --check-external --diff <(git diff -U0 origin/main)` +- **Prove changed links resolve on a deploy**: `node dev/verify-links-live.mjs --site ` prints a Markdown table for the PR description +- **Vercel build failures**: Vercel shows build logs only to its team members, so `.github/workflows/vercel-build-report.yml` attaches the log to the Vercel Slack app's "failed to deploy" post in `#alerts-vercel-doc-site` and comments a link to it on the PR (see `dev/report-vercel-build.mjs`). The log itself never goes on the PR, since the repository is public. It reads Vercel with the `VERCEL_TOKEN` repo secret, a token scoped to the `sourcegraph-docs` project that expires 2026-12-10; mint a new one with `POST /v3/user/tokens?teamId=` and `projectId` in the body. Slack needs the `SLACK_BOT_TOKEN` repo secret and `SLACK_CHANNEL_ID` repo variable. The bot is the Slack app in `dev/slack-app-vercel-build-report.json`; to recreate it, paste that manifest at (From a manifest), install it, copy its Bot User OAuth Token into the secret, and `/invite @Vercel build log` to the channel ## AI Chat Integration @@ -39,14 +43,7 @@ To update the runLLM assistant ID or other settings, modify the Script component /> ``` -### Previous Integration - -Previously used **Langbase** with custom React components. This has been completely removed: - -- Removed `@langbase/components` and `langbase` packages -- Removed custom chat components (`src/app/chat.tsx`, `src/components/ChatBot/`) -- Removed API routes (`src/app/api/chat/`) -- Removed memory creation scripts +The previous **Langbase** / `baseai` chatbot integration has been fully removed (packages, components, API routes, `baseai/` memory config, and the `pnpm sync` script). ## Important Notes diff --git a/README.md b/README.md index e2b1eea27..3f050ff44 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ cd docs Before the dependencies are installed make sure your local machine has the following versions of `node` and `pnpm` installed: -- node: `v20.19.6` +- node: `v24.21.0` - pnpm: `10.25.0` **Note**: If you have `mise` available you can install the above versions for only this repository by running the following command from your terminal in the root folder: @@ -36,6 +36,8 @@ Now that the base requirements of the project have been satisfied, we can instal pnpm install ``` +Spell checking is not part of the project dependencies. To run it locally: `npx cspell@10 --no-progress --dot '**/*'` + Next, run the development server: ```sh diff --git a/baseai/baseai.config.ts b/baseai/baseai.config.ts deleted file mode 100644 index 41bcc3979..000000000 --- a/baseai/baseai.config.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type {BaseAIConfig} from 'baseai'; - -export const config: BaseAIConfig = { - log: { - isEnabled: true, - logSensitiveData: false, - pipe: true, - 'pipe.completion': true, - 'pipe.request': true, - 'pipe.response': true, - tool: true, - memory: true - }, - memory: { - useLocalEmbeddings: false - }, - envFilePath: '.env' -}; diff --git a/baseai/memory/memory-sg-docs-live/index.ts b/baseai/memory/memory-sg-docs-live/index.ts deleted file mode 100644 index ba412823f..000000000 --- a/baseai/memory/memory-sg-docs-live/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -import {MemoryI} from '@baseai/core'; - -const memoryDocs = (): MemoryI => ({ - name: 'memory-sg-docs-live', - description: 'An AI memory storing all Sourcegraph docs.', - git: { - enabled: true, - include: ['**/*.mdx'], - gitignore: true, - embeddedAt: '', - deployedAt: '2506bf08459286cb0fe9f1bdebb6c73f0e19d765' - }, - documents: { - meta: doc => { - const url = `https://sourcegraph.com/docs/${doc.path}`; - return { - url, - name: doc.name - }; - } - } -}); - -export default memoryDocs; - -// Old -// deployedAt: '5f3fec8530280d01a783aadcdeb0ccc3f9cd8b70', diff --git a/contentlayer.config.ts b/contentlayer.config.ts index b7c46d824..e87527b09 100644 --- a/contentlayer.config.ts +++ b/contentlayer.config.ts @@ -1,4 +1,4 @@ -import {defineDocumentType, makeSource} from 'contentlayer/source-files'; +import {defineDocumentType, makeSource} from 'contentlayer2/source-files'; import fs from 'fs'; import rehypeAutolinkHeadings from 'rehype-autolink-headings'; import rehypePrettyCode from 'rehype-pretty-code'; @@ -9,6 +9,35 @@ import {searchMetadata} from './src/data/search'; import GithubSlugger from 'github-slugger'; import {visit} from 'unist-util-visit'; +// A fence opener/closer is a run of 3+ backticks or tildes at the start of a line. +const regXFenceLine = /^\s*(`{3,}|~{3,})/; + +// Remove fenced code blocks so `# comment` lines inside them are not mistaken for +// headings. Walks line by line: a naive /```[\s\S]*?```/ regex also matches inline +// backtick runs in prose (e.g. `"true```), which flips every later fence pairing. +function stripFencedCodeBlocks(markdown: string): string { + const keptLines: string[] = []; + let openFence: string | undefined; + for (const line of markdown.split('\n')) { + const fence = line.match(regXFenceLine)?.[1]; + if (openFence) { + const closesOpenFence = + fence !== undefined && + fence[0] === openFence[0] && + fence.length >= openFence.length && + line.trim() === fence; + if (closesOpenFence) { + openFence = undefined; + } + } else if (fence) { + openFence = fence; + } else { + keptLines.push(line); + } + } + return keptLines.join('\n'); +} + export const Post = defineDocumentType(() => ({ name: 'Post', filePathPattern: `**/*.mdx`, @@ -28,17 +57,10 @@ export const Post = defineDocumentType(() => ({ type: 'json', resolve: async doc => { const regXHeader = /^ *(?#{1,6})\s+(?.+)/gm; - const regXCodeBlock = /```[\s\S]*?```/g; const slugger = new GithubSlugger(); - // Ignore content within code blocks – No headings there - const bodyWithoutCodeBlocks = doc.body.raw.replace( - regXCodeBlock, - '' - ); - const headings = Array.from( - bodyWithoutCodeBlocks.matchAll(regXHeader) + stripFencedCodeBlocks(doc.body.raw).matchAll(regXHeader) ).map(({groups}) => { const flag = groups?.flag; // Handles headings with links eg: diff --git a/cspell-allow-list.txt b/cspell-allow-list.txt new file mode 100644 index 000000000..abe1a0e1a --- /dev/null +++ b/cspell-allow-list.txt @@ -0,0 +1,603 @@ +# Words CSpell should accept +# Matching is case- and accent-insensitive + +# Sourcegraph Chrome extension ID +dgjhfomjieaadpoljlnidmbgkdffpack + +# Base64-encoded GraphQL ID "User:1" +VXNlcjox + +# Algolia application ID +0EBA2NRQU3 + +# Base64-encoded GraphQL ID "SearchJob:1" +U2VhcmNoSm9iOjY5 + +# Fragments of truncated shell output in examples +actr +autol +ified +mtok +pousr +sourcegr +stabili + +# Words +acmeco +ADFS +advisements +airgapped +Aiven +algoliasearch +allkeys +amcheck +amname +ampcode +anotherproject +anotherrepo +apidocs +appendonly +appengine +appgw +ARGF +argjson # jq option +atoburl +atoi +attnum +attrelid +autocompletions +autoedit +autoedits +autoindexing +automations +autoscalers +autoupgrade +autovacuum +Awais +awscli +awsrepos +azuredevops +backfiller +baseai +batchchange +batchchanges +batcheshelper +batchignore +batchutils +behaviour +behaviours +beyang +bgwriter +binutils +bitbucketcloud +bitbucketserver +bitmapscan +Bitwarden +blkid +blobstore +blurple +browsable +buildconfig +Burkina +burndown +BYOK +BYOLLM +Cabo +cacerts +cadvisor +callsign +callsigns +callsites +camdentest +CAROOT +Certbot +certonly +changesettemplate +changesettemplatebody +changesettemplatebranch +changesettemplatecommit +changesettemplatecommitauthor +changesettemplatecommitmessage +changesettemplatefork +changesettemplatepublished +changesettemplatetitle +chatbots +cheatsheet +checkhealth +citext +Citus +clickjacking +clientauthconfig +cloneproxy +cloudasset +cloudkms +cloudnotifications +cloudtrace +clusterrole +clusterrolebinding +cmdline +codecompletions +codegraph +codehost +codehosts +codeinsights +codeintel +codellama +codemod +codemonitors +codesearch +collatable +colordiff +combinatorially +comby +commitgraph +committerdate +configmap +conntrack +Consolas +contentlayer +Côte +crashloop +crashloopbackoff +createdat +cstring +ctid +ctstate +CUDA +customcert +customising +d'Ivoire +daemonset +darkhold +datalake +datcollversion +datname +dbname +dbstore +dbug +decrypter +deepsearch +deepseek +demopasswordchangeme +DEVMINOR +DEVPATCH +devrel +dgrijalva +diffstat +dind +distros +dmsetup +dockerfiles +docsite +docstrings +doombot +downto +dport +drilldown +drilldowns +drwxr +dshm +dumpall +eastus +eksctl +ELEC +elif +emailaddress +encrypter +Enry +entrycommand +envsubst +errgroup +errorf +esbenp +Eswatini +etcdctl +euxo +EVICTEDPOD +exfiltration +explorable +externalapi +extsvc +Falco +Faso +favorited +favoriting +favourite +federationmetadata +Filippo +finetuning +finishedat +fmtlib +forrester +frontmatter +FSTYPE +fullchain +fullpath +gifs +gitdir +gitea +githost +Gitolite +gitserver +gnachman +gobwas +gofmt +goimports +GOMAXPROCS +googlecloud +gopkg +goreleaser +goroutines +graphbook +graphd +graphqlbackend +gsuite +healthcheck +healthz +HELO +horsegraph +horsten +hostmatcher +hostpath +HSTS +httptest +huggingface +Iconify +IDPSSO +imagepullsecrets +importchangesets +importchangesetsexternalids +importchangesetsrepository +incidentio +inconshreveable +indclass +indcollation +indexability +indexrelid +indexroot +indexscan +indisready +indisvalid +indkey +indnkeyatts +indrelid +intarray +intelli +isopen +istari +itable +iterm +itoa +JCEF +jiapantw +journalctl +Jsonnet +jsonschemadoc +Kaspersky +kbar +keegancsmith +keymap +keymappings +keypair +killall +Kitts +kubeconfig +kubelet +kustomization +Langbase +languagemodels +Laravel +latencytest +latveria +lefthand +Leste +letsencrypt +libsecret +lockfiles +logfmt +logpush +logtost +logurl +lookarounds +losetup +lsif +Luhansk +maedahbatool +Marino +maxage +maxmemory +maxpages +megarepo +Memorystore +Menlo +migops +mixtral +mktemp +mmap +modelconfig +mountpoint +mpim +multiplicatively +multiqueue +multiversion +mvnw +mycompany +myconfig +mydockerhub +myext +myextension +mygroup +mykey +myorg +myotherrepo +mypassword +myproject +myregistry +myrepo +mysqli +mystring +mytag +myteam +myvalue +nameid +nameopt +navigations +NETRC +nodeport +noeviction +nofail +noheadings +nonexistenturl +nonprivileged +noout +NOSYSTEM +nslookup +nspname +numpy +nvim +oauthconfig +objectname +oidvectorin +oldtbl +Ollama +omnibox +oneclick +onelogin +onrepositoriesmatchingquery +onrepository +OOBMIGRATION +opclass +opcmethod +openaicompatible +opencode +openctx +opengrok +openidconnect +optim +organisation +organisations +ORGID +orgpolicy +overcounted +oyaml +pagure +Parsely +parsewithclaims +patterntype +PCIW +PCRE +pekko +persistentvolumeclaims +pgbouncer +pgcrypto +PGDATABASE +PGDATASOURCE +pgdump +PGHOST +PGPASSWORD +PGPORT +pgsql +PGSSLMODE +PGUSER +phpdoc +pipefail +PKCE +PKEY +plpgsql +PODNAME +polysans +pooler +PREEMPTIBLE +preg +premade +prerendered +Príncipe +privkey +projectname +protoc +pubout +pullrequest +pvcs +pykafka +quicklinks +Qwen +rbacs +readwrite +reclone +recloning +reexecuting +refreshable +registryurl +reindexdb +reindexing +relid +relkind +relname +relnamespace +relpages +relpersistence +repogroup +repohasfile +reponame +requestclient +requirepass +rescope +resourcemanager +Rica +Rockskip +rootfs +rstrip +runllm +ruplacer +samltool +sams +sandboxing +sboms +SCIM +scip +SCIW +seccomp +secretmanager +secretname +Sectigo +Sendgrid +serde +serilog +setext +settingsjson +setx +sgdev +sggray +sgquery +sgtest +sgviolet +Shadcn +shellsession +shiki +showcerts +SIEM +sigalg +SLES +slurpfile +Snek +snekpm +somerandom +Sourcegraph +sourcegraphcloud +sourcegraphcom +spacebar +specialised +spectaql +SQLSTATE +srcgraph +ssbc +sshuttle +sslmode +stackexchange +standardly +starcoder +Starlark +startedat +startswith +statefulsets +stepscontainer +stepsenv +stepsfiles +stepsif +stepsmount +stepsoutputs +stepsoutputsnameformat +stepsoutputsnamevalue +stepsrun +storageaccounttype +STORAGECLASS +Strato +strconv +subchart +subcharts +subexpression +subnetworks +subpackages +subsubgroup +subteam +subteams +subwords +sunsetting +symf +syncer +syncers +synchronised +Syntect +Talkhouse +TARGETGROUP +testroute +testsourcegraph +testusers +Textualize +theirrepo +thorsten +thorstens +threadcreate +timedout +timemachine +tini +tjdevries +tolerations +Tomé +toolcall +topk +topsecretorg +topsecretproject +tostring # jq builtin +transactionally +transformchanges +transformchangesgroup +transformchangesgroupbranch +transformchangesgroupdirectory +transformchangesgrouprepository +trgm +triaging +trialling +Türkiye +typesafe +typescriptjavascript +unarchived +unarchiving +uncordon +underallocated +undercount +undercounted +underprovisioned +unibeautify +unindexed +unioned +unmigrated +unminimize +untar +updatecheck +upperand +urlencode +urlsafe +userprincipalname +USERTRUST +Valsorda +vegeta +vercel +Verilog +VHDL +Viktor +vscodesourcegraph +waitlist +Weaveworks +winsize +workspacesin +workspacesonlyfetchworkspace +workspacesrootatlocationof +worktree +XGET +xlarge +yourorgname +YOURUSERNAME +Zaporizhzhia +Zoekt +zoomable diff --git a/cspell-block-list.txt b/cspell-block-list.txt new file mode 100644 index 000000000..a21c45e61 --- /dev/null +++ b/cspell-block-list.txt @@ -0,0 +1,112 @@ +# Words CSpell should flag +# Matching is case- and accent-insensitive +accomodate +acepted +activites +alignemnt +authentitcation +authroized +autoiondex +compatability +comphrensive +composit +conection +configuredimageimage +consule +contiainer +contraint +crtl +custmoize +databse +dedupliacte +defininition +depdendencies +determinstic +documenta +eclispe +embeeddings +emtpy +endable +exector +executon +exernal +exisiting +explictly +fetchinig +fulfiill +fulfuill +gatway +gemin +gitab +gitgnore +gomft +guarauntees +hopefull +horizonal +identitfier +ingore +insufficent +intterupt +invididual +jeager +limitting +managmeent +matadata +migtrator +mulitenant +necesarily +occured +occurence +occuring +opean +optimisitic +optimizated +ordererd +owernship +paramter +peforce +percaution +permanant +permanentaly +permisisons +postgress +prerequisit +privileized +privisioning +promotheus +pugins +pumbling +puslishing +reapplyed +repliacas +rermoved +reveiver +santizes +screeen +scret +seach +searcg +searchs +searhces +seperate +serch +simiply +simpify +sitiched +sorucegraph +soucegraph +soureces +specifiy +stablility +stiched +sufficiantly +symbos +synchronizzation +tailling +targetting +teamates +telemetery +tetsted +unmarsha +unsued +workpsaces diff --git a/cspell.json b/cspell.json new file mode 100644 index 000000000..749ab8131 --- /dev/null +++ b/cspell.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json", + "version": "0.2", + "useGitignore": true, + "dictionaryDefinitions": [ + { + "name": "sourcegraph-docs", + "path": "./cspell-allow-list.txt", + "addWords": true + }, + { + "name": "sourcegraph-docs-block-list", + "path": "./cspell-block-list.txt", + "kind": "flag-words" + } + ], + "dictionaries": ["sourcegraph-docs", "sourcegraph-docs-block-list"], + "ignoreRegExpList": [ + "/\\\\[nrt]/g", + "/(? Repository to check (default: this repository) + * --format Output as text (default), json, or markdown + * --baseline Only report findings absent from this JSON file + * (produced by --format json on another revision) + * --link-base Markdown output links each file path to /, + * e.g. https://github.com/sourcegraph/docs/blob/ + * --diff Unified diff of the change under review, e.g. from + * `git diff -U0 origin/main`. Markdown output splits + * findings into outbound (in a file the diff touches) and + * inbound (elsewhere); the added lines scope the two flags below + * --check-external Request every external link on an added line and report + * 404s and 410s. Follows redirects; ignores #anchors, other + * statuses, and network errors. Requires --diff + * --review Write a GitHub pull request review (JSON body for + * POST /repos/{owner}/{repo}/pulls/{n}/reviews) with one + * suggested-change comment per added line that has a fix + * + * Exits 1 when any finding is reported. */ import fs from 'fs'; import path from 'path'; -import { glob } from 'glob'; import GithubSlugger from 'github-slugger'; -import { fileURLToPath } from 'url'; +import { fileURLToPath, pathToFileURL } from 'url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); -const DOCS_DIR = path.join(path.dirname(__dirname), 'docs'); - // Parse CLI flags const args = process.argv.slice(2); const CHECK_ANCHORS = args.includes('--check-anchors'); +const CHECK_SELF_LINKS = args.includes('--check-self-links'); +const ROOT_DIR = path.resolve(flagValue('--root') ?? path.dirname(__dirname)); +const FORMAT = flagValue('--format') ?? 'text'; +const BASELINE_FILE = flagValue('--baseline'); +const LINK_BASE = flagValue('--link-base')?.replace(/\/$/, ''); +const DIFF = parseDiff(flagValue('--diff')); +const CHECK_EXTERNAL = args.includes('--check-external'); +const REVIEW_FILE = flagValue('--review'); + +if (CHECK_EXTERNAL && !DIFF) { + throw new Error('--check-external needs --diff, to know which lines were added'); +} + +// Files and added lines of a unified diff, with paths relative to the repository: +// { files: Set<'docs/foo.mdx'>, addedLines: Map<'docs/foo.mdx', Set> }. +// Works with any amount of context, so `git diff` and `git diff -U0` both do. +function parseDiff(file) { + if (!file) return undefined; + const files = new Set(); + const addedLines = new Map(); + let currentFile; + let lineNumber; + for (const line of fs.readFileSync(file, 'utf-8').split('\n')) { + if (line.startsWith('diff --git ')) { + currentFile = undefined; + } else if (line.startsWith('+++ ') && !currentFile) { + // `+++ /dev/null` is a deleted file, which has no added lines + currentFile = line.slice(4).replace(/^b\//, ''); + if (currentFile === '/dev/null') continue; + files.add(currentFile); + addedLines.set(currentFile, new Set()); + } else if (line.startsWith('@@ ')) { + lineNumber = Number(line.match(/^@@ -\S+ \+(\d+)/)[1]); + } else if (line.startsWith('+') && currentFile) { + addedLines.get(currentFile).add(lineNumber++); + } else if (line.startsWith(' ')) { + lineNumber++; + } + } + return { files, addedLines }; +} + +const DOCS_DIR = path.join(ROOT_DIR, 'docs'); +// Files whose links are checked. Only .mdx files become site routes; see +// `filePathPattern` in contentlayer.config.ts. +const SOURCE_EXTENSIONS = ['.md', '.mdx']; +const ROUTE_EXTENSIONS = ['.mdx']; + +function flagValue(name) { + const index = args.indexOf(name); + return index === -1 ? undefined : args[index + 1]; +} + +// Sorted relative paths of every file under dir, optionally limited to some +// extensions. Sorted so foo.mdx precedes foo/index.mdx; when both exist the +// site serves the first match (allPosts.find), so the first file owns the route. +export function listFiles(dir, extensions) { + if (!fs.existsSync(dir)) return []; + return fs + .readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter(entry => entry.isFile() && (!extensions || extensions.includes(path.extname(entry.name)))) + .map(entry => path.relative(dir, path.join(entry.parentPath, entry.name))) + .sort(); +} // Regex patterns for extracting links const MARKDOWN_LINK_REGEX = /\[([^\]]*)\]\(([^)]+)\)/g; const JSX_HREF_REGEX = /href=["']([^"']+)["']/g; const SRC_ATTR_REGEX = /src=["']([^"']+)["']/g; -// Extract headings from MDX content to build anchor map -function extractHeadings(content) { +// A fence opener/closer is a run of 3+ backticks or tildes at the start of a line. +const FENCE_LINE_REGEX = /^\s*(`{3,}|~{3,})/; + +// Blank out fenced code blocks, keeping line numbers intact, so `# comment` +// lines and example links inside them are ignored. Walks line by line: a naive +// /```[\s\S]*?```/ regex also matches inline backtick runs in prose (e.g. +// `"true```), which flips every later fence pairing. +function stripFencedCodeBlocks(content) { + let openFence; + return content.split('\n').map(line => { + const fence = line.match(FENCE_LINE_REGEX)?.[1]; + if (openFence) { + const closesOpenFence = + fence !== undefined && + fence[0] === openFence[0] && + fence.length >= openFence.length && + line.trim() === fence; + if (closesOpenFence) { + openFence = undefined; + } + return ''; + } + if (fence) { + openFence = fence; + return ''; + } + return line; + }).join('\n'); +} + +// Extract anchor targets from MDX content: heading slugs, plus explicit +// and id="..." attributes +export function extractHeadings(content) { const slugger = new GithubSlugger(); const headingRegex = /^#{1,6}\s+(.+)$/gm; + const explicitAnchorRegex = /<[a-zA-Z][^>]*\s(?:id|name)=["']([^"']+)["']/g; const headings = new Set(); - // Remove code blocks to avoid false positives - const contentWithoutCode = content.replace(/```[\s\S]*?```/g, ''); + const contentWithoutCode = stripFencedCodeBlocks(content); let match; while ((match = headingRegex.exec(contentWithoutCode)) !== null) { - // Handle headings with links: [Text](/path) -> Text - const linkMatch = match[1].match(/\[([^\]]+)\]\([^)]+\)/); - const title = linkMatch ? linkMatch[1] : match[1]; + // rehype-slug slugs the heading's full text, with links reduced to their text: + // "How can I use [GitHub expression syntax](url) literally" -> "How can I use GitHub expression syntax literally" + const title = match[1].replace(/\[([^\]]+)\]\([^)]+\)/g, '$1'); headings.add(slugger.slug(title.trim())); } + while ((match = explicitAnchorRegex.exec(contentWithoutCode)) !== null) { + headings.add(match[1]); + } + return headings; } +// Site route for a file under docs/: foo/bar.mdx -> /foo/bar, foo/index.mdx -> /foo, index.mdx -> / +export function routeFor(file) { + return '/' + file.replace(/\.mdx$/, '').replace(/(^|\/)index$/, ''); +} + // Get all MDX files and build a map of valid paths -async function buildPathMap() { - const files = await glob('**/*.mdx', { cwd: DOCS_DIR }); +function buildPathMap() { + const files = listFiles(DOCS_DIR, ROUTE_EXTENSIONS); const pathMap = new Map(); + // Lowercased route -> real route, to detect case mismatches + const routesByLowerCase = new Map(); const headingsMap = new Map(); + // Absolute file path -> headings, for same-page #anchor links + const headingsByFile = new Map(); for (const file of files) { const fullPath = path.join(DOCS_DIR, file); - const content = fs.readFileSync(fullPath, 'utf-8'); + const headings = extractHeadings(fs.readFileSync(fullPath, 'utf-8')); + headingsByFile.set(fullPath, headings); - // Route path (without .mdx extension) - const routePath = '/' + file.replace(/\.mdx$/, '').replace(/\/index$/, ''); + const routePath = routeFor(file); + if (pathMap.has(routePath)) continue; // Also allow trailing slash variant pathMap.set(routePath, fullPath); pathMap.set(routePath + '/', fullPath); - - // Handle index files - if (file.endsWith('index.mdx')) { - const dirPath = '/' + file.replace(/\/index\.mdx$/, ''); - pathMap.set(dirPath, fullPath); - pathMap.set(dirPath + '/', fullPath); - } - - // Extract headings for anchor validation - const headings = extractHeadings(content); + routesByLowerCase.set(routePath.toLowerCase(), routePath); headingsMap.set(routePath, headings); headingsMap.set(routePath + '/', headings); } - return { pathMap, headingsMap }; + return { + pathMap, + routesByLowerCase, + headingsMap, + headingsByFile, + assetsByLowerCase: buildAssetMap(), + redirects: loadRedirects() + }; } -// Check if a path exists in public directory -function checkPublicPath(linkPath) { - const publicPath = path.join(path.dirname(__dirname), 'public', linkPath); - return fs.existsSync(publicPath); +// Source route -> destination of src/data/redirects.ts. The middleware uses the +// first rule whose source equals the requested path, so first entry wins here too. +function loadRedirects() { + const redirects = new Map(); + const source = fs.readFileSync(path.join(ROOT_DIR, 'src/data/redirects.ts'), 'utf-8'); + const ruleRegex = /source:\s*(['"])(.*?)\1,\s*destination:\s*(['"])(.*?)\3/gs; + for (const [, , from, , to] of source.matchAll(ruleRegex)) { + if (!redirects.has(from)) redirects.set(from, to); + } + return redirects; } -// Check if a path exists in docs directory (for images in docs/) -function checkDocsPath(linkPath) { - const docsPath = path.join(DOCS_DIR, linkPath); - return fs.existsSync(docsPath); +// Lowercased link path -> real link path, for files under public/ and docs/ +// (images, PDFs, ...). An enumerated map rather than fs.existsSync, which is +// case-insensitive on macOS and would hide links that 404 on Linux. +function buildAssetMap() { + const assetsByLowerCase = new Map(); + for (const dir of ['public', 'docs']) { + for (const file of listFiles(path.join(ROOT_DIR, dir))) { + const linkPath = '/' + file; + assetsByLowerCase.set(linkPath.toLowerCase(), linkPath); + } + } + return assetsByLowerCase; } // Parse and validate links in a single file function extractLinks(content, filePath) { const links = []; - // Remove code blocks to avoid checking links in code examples - const contentWithoutCode = content.replace(/```[\s\S]*?```/g, (match) => { - // Replace with same number of newlines to preserve line numbers - return match.replace(/[^\n]/g, ' '); - }); + const contentWithoutCode = stripFencedCodeBlocks(content); // Extract markdown links [text](url) let match; @@ -136,12 +278,65 @@ function extractLinks(content, filePath) { return links; } -// Check if a link is valid -function validateLink(link, currentFile, pathMap, headingsMap) { +// Absolute links to this site, in every form the docs have used: http or https, +// scheme-relative, www., the legacy docs.sourcegraph.com host, or sourcegraph.com/docs. +// Links pinned to an old version (/@5.1/..., /v/5.1/...) are external: the +// middleware sends them to that version's own site (5.1.sourcegraph.com), whose +// pages are not in this repo, so only --check-external can validate them. +const SELF_LINK_REGEX = /^(?:https?:)?\/\/(?:www\.)?(?:docs\.sourcegraph\.com|sourcegraph\.com\/docs)(?=[/#?]|$)(?!\/@|\/v\/)/i; + +export function isSelfLink(url) { + return SELF_LINK_REGEX.test(url); +} + +// The relative form of an absolute self-link: https://sourcegraph.com/docs/a/b/#c -> /a/b#c. +// A ?query has no meaning on a docs page and is dropped. +function relativeSelfLink(url) { + const [pathAndQuery, anchor] = url.replace(SELF_LINK_REGEX, '').split('#'); + const route = pathAndQuery.split('?')[0].replace(/\/$/, '') || '/'; + return anchor ? `${route}#${anchor}` : route; +} + +// Absolute self-links break on preview deployments and local dev, and hide moved +// pages behind redirects, so they are findings even when the target exists. The +// fix is the relative link, following src/data/redirects.ts when the page moved. +// The redirect destination's own #anchor wins over the link's, like the middleware. +function validateSelfLink(url, currentFile, maps) { + const relative = relativeSelfLink(url); + const anchor = relative.split('#')[1]; + const visited = new Set(); + let candidate = relative; + while (true) { + const error = candidate === relative ? 'Absolute link to this site' : 'Absolute link to a moved page'; + const problem = validateLink({ url: candidate }, currentFile, maps); + if (!problem) { + return { error, fix: candidate }; + } + const destination = maps.redirects.get(candidate.split('#')[0]); + if (!destination || visited.has(destination)) { + const message = problem.error ?? problem; + const replaced = candidate === relative ? ', and' : `; "${candidate}" replaced it, but`; + return { error: `${error}${replaced} ${message[0].toLowerCase()}${message.slice(1)}` }; + } + visited.add(destination); + candidate = isSelfLink(destination) ? relativeSelfLink(destination) : destination; + if (anchor && !candidate.includes('#') && !/^https?:/.test(candidate)) { + candidate += `#${anchor}`; + } + } +} + +// Check if a link is valid. Returns null, an error string, or { error, fix }. +function validateLink(link, currentFile, maps) { + const { pathMap, routesByLowerCase, headingsMap, headingsByFile, assetsByLowerCase } = maps; const { url } = link; + + if (isSelfLink(url)) { + return CHECK_SELF_LINKS ? validateSelfLink(url, currentFile, maps) : null; + } // Skip external links, mailto, tel, javascript, etc. - if (url.startsWith('http://') || url.startsWith('https://') || + if (url.startsWith('http://') || url.startsWith('https://') || url.startsWith('//') || url.startsWith('mailto:') || url.startsWith('tel:') || url.startsWith('javascript:') || url.startsWith('data:') || url.startsWith('command:')) { @@ -164,10 +359,7 @@ function validateLink(link, currentFile, pathMap, headingsMap) { return null; } const anchor = url.substring(1); - const currentRoute = '/' + path.relative(DOCS_DIR, currentFile) - .replace(/\.mdx$/, '') - .replace(/\/index$/, ''); - const headings = headingsMap.get(currentRoute); + const headings = headingsByFile.get(currentFile); if (headings && !headings.has(anchor)) { return `Anchor "${anchor}" not found in current file`; @@ -211,80 +403,286 @@ function validateLink(link, currentFile, pathMap, headingsMap) { return null; } - // Check if it's a public asset - if (checkPublicPath(resolvedPath)) { + // Check if it's an asset under public/ or docs/ + const realAsset = assetsByLowerCase.get(resolvedPath.toLowerCase()); + if (realAsset === resolvedPath) { return null; } + // Same route or asset with different case: resolves on macOS, 404s on the Linux build + const realPath = realAsset ?? routesByLowerCase.get( + resolvedPath.replace(/\/$/, '').toLowerCase() + ); + if (realPath) { + return { error: 'Path case mismatch: works on macOS, 404s on the Linux build', fix: anchor ? `${realPath}#${anchor}` : realPath }; + } + // Check if it's a file with extension (like .png, .pdf) if (path.extname(resolvedPath)) { - // Could be an asset - check public folder or docs folder - if (checkPublicPath(resolvedPath) || checkDocsPath(resolvedPath)) { - return null; - } return `File not found: "${resolvedPath}"`; } return `Page not found: "${resolvedPath}"`; } -async function main() { - console.log('🔍 Checking for dead links in MDX files...\n'); - - const { pathMap, headingsMap } = await buildPathMap(); - const files = await glob('**/*.mdx', { cwd: DOCS_DIR }); - - let totalErrors = 0; - const errors = []; +function isAddedLine(file, line) { + return DIFF?.addedLines.get(file)?.has(line) ?? false; +} + +// Find every broken link: [{ file, line, url, error, fix? }] +async function findBrokenLinks() { + const maps = buildPathMap(); + const findings = []; + const externalLinks = []; - for (const file of files) { + for (const file of listFiles(DOCS_DIR, SOURCE_EXTENSIONS)) { const fullPath = path.join(DOCS_DIR, file); const content = fs.readFileSync(fullPath, 'utf-8'); - const links = extractLinks(content, fullPath); - const fileErrors = []; - - for (const link of links) { - const error = validateLink(link, fullPath, pathMap, headingsMap); - if (error) { - fileErrors.push({ - line: link.lineNumber, - url: link.url, - error - }); + for (const link of extractLinks(content, fullPath)) { + const location = { file: `docs/${file}`, line: link.lineNumber, url: link.url }; + const problem = validateLink(link, fullPath, maps); + if (problem) { + findings.push({ ...location, ...(typeof problem === 'string' ? { error: problem } : problem) }); + } else if (CHECK_EXTERNAL && isExternalLink(link.url) && isAddedLine(location.file, location.line)) { + externalLinks.push(location); } } - - if (fileErrors.length > 0) { - errors.push({ - file: `docs/${file}`, - errors: fileErrors - }); - totalErrors += fileErrors.length; - } } - // Output results - if (errors.length === 0) { - console.log('✅ No dead links found!'); - process.exit(0); + return [...findings, ...(await findDeadExternalLinks(externalLinks))]; +} + +// Hosts reserved for examples and documentation (RFC 2606, RFC 6761), never requested +const PLACEHOLDER_HOST_REGEX = /(^|\.)(example\.(com|net|org)|example|test|invalid|localhost|local|internal)$/i; +// Templated URLs like https:/// or https://$HOST/, never requested +const PLACEHOLDER_URL_REGEX = /[<>{}$*]/; + +function isExternalLink(url) { + if (!/^https?:\/\//i.test(url) || PLACEHOLDER_URL_REGEX.test(url)) return false; + try { + return !PLACEHOLDER_HOST_REGEX.test(new URL(url).hostname); + } catch { + return false; + } +} + +// HTTP status of url after redirects, or undefined on a network error or timeout. +// HEAD first; some servers refuse or misreport HEAD, so an error status is +// confirmed with a GET whose body is not read. +async function probeUrl(url) { + const request = method => + fetch(url, { + method, + redirect: 'follow', + signal: AbortSignal.timeout(15_000), + headers: { 'user-agent': 'sourcegraph-docs-check-links (+https://github.com/sourcegraph/docs)' } + }); + try { + let response = await request('HEAD'); + if (response.status >= 400) { + response = await request('GET'); + await response.body?.cancel(); + } + return response.status; + } catch { + return undefined; + } +} + +// Findings for external links whose target is gone. Only 404 and 410 count: rate +// limits, bot blocks, server errors, and network failures are not the PR's fault. +async function findDeadExternalLinks(links) { + const urls = [...new Set(links.map(link => link.url.split('#')[0]))]; + const statusByUrl = new Map(); + const queue = [...urls]; + const worker = async () => { + for (let url = queue.shift(); url !== undefined; url = queue.shift()) { + statusByUrl.set(url, await probeUrl(url)); + } + }; + await Promise.all(Array.from({ length: 8 }, worker)); + + return links.flatMap(link => { + const status = statusByUrl.get(link.url.split('#')[0]); + return status === 404 || status === 410 ? [{ ...link, error: `External link returns HTTP ${status}` }] : []; + }); +} + +// Identity of a finding across revisions: line numbers shift, so ignore them +function findingKey({ file, url, error }) { + return `${file}\n${url}\n${error}`; +} + +function withoutBaseline(findings, baselineFile) { + const baseline = new Set( + JSON.parse(fs.readFileSync(baselineFile, 'utf-8')).map(findingKey) + ); + return findings.filter(finding => !baseline.has(findingKey(finding))); +} + +function groupByFile(findings) { + const byFile = new Map(); + for (const finding of findings) { + if (!byFile.has(finding.file)) { + byFile.set(finding.file, []); + } + byFile.get(finding.file).push(finding); + } + return byFile; +} + +function formatText(findings) { + const scope = BASELINE_FILE ? 'new ' : ''; + if (findings.length === 0) { + return `✅ No ${scope}dead links found!\n`; } - console.log(`❌ Found ${totalErrors} dead link(s) in ${errors.length} file(s):\n`); + const byFile = groupByFile(findings); + const lines = [ + `❌ Found ${findings.length} ${scope}dead link(s) in ${byFile.size} file(s):\n` + ]; + for (const [file, fileFindings] of byFile) { + lines.push(`\n📄 ${file}`); + for (const { line, url, error, fix } of fileFindings) { + lines.push(` Line ${line}: ${url}`); + lines.push(` └─ ${error}${fix ? `; use ${fix}` : ''}`); + } + } + return lines.join('\n') + '\n'; +} + +function linkTo(text, url) { + return url ? `[${text}](${url})` : text; +} + +// Markdown list of findings grouped by file, one line per fact, linked to the +// source when --link-base is set +function markdownFindingList(findings) { + const lines = []; + for (const [file, fileFindings] of groupByFile(findings)) { + // ?plain=1 opens GitHub's code view, where #L anchors work; the rendered + // Markdown preview ignores them + const fileUrl = LINK_BASE && `${LINK_BASE}/${file}?plain=1`; + lines.push(linkTo(`**\`${file}\`**`, fileUrl)); + for (const { line, url, error, fix } of fileFindings) { + lines.push( + `- ${linkTo(`line ${line}`, fileUrl && `${fileUrl}#L${line}`)}`, + ` - Link: \`${url}\``, + ` - Problem: ${error}`, + ...(fix ? [` - Fix: \`${fix}\``] : []) + ); + } + lines.push(''); + } + return lines; +} + +const ABSOLUTE_LINKS_ADVICE = + 'Write links on this site as relative paths (`/admin/config/site-config`), ' + + 'not `https://sourcegraph.com/docs/…`: absolute links leave the preview ' + + 'deployment and local dev server, and hide moved pages behind redirects.'; + +// Body for a pull request comment. With --diff, findings are split into +// outbound (in a file this PR changed: the PR added or edited a bad link), +// absolute links to this site, and inbound (in a file the PR did not change: +// the PR renamed or removed a link target). +function formatMarkdown(findings) { + if (findings.length === 0) { + return '### ✅ This PR introduces no broken links\n'; + } - for (const { file, errors: fileErrors } of errors) { - console.log(`\n📄 ${file}`); - for (const { line, url, error } of fileErrors) { - console.log(` Line ${line}: ${url}`); - console.log(` └─ ${error}`); + const lines = [`### ❌ This PR introduces ${findings.length} broken link(s)`, '']; + const section = (heading, intro, sectionFindings) => { + if (sectionFindings.length > 0) { + lines.push(`### ${heading}`, '', intro, '', ...markdownFindingList(sectionFindings)); + } + }; + if (DIFF) { + const absolute = findings.filter(finding => isSelfLink(finding.url)); + const outbound = findings.filter(finding => !isSelfLink(finding.url) && DIFF.files.has(finding.file)); + const inbound = findings.filter(finding => !isSelfLink(finding.url) && !DIFF.files.has(finding.file)); + section('Outbound', 'Your PR includes links to pages or anchors that do not exist.', outbound); + section('Absolute links', ABSOLUTE_LINKS_ADVICE, absolute); + section( + 'Inbound', + 'A change your PR made broke inbound links from these other files. Please fix the inbound links in these other files.', + inbound + ); + } else { + lines.push(...markdownFindingList(findings)); + if (findings.some(finding => isSelfLink(finding.url))) { + lines.push(ABSOLUTE_LINKS_ADVICE, ''); } } + lines.push( + 'Reproduce locally with `pnpm check links --check-anchors --check-self-links` ' + + '(see `dev/check-links.mjs`).', + '', + 'Adding a redirect in `src/data/redirects.ts` does not satisfy this ' + + 'check, because it’s a workaround instead of a fix.' + ); + return lines.join('\n') + '\n'; +} + +// First line of a review comment, so the workflow can match the comments it +// posted earlier to the findings still present and delete the rest +const REVIEW_MARKER = '`, + `Link: \`${url}\``, + `Problem: ${error}`, + `Fix: \`${fix}\``, + '````suggestion', + source.split(url).join(fix), + '````' + ]; + return { path: file, line, side: 'RIGHT', body: body.join('\n') }; + }); + return { event: 'COMMENT', body: '', comments }; +} + +const FORMATTERS = { + text: formatText, + json: findings => JSON.stringify(findings, null, '\t') + '\n', + markdown: formatMarkdown +}; + +async function main() { + const format = FORMATTERS[FORMAT]; + if (!format) { + throw new Error(`Unknown --format "${FORMAT}"; use text, json, or markdown`); + } - console.log('\n'); - process.exit(1); + if (FORMAT === 'text') { + console.log('🔍 Checking for dead links in MDX files...\n'); + } + + let findings = await findBrokenLinks(); + if (BASELINE_FILE) { + findings = withoutBaseline(findings, BASELINE_FILE); + } + + if (REVIEW_FILE) { + fs.writeFileSync(REVIEW_FILE, JSON.stringify(reviewRequest(findings), null, '\t') + '\n'); + } + process.stdout.write(format(findings)); + process.exit(findings.length === 0 ? 0 : 1); } -main().catch(err => { - console.error('Error running link checker:', err); - process.exit(1); -}); +// Only run when executed directly; dev/verify-links-live.mjs and dev/check-redirects.mjs +// import the exported helpers. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/dev/check-spelling.mjs b/dev/check-spelling.mjs new file mode 100644 index 000000000..cf07ab1c9 --- /dev/null +++ b/dev/check-spelling.mjs @@ -0,0 +1,187 @@ +#!/usr/bin/env node + +/** + * Reports CSpell findings on lines added by a Git diff, and dictionary entries + * added out of alphabetical order. + * + * Usage: node dev/check-spelling.mjs --base [--format text|json] + * + * The json format feeds dev/post-spelling-review.mjs. + * Exits 1 when spelling issues are found and 2 for operational errors. + */ + +import {execFileSync, spawnSync} from 'child_process'; +import {readFileSync} from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const args = process.argv.slice(2); +const BASE = flagValue('--base'); +const FORMAT = flagValue('--format') ?? 'text'; + +function flagValue(name) { + const index = args.indexOf(name); + return index === -1 ? undefined : args[index + 1]; +} + +function addedLineRanges(base) { + const diff = execFileSync( + 'git', + ['diff', '--unified=0', '--no-color', '--find-renames', base, '--'], + {encoding: 'utf8', maxBuffer: 50 * 1024 * 1024} + ); + const ranges = new Map(); + let file; + + for (const line of diff.split('\n')) { + const fileMatch = line.match(/^\+\+\+ b\/(.+)$/); + if (fileMatch) { + file = fileMatch[1]; + if (!ranges.has(file)) { + ranges.set(file, []); + } + continue; + } + + const hunkMatch = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/); + if (file && hunkMatch) { + const start = Number(hunkMatch[1]); + const count = hunkMatch[2] === undefined ? 1 : Number(hunkMatch[2]); + if (count > 0) { + ranges.get(file).push([start, start + count - 1]); + } + } + } + + return ranges; +} + +function runCSpell(files) { + if (files.length === 0) { + return []; + } + + const result = spawnSync( + 'cspell', + [ + '--no-progress', + '--show-suggestions', + '--reporter', + '@cspell/cspell-json-reporter', + '--file', + ...files + ], + {encoding: 'utf8', maxBuffer: 50 * 1024 * 1024} + ); + + if (result.error) { + throw result.error; + } + + const report = JSON.parse(result.stdout); + if (![0, 1].includes(result.status) || report.error.length > 0) { + throw new Error(result.stderr || JSON.stringify(report.error)); + } + + return report.issues.map(issue => ({ + file: path.relative(process.cwd(), fileURLToPath(issue.uri)), + line: issue.row, + column: issue.col, + word: issue.text, + suggestions: issue.suggestions?.slice(0, 3) ?? [], + text: issue.line.text.replace(/\r?\n$/, '') + })); +} + +// Entries in the dictionary files must be sorted, so duplicates stand out and +// merges are clean. Sorted the way CSpell matches: case- and accent-insensitive. +// Blank lines and comments start a new sorted run, so the lists can be sectioned. +const DICTIONARY_FILES = ['cspell-allow-list.txt', 'cspell-block-list.txt']; +const collator = new Intl.Collator('en', {sensitivity: 'base'}); + +function unsortedDictionaryEntries(files) { + const findings = []; + for (const file of files.filter(file => DICTIONARY_FILES.includes(file))) { + let run = []; // [{word, line}] of the current sorted run + readFileSync(file, 'utf8') + .split('\n') + .forEach((text, index) => { + if (/^\s*(#|$)/.test(text)) { + run = []; + return; + } + const word = text.replace(/\s*#.*/, '').trim(); + const line = index + 1; + const belongsBefore = run.find( + entry => collator.compare(word, entry.word) < 0 + ); + if (belongsBefore) { + findings.push({ + file, + line, + column: 1, + word, + suggestions: [], + text, + message: `\`${word}\` is out of alphabetical order: move it above \`${belongsBefore.word}\``, + relatedLine: belongsBefore.line // rendered as a link after the message + }); + } else { + run.push({word, line}); + } + }); + } + return findings; +} + +function findingsOnAddedLines(ranges, issues) { + return issues.filter(issue => + (ranges.get(issue.file) ?? []).some( + ([start, end]) => issue.line >= start && issue.line <= end + ) + ); +} + +function formatText(findings) { + if (findings.length === 0) { + return 'No issues found in added lines.\n'; + } + + const lines = [ + `Found ${findings.length} issue(s) in added lines:` + ]; + for (const {file, line, column, word, message, relatedLine} of findings) { + const detail = message + ? `${message}${relatedLine ? ` on line ${relatedLine}` : ''}` + : `Unknown word (${word})`; + lines.push(`${file}:${line}:${column} - ${detail}`); + } + return lines.join('\n') + '\n'; +} + +async function main() { + if (!BASE) { + throw new Error('Missing required --base '); + } + if (!['text', 'json'].includes(FORMAT)) { + throw new Error(`Unknown --format "${FORMAT}"; use text or json`); + } + + const ranges = addedLineRanges(BASE); + const files = [...ranges.keys()]; + const findings = findingsOnAddedLines(ranges, [ + ...runCSpell(files), + ...unsortedDictionaryEntries(files) + ]); + process.stdout.write( + FORMAT === 'json' + ? JSON.stringify(findings, null, '\t') + '\n' + : formatText(findings) + ); + process.exit(findings.length === 0 ? 0 : 1); +} + +main().catch(error => { + console.error(error); + process.exit(2); +}); diff --git a/dev/checks.mjs b/dev/checks.mjs new file mode 100644 index 000000000..c947a9521 --- /dev/null +++ b/dev/checks.mjs @@ -0,0 +1,59 @@ +#!/usr/bin/env node + +/** + * Runs the docs checks in dev/check-*.mjs. `npm run build` runs them all + * before `next build`. + * + * Usage: node dev/checks.mjs [check ...] [flags] + * node dev/checks.mjs every check + * node dev/checks.mjs links filenames only those + * node dev/checks.mjs links --check-anchors flags go to that one check + * + * Every check runs even when an earlier one fails; exits 1 if any failed. + */ + +import {spawnSync} from 'child_process'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const CHECKS = { + links: 'check-links.mjs', + filenames: 'check-filenames.mjs', + images: 'check-images.mjs' +}; + +const args = process.argv.slice(2); +const names = []; +while (args.length > 0 && CHECKS[args[0]]) { + names.push(args.shift()); +} +const flags = args; + +if (flags.length > 0 && !flags[0].startsWith('-')) { + console.error( + `Unknown check "${flags[0]}"; use ${Object.keys(CHECKS).join(', ')}` + ); + process.exit(1); +} +if (flags.length > 0 && names.length !== 1) { + console.error( + `Flags ${flags.join(' ')} need exactly one check to apply to` + ); + process.exit(1); +} + +const failed = []; +for (const name of names.length > 0 ? names : Object.keys(CHECKS)) { + const script = path.join(__dirname, CHECKS[name]); + const {status} = spawnSync(process.execPath, [script, ...flags], { + stdio: 'inherit' + }); + if (status !== 0) failed.push(name); +} + +if (failed.length > 0) { + console.error(`\n❌ Failed checks: ${failed.join(', ')}`); + process.exit(1); +} diff --git a/dev/post-spelling-review.mjs b/dev/post-spelling-review.mjs new file mode 100644 index 000000000..94eacb399 --- /dev/null +++ b/dev/post-spelling-review.mjs @@ -0,0 +1,318 @@ +#!/usr/bin/env node + +/** + * Reports CSpell findings on a pull request: one summary comment in the + * discussion, plus an inline review comment on each flagged line. Once the + * findings are fixed, the summary is minimized as resolved and the inline + * comments are deleted; the review that carried them has no body, so nothing + * of it remains visible. + * + * Usage: node dev/post-spelling-review.mjs --findings [--dry-run] + * + * Reads the JSON written by `dev/check-spelling.mjs --format json`. + * Requires GH_TOKEN, GITHUB_REPOSITORY, PR_NUMBER, HEAD_SHA and HEAD_REF. + */ + +import {readFileSync} from 'fs'; + +const args = process.argv.slice(2); +const FINDINGS_FILE = args[args.indexOf('--findings') + 1]; +const DRY_RUN = args.includes('--dry-run'); +const MAX_INLINE_COMMENTS = 25; + +const API_URL = process.env.GITHUB_API_URL ?? 'https://api.github.com'; +const REPOSITORY = process.env.GITHUB_REPOSITORY; +const PR_NUMBER = process.env.PR_NUMBER; +const HEAD_SHA = process.env.HEAD_SHA; +const HEAD_REF = process.env.HEAD_REF; + +// Link to the PR branch, not the commit, so GitHub's edit button works from it +const ALLOW_LIST_LINK = `[\`cspell-allow-list.txt\`](https://github.com/${REPOSITORY}/blob/${HEAD_REF}/cspell-allow-list.txt)`; + +const SUMMARY_MARKER = ''; +const INLINE_MARKER = '/)?.[1]; + return word && findingKey({file: comment.path, line: comment.line, word}); +} + +// CSpell suggests case-insensitively, so prefer a suggestion whose first +// letter matches the case of the flagged word. +function bestSuggestion({word, suggestions}) { + const isUpper = letter => letter === letter.toUpperCase(); + return ( + suggestions.find( + suggestion => isUpper(suggestion[0]) === isUpper(word[0]) + ) ?? suggestions[0] + ); +} + +// One GitHub suggestion block with an apply button. A four-backtick fence so +// lines containing ``` cannot break out of the block. +function suggestionBlock(finding) { + const suggestion = bestSuggestion(finding); + if (!suggestion) { + return []; + } + const {text, column, word} = finding; + const start = column - 1; + return [ + `Did you mean \`${suggestion}\`?`, + '', + '````suggestion', + text.slice(0, start) + suggestion + text.slice(start + word.length), + '````', + '' + ]; +} + +function inlineBody(finding) { + const explanation = finding.message + ? [messageText(finding)] + : [ + `\`${finding.word}\` is not in the dictionary.`, + '', + ...suggestionBlock(finding), + `Please correct the spelling, or add the word to ${ALLOW_LIST_LINK} if it is correct.` + ]; + return [`${INLINE_MARKER} ${finding.word} -->`, ...explanation].join('\n'); +} + +async function syncInlineComments(findings) { + const wanted = new Map( + findings.map(finding => [findingKey(finding), finding]) + ); + const comments = await githubList( + `/repos/${REPOSITORY}/pulls/${PR_NUMBER}/comments` + ); + + for (const comment of comments) { + const key = existingCommentKey(comment); + if (!key) { + continue; + } + if (wanted.has(key)) { + wanted.delete(key); + } else { + await githubWrite( + 'DELETE', + `/repos/${REPOSITORY}/pulls/comments/${comment.id}` + ); + } + } + + const fresh = [...wanted.values()]; + if (fresh.length === 0) { + return; + } + // No review body: the inline comments say it all, and a submitted review + // cannot be deleted, so a body would outlive the comments once fixed. + await githubWrite( + 'POST', + `/repos/${REPOSITORY}/pulls/${PR_NUMBER}/reviews`, + { + commit_id: HEAD_SHA, + event: 'COMMENT', + body: '', + comments: fresh.slice(0, MAX_INLINE_COMMENTS).map(finding => ({ + path: finding.file, + line: finding.line, + side: 'RIGHT', + body: inlineBody(finding) + })) + } + ); +} + +async function main() { + for (const name of [ + 'GH_TOKEN', + 'GITHUB_REPOSITORY', + 'PR_NUMBER', + 'HEAD_SHA', + 'HEAD_REF' + ]) { + if (!process.env[name]) { + throw new Error(`Missing required environment variable ${name}`); + } + } + if (!FINDINGS_FILE) { + throw new Error('Missing required --findings '); + } + + const findings = JSON.parse(readFileSync(FINDINGS_FILE, 'utf8')); + console.log(`${findings.length} finding(s) to report`); + await upsertSummaryComment(findings); + await syncInlineComments(findings); +} + +main().catch(error => { + console.error(error); + process.exit(2); +}); diff --git a/dev/report-vercel-build.mjs b/dev/report-vercel-build.mjs new file mode 100644 index 000000000..caddf06d7 --- /dev/null +++ b/dev/report-vercel-build.mjs @@ -0,0 +1,498 @@ +#!/usr/bin/env node + +/** + * Reports a failed Vercel build on its pull request, since Vercel only shows + * build logs to members of the Vercel team. The log itself goes to Slack, so + * anything sensitive a build prints stays inside the workspace instead of a + * public PR; the PR comment only links to it. When a later revision builds, + * the same comment is updated to say so. + * + * Usage: + * node dev/report-vercel-build.mjs fetch-log + * node dev/report-vercel-build.mjs slack [--dry-run] + * node dev/report-vercel-build.mjs comment [--dry-run] + * + * fetch-log writes the build log to , and `pull_request`, the PR Vercel + * built the deployment for, to GITHUB_OUTPUT. It needs VERCEL_TOKEN, and + * VERCEL_TEAM_ID unless the token is scoped to the project. + * + * slack uploads into the thread of the Vercel Slack app's "failed to + * deploy" post for the deployment in SLACK_CHANNEL_ID, looking back a week (so + * a re-run by hand still finds it) and waiting up to 5 minutes for the post + * to appear, then writes the reply's `permalink` to GITHUB_OUTPUT. It needs + * SLACK_BOT_TOKEN (see dev/slack-app-vercel-build-report.json) and does + * nothing when that or SLACK_CHANNEL_ID is unset. With --dry-run the post is + * found but nothing is uploaded. + * + * comment posts on PR_NUMBER, the PR Vercel built the deployment for. When + * unset (the success path, or a deployment Vercel recorded no PR for) it falls + * back to every open PR at COMMIT_SHA. A failure comment links + * SLACK_PERMALINK, or the channel when the upload did not happen. With + * --dry-run the comment is printed instead. + * + * All need DEPLOYMENT_ID, DEPLOYMENT_STATE (failed or success), COMMIT_SHA, + * GH_TOKEN and GITHUB_REPOSITORY. + */ + +import {appendFileSync, readFileSync, writeFileSync} from 'fs'; + +const [command, logFile] = process.argv + .slice(2) + .filter(argument => !argument.startsWith('--')); +const DRY_RUN = process.argv.includes('--dry-run'); +// A week, so a re-run by hand finds the post; the deployment ID match is +// exact, so the wider window cannot pick a wrong post +const SLACK_HISTORY_DAYS = 7; +const SLACK_WAIT_MINUTES = 5; +const SLACK_POLL_SECONDS = 15; + +const API_URL = process.env.GITHUB_API_URL ?? 'https://api.github.com'; +const REPOSITORY = process.env.GITHUB_REPOSITORY; +const {DEPLOYMENT_ID, DEPLOYMENT_STATE, COMMIT_SHA, SLACK_CHANNEL_ID} = + process.env; + +// Comments from before the log moved to Slack carry an artifact ID here +const MARKER = ''; +const MARKER_PATTERN = /^/; + +async function fetchJson(url, headers) { + const response = await fetch(url, {headers}); + if (!response.ok) { + throw new Error( + `GET ${url} failed: ${response.status} ${await response.text()}` + ); + } + return response.json(); +} + +async function github(method, route, body) { + const response = await fetch(`${API_URL}${route}`, { + method, + headers: { + authorization: `Bearer ${process.env.GH_TOKEN}`, + accept: 'application/vnd.github+json', + 'x-github-api-version': '2022-11-28', + ...(body && {'content-type': 'application/json'}) + }, + body: body && JSON.stringify(body) + }); + if (!response.ok) { + throw new Error( + `${method} ${route} failed: ${response.status} ${await response.text()}` + ); + } + return response.status === 204 ? undefined : response.json(); +} + +async function githubList(route) { + const items = []; + for (let page = 1; ; page++) { + const batch = await github('GET', `${route}?per_page=100&page=${page}`); + items.push(...batch); + if (batch.length < 100) { + return items; + } + } +} + +// Vercel records which PR a deployment was built for. Empty when the branch +// was deployed before its PR was opened. +async function fetchDeploymentPullRequestNumber() { + const url = new URL( + `https://api.vercel.com/v13/deployments/${DEPLOYMENT_ID}` + ); + if (process.env.VERCEL_TEAM_ID) { + url.searchParams.set('teamId', process.env.VERCEL_TEAM_ID); + } + const deployment = await fetchJson(url, { + authorization: `Bearer ${process.env.VERCEL_TOKEN}` + }); + // The deployment ID and commit arrive as separate inputs; only publish + // the log of the deployment Vercel built from that commit + const builtSha = + deployment.meta?.githubCommitSha ?? deployment.gitSource?.sha; + if (builtSha !== COMMIT_SHA) { + throw new Error( + `Deployment ${DEPLOYMENT_ID} was built from ${builtSha}, not ${COMMIT_SHA}` + ); + } + return deployment.meta?.githubPrId; +} + +// The dispatch payload has no PR number. The failure path gets it from the +// deployment; the success path only knows the commit, so it looks up the PRs +// at that head. Either way a stale event for a commit a PR has moved past is +// ignored, and so are fork PRs. +async function findPullRequests(pullRequestNumber) { + const pulls = pullRequestNumber + ? [ + await github( + 'GET', + `/repos/${REPOSITORY}/pulls/${pullRequestNumber}` + ) + ] + : await github( + 'GET', + `/repos/${REPOSITORY}/commits/${COMMIT_SHA}/pulls` + ); + const open = pulls.filter(pull => { + if (pull.state !== 'open' || pull.head.sha !== COMMIT_SHA) { + return false; + } + // head.repo is null when the fork was deleted + if (pull.head.repo?.full_name !== REPOSITORY) { + console.log(`PR #${pull.number} is from a fork; not reporting`); + return false; + } + return true; + }); + if (open.length === 0) { + console.log(`No open PR with head ${COMMIT_SHA}; nothing to do`); + } + return open; +} + +// Build log lines, oldest first. Vercel keeps them as events; only the ones +// with text are log lines. +async function fetchBuildLog() { + const url = new URL( + `https://api.vercel.com/v3/deployments/${DEPLOYMENT_ID}/events` + ); + url.searchParams.set('limit', '-1'); + url.searchParams.set('direction', 'forward'); + if (process.env.VERCEL_TEAM_ID) { + url.searchParams.set('teamId', process.env.VERCEL_TEAM_ID); + } + const events = await fetchJson(url, { + authorization: `Bearer ${process.env.VERCEL_TOKEN}` + }); + return events + .map(event => event.payload?.text ?? event.text) + .filter(text => typeof text === 'string') + .flatMap(text => text.replace(/\n$/, '').split('\n')) + .map(redact); +} + +// Credential shapes a build might print. The log only goes to Slack, but the +// build gets VERCEL_OIDC_TOKEN and friends, so a left-in +// `console.log(process.env)` should still not hand them to the whole channel. +// Not a complete list. +// cspell:disable -- token prefixes, not words +const REDACTION_PATTERNS = [ + [/\beyJ[\w-]{10,}\.[\w-]{10,}\.[\w-]+/g, '[redacted-jwt]'], + [ + /\b(?:vcp_|gh[pousr]_|github_pat_|sk-|xox[abpr]-)[\w-]{16,}|\bAKIA[0-9A-Z]{16}\b/g, + '[redacted-token]' + ], + [/(\bBearer\s+)\S+/gi, '$1[redacted]'], + [ + /(\w*(?:TOKEN|SECRET|PASSW(?:OR)?D|CREDENTIALS?|API_?KEY|PRIVATE_KEY|ENC_KEY|DEPLOYMENT_KEY)\w*["']?\s*[=:]\s*["']?)\S+/gi, + '$1[redacted]' + ] +]; +// cspell:enable + +function redact(line) { + return REDACTION_PATTERNS.reduce( + (text, [pattern, replacement]) => text.replace(pattern, replacement), + line + ); +} + +function writeOutput(name, value) { + if (process.env.GITHUB_OUTPUT) { + appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`); + } +} + +async function fetchLog() { + if (!process.env.VERCEL_TOKEN) { + throw new Error('VERCEL_TOKEN is required to read the build log'); + } + // Ask GitHub before Vercel, so the Vercel token is only ever used for a + // commit that an open PR from this repository is at, i.e. pushed by + // someone who can already push here + if ((await findPullRequests()).length === 0) { + return; + } + const pullRequestNumber = await fetchDeploymentPullRequestNumber(); + const logLines = await fetchBuildLog(); + writeFileSync(logFile, logLines.join('\n') + '\n'); + console.log(`Wrote ${logLines.length} log lines to ${logFile}`); + writeOutput('pull_request', pullRequestNumber ?? ''); +} + +// The log stays in Slack, where only the workspace can read it; the public +// comment says where to look +function failureBody() { + const {SLACK_PERMALINK} = process.env; + const where = SLACK_PERMALINK + ? `[attached to its Slack post](${SLACK_PERMALINK})` + : 'in Slack'; + return [ + MARKER, + '### ❌ The Vercel build failed for this PR', + '', + `Vercel only shows build logs to members of its team, so the build log is ${where} in #alerts-vercel-doc-site.`, + '' + ].join('\n'); +} + +async function comment() { + const pulls = await findPullRequests(process.env.PR_NUMBER); + for (const pull of pulls) { + await report(pull); + } +} + +// Comment only when the build failed, or an earlier failure is resolved +async function report(pull) { + const comments = await githubList( + `/repos/${REPOSITORY}/issues/${pull.number}/comments` + ); + const existing = comments.find(comment => + MARKER_PATTERN.test(comment.body) + ); + + let body; + if (DEPLOYMENT_STATE === 'failed') { + body = failureBody(); + } else if (existing) { + body = `${MARKER}\n### ✅ The Vercel build that failed on an earlier revision of this PR passes\n`; + } else { + console.log(`PR #${pull.number} has no failed build to resolve`); + return; + } + + if (DRY_RUN) { + console.log( + `[dry-run] would ${existing ? 'update' : 'create'} comment on PR #${pull.number}:\n` + ); + console.log(body); + } else if (existing) { + console.log(`Updating comment ${existing.id} on PR #${pull.number}`); + await github( + 'PATCH', + `/repos/${REPOSITORY}/issues/comments/${existing.id}`, + {body} + ); + } else { + console.log(`Commenting on PR #${pull.number}`); + await github( + 'POST', + `/repos/${REPOSITORY}/issues/${pull.number}/comments`, + {body} + ); + } +} + +// Every method used here takes form encoding, including the file one +async function slackApi(method, parameters) { + const response = await fetch(`https://slack.com/api/${method}`, { + method: 'POST', + headers: { + authorization: `Bearer ${process.env.SLACK_BOT_TOKEN}`, + 'content-type': 'application/x-www-form-urlencoded' + }, + body: new URLSearchParams(parameters) + }); + const result = await response.json(); + if (!result.ok) { + throw new Error(`Slack ${method} failed: ${result.error}`); + } + return result; +} + +// Every string in a Slack message: the top-level text, plus legacy +// attachments and Block Kit blocks, where apps often put the real content +function slackMessageText(message) { + const strings = []; + const collect = value => { + if (typeof value === 'string') { + strings.push(value); + } else if (Array.isArray(value)) { + value.forEach(collect); + } else if (value && typeof value === 'object') { + Object.values(value).forEach(collect); + } + }; + collect([message.text, message.attachments, message.blocks]); + return strings.join('\n'); +} + +// The Vercel Slack app posts " failed to deploy" for each +// failed deployment, with the short SHA in a context block and an Inspect +// button whose URL ends in the deployment ID. The ID is matched, since two +// PRs at one commit get two deployments and two posts. The app and this +// workflow are triggered by the same event, so its post can land after this +// runs; keep looking for a while before giving up. +async function findVercelFailurePost() { + const deploymentId = DEPLOYMENT_ID.replace(/^dpl_/, ''); + const oldest = Date.now() / 1000 - SLACK_HISTORY_DAYS * 24 * 60 * 60; + const deadline = Date.now() + SLACK_WAIT_MINUTES * 60_000; + for (;;) { + // Newest first, a page at a time + const seen = []; + for (let cursor; ; ) { + const {messages, response_metadata: metadata} = await slackApi( + 'conversations.history', + { + channel: SLACK_CHANNEL_ID, + oldest, + limit: 200, + ...(cursor && {cursor}) + } + ); + const post = messages.find(message => { + const text = slackMessageText(message); + return ( + text.includes('failed to deploy') && + text.includes(deploymentId) + ); + }); + if (post) { + return post; + } + seen.push(...messages); + cursor = metadata?.next_cursor; + if (!cursor) { + break; + } + } + if (Date.now() >= deadline) { + console.log( + `No Vercel "failed to deploy" post for ${DEPLOYMENT_ID} in the last ${SLACK_HISTORY_DAYS} days; giving up. Newest messages seen:` + ); + for (const message of seen.slice(0, 20)) { + console.log( + ` ${message.ts} bot_id=${message.bot_id ?? '-'} user=${message.user ?? '-'} subtype=${message.subtype ?? '-'} ${JSON.stringify(slackMessageText(message).slice(0, 120))}` + ); + } + return undefined; + } + console.log( + `No Vercel post for ${DEPLOYMENT_ID} yet; checking again in ${SLACK_POLL_SECONDS}s` + ); + await new Promise(resolve => + setTimeout(resolve, SLACK_POLL_SECONDS * 1000) + ); + } +} + +// Slack takes files in three steps: ask for an upload URL, POST the bytes to +// it, then say which channel and thread to share the file in. Returns the +// permalink of the reply carrying the file, for the PR comment. +async function uploadLogToThread(post, pulls) { + const log = readFileSync(logFile); + const shortSha = COMMIT_SHA.slice(0, 7); + // .txt, so Slack shows it inline instead of offering a download + const filename = `vercel-build-${shortSha}.txt`; + const links = pulls + .map(pull => `<${pull.html_url}|#${pull.number}>`) + .join(', '); + const initialComment = `Build log attached; PR ${links} links here.`; + + if (DRY_RUN) { + console.log( + `[dry-run] would upload ${filename} (${log.length} bytes) to thread ${post.ts} in ${SLACK_CHANNEL_ID}:\n${initialComment}` + ); + return undefined; + } + const {upload_url: uploadUrl, file_id: fileId} = await slackApi( + 'files.getUploadURLExternal', + {filename, length: log.length} + ); + const upload = await fetch(uploadUrl, {method: 'POST', body: log}); + if (!upload.ok) { + throw new Error( + `Uploading ${filename} to Slack failed: ${upload.status} ${await upload.text()}` + ); + } + await slackApi('files.completeUploadExternal', { + files: JSON.stringify([ + {id: fileId, title: `Vercel build log for ${shortSha}`} + ]), + channel_id: SLACK_CHANNEL_ID, + thread_ts: post.ts, + initial_comment: initialComment + }); + console.log( + `Uploaded ${filename} to thread ${post.ts} in ${SLACK_CHANNEL_ID}` + ); + + // The upload response names only the file, so find the reply it made; + // fall back to the post itself rather than leave the PR without a link + const {messages} = await slackApi('conversations.replies', { + channel: SLACK_CHANNEL_ID, + ts: post.ts + }); + const reply = messages.find(message => + message.files?.some(file => file.id === fileId) + ); + const {permalink} = await slackApi('chat.getPermalink', { + channel: SLACK_CHANNEL_ID, + message_ts: reply?.ts ?? post.ts + }); + return permalink; +} + +async function slack() { + if (!process.env.SLACK_BOT_TOKEN || !SLACK_CHANNEL_ID) { + console.log( + 'SLACK_BOT_TOKEN or SLACK_CHANNEL_ID unset; not posting to Slack' + ); + return; + } + if (DEPLOYMENT_STATE !== 'failed') { + console.log('The build passed; Vercel already posts that to Slack'); + return; + } + // The same guard as fetch-log and comment, so Slack only ever gets logs + // for commits an open PR from this repository is at + const pulls = await findPullRequests(process.env.PR_NUMBER); + if (pulls.length === 0) { + return; + } + const post = await findVercelFailurePost(); + if (post) { + const permalink = await uploadLogToThread(post, pulls); + if (permalink) { + writeOutput('permalink', permalink); + } + } +} + +async function main() { + for (const name of [ + 'DEPLOYMENT_ID', + 'DEPLOYMENT_STATE', + 'COMMIT_SHA', + 'GH_TOKEN', + 'GITHUB_REPOSITORY' + ]) { + if (!process.env[name]) { + throw new Error(`Missing required environment variable ${name}`); + } + } + if (!['failed', 'success'].includes(DEPLOYMENT_STATE)) { + throw new Error(`Unexpected DEPLOYMENT_STATE ${DEPLOYMENT_STATE}`); + } + const usage = + 'Usage: node dev/report-vercel-build.mjs fetch-log|slack , or comment'; + if (command === 'comment') { + await comment(); + } else if (!logFile) { + throw new Error(usage); + } else if (command === 'fetch-log') { + await fetchLog(); + } else if (command === 'slack') { + await slack(); + } else { + throw new Error(usage); + } +} + +main().catch(error => { + console.error(error); + process.exit(2); +}); diff --git a/dev/slack-app-vercel-build-report.json b/dev/slack-app-vercel-build-report.json new file mode 100644 index 000000000..ebdfc7c3b --- /dev/null +++ b/dev/slack-app-vercel-build-report.json @@ -0,0 +1,22 @@ +{ + "display_information": { + "name": "Vercel build log", + "description": "Attaches failed Vercel build logs to the Vercel app's posts" + }, + "features": { + "bot_user": { + "display_name": "Vercel build log", + "always_online": false + } + }, + "oauth_config": { + "scopes": { + "bot": ["channels:history", "files:write"] + } + }, + "settings": { + "org_deploy_enabled": false, + "socket_mode_enabled": false, + "token_rotation_enabled": false + } +} diff --git a/dev/sync-review-comments.sh b/dev/sync-review-comments.sh new file mode 100755 index 000000000..6ecd7b806 --- /dev/null +++ b/dev/sync-review-comments.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env sh +# Make a check's suggested-change review comments on a PR match a review.json +# ({comments: [{path, line, start_line?, body}]}): post the new ones, update +# the ones whose text changed, and delete the ones whose finding is gone. +# GitHub sets line to null on comments it could not carry to the new revision, +# so those are deleted too. Comments are matched by file, line, and the +# marker comment on their first line, e.g. "". +# +# Usage: dev/sync-review-comments.sh '