From 24daf21ed086dce3d9f43f931c3c2a951220ede2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 06:23:22 +0000 Subject: [PATCH 1/2] feat: add macOS arm64 self-hosted runners to the wavekat-ci pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `runs-on: [self-hosted, wavekat-ci]` is a label pool, not a machine, so adding the Mac mini is a registration problem rather than a workflow one. Add the macOS twin of the Linux setup/uninstall scripts, registering with the same `wavekat-ci` label so jobs land on whichever host is idle. macOS forces three differences from the Linux script: the osx-arm64 runner package, a launchd LaunchAgent instead of a systemd unit (so runners live under $HOME and need no sudo), and a per-runner `.path` file, since launchd does not source the shell profile and Homebrew would otherwise be invisible. A mixed pool also means any job can land on BSD userland, so `preview.yml` loses its `grep -oP` — BSD grep has no PCRE, and that step would have failed every time the deploy job landed on the Mac. `docs/06` records the rest of the portability traps and the headless-Mac requirements (auto-login, no sleep) that keep the agents online across reboots. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TxidDNZBX6a7YqiPdmjf6C --- .github/workflows/preview.yml | 7 +- CLAUDE.md | 8 + docs/06-self-hosted-runners.md | 145 ++++++++++++++++++ scripts/setup-gha-runners-macos.sh | 201 +++++++++++++++++++++++++ scripts/uninstall-gha-runners-macos.sh | 68 +++++++++ 5 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 docs/06-self-hosted-runners.md create mode 100755 scripts/setup-gha-runners-macos.sh create mode 100755 scripts/uninstall-gha-runners-macos.sh diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 3f4cc59..cad5fb5 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -44,7 +44,12 @@ jobs: run: | OUTPUT=$(npx wrangler pages deploy ./dist --project-name=wavekat-com --branch="${{ github.head_ref || github.ref_name }}" 2>&1) echo "$OUTPUT" - ALIAS=$(echo "$OUTPUT" | grep -oP '(?<=Deployment alias URL: )https://\S+') + # POSIX sed, not `grep -oP`: this job also runs on the macOS + # runners, whose BSD grep has no -P. Keep every `run:` block in + # this repo portable across the pool — see docs/06. + ALIAS=$(printf '%s\n' "$OUTPUT" \ + | sed -nE 's|.*Deployment alias URL: (https://[^[:space:]]+).*|\1|p' \ + | tail -n 1) echo "url=${ALIAS}" >> "$GITHUB_OUTPUT" env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} diff --git a/CLAUDE.md b/CLAUDE.md index 834318b..029e34a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -55,6 +55,14 @@ This repo uses **release-please**. Since GitHub squash-merges use the PR title a - **Deployment**: Cloudflare Pages (consistent with rest of org) - **Domain**: `wavekat.com` — DNS to be pointed at Cloudflare Pages once site is ready +## CI runs on a mixed self-hosted pool — keep every `run:` block portable + +All workflows use `runs-on: [self-hosted, wavekat-ci]`. That label is a **pool**, not a machine: a Linux x86-64 workstation *and* a macOS arm64 Mac mini both carry it, so any job lands on either host non-deterministically. Setup lives in `scripts/setup-gha-runners.sh` (Linux, systemd), `scripts/setup-gha-runners-docker.sh` (Linux, containerised) and `scripts/setup-gha-runners-macos.sh` (macOS, launchd); the full story is `docs/06-self-hosted-runners.md`. + +The practical consequence: **macOS ships BSD userland, so GNU-only shell in a `run:` block fails about half the time, on PRs that changed nothing.** No `grep -oP`, no bare `sed -i`, no `readlink -f`, no `date -d`, no `sha256sum`, no `xargs -r`, and no bash 4+ syntax (macOS `/bin/bash` is 3.2). Docker-based actions can't run on macOS at all — every action we use must be a JavaScript action. And `npm ci` needs the `darwin-arm64` optional deps to stay in `package-lock.json`, so never regenerate the lockfile with `--no-optional`. + +To pin a job to one host, add that host's automatic label: `runs-on: [self-hosted, wavekat-ci, macOS]` or `..., Linux]`. + ## SEO & GEO — every new page must be both This site is optimized for classic search (SEO) **and** generative answer engines (GEO — being quoted by ChatGPT, Perplexity, Google AI Overviews, Claude). The two overlap but aren't identical: SEO wants crawlable, well-described, linkable pages; GEO wants self-contained, factual, extractable passages an LLM can lift verbatim. Build for both on every page. diff --git a/docs/06-self-hosted-runners.md b/docs/06-self-hosted-runners.md new file mode 100644 index 0000000..03ea4ac --- /dev/null +++ b/docs/06-self-hosted-runners.md @@ -0,0 +1,145 @@ +# 06 — Self-hosted runners: adding the Mac mini + +Every workflow in this repo runs on `runs-on: [self-hosted, wavekat-ci]`. Until +now that label existed on exactly one machine — a Linux x86-64 box set up by +`scripts/setup-gha-runners.sh`. This doc adds a Mac mini (Apple Silicon, +arm64) to the same pool, and records the one rule that keeps a mixed-OS pool +from breaking CI at random. + +## 1. How the pool actually works + +`runs-on` is a **label set**, not a machine. A job runs on any idle runner that +carries *all* the listed labels. So the whole of "how do we run on the Mac too" +is: register the Mac with the `wavekat-ci` label. Nothing in +`.github/workflows/` changes. + +Each runner also gets automatic labels it never asked for, which is how you +pin work to one host when you need to: + +| Host | Automatic labels | Labels we add | +|------|------------------|---------------| +| Linux workstation | `self-hosted`, `Linux`, `X64` | `wavekat-ci`, `` | +| Mac mini | `self-hosted`, `macOS`, `ARM64` | `wavekat-ci`, `` | + +Both scripts default `RUNNER_LABELS` to `wavekat-ci,`, so after setup +you can force a job onto one machine with `runs-on: [self-hosted, wavekat-ci, macOS]` +without disturbing anything else. + +Jobs are **not** load-balanced by cost or speed. GitHub hands a queued job to +the first idle matching runner, so with both hosts registered a given PR's CI +may land on Linux one run and macOS the next. That is the point — and it is +also why section 3 matters. + +## 2. Setting up the Mac mini + +`scripts/setup-gha-runners-macos.sh` is the macOS twin of the Linux script and +takes the same environment variables (`RUNNER_ORG`, `RUNNER_COUNT`, +`RUNNER_PREFIX`, `RUNNER_BASE_DIR`, `RUNNER_LABELS`, `RUNNER_VERSION`, +`RUNNER_TOKEN`). On the Mac: + +```sh +xcode-select --install # git — actions/checkout shells out to it +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +brew install gh && gh auth login # an account with wavekat org admin + +git clone git@github.com-wavekat:wavekat/wavekat.com.git +cd wavekat.com +RUNNER_KEEP_AWAKE=1 ./scripts/setup-gha-runners-macos.sh +``` + +Tear down with `./scripts/uninstall-gha-runners-macos.sh` (same variables). + +What the script does differently from the Linux one, and why: + +- **`actions-runner-osx-arm64`**, resolved to the latest release. There is a + native Apple Silicon build; do not run the x64 one under Rosetta. +- **launchd, not systemd.** `svc.sh install` on macOS writes a *LaunchAgent* + to `~/Library/LaunchAgents/` and runs as your user — no `sudo`, and runners + therefore live under `$HOME/actions-runners` instead of `/opt`. +- **A `.path` file per runner.** launchd does not source `~/.zprofile`, so + `/opt/homebrew/bin` is invisible to the runner process unless we write it + into the runner's `.path`. (`actions/setup-node` prepends its own Node ahead + of this, so this is about `git`, `gh`, `jq` and anything a job shells out to.) +- **Quarantine clearing.** A curl-fetched tarball carries no + `com.apple.quarantine`, but a browser-downloaded one does and Gatekeeper + then kills the runner binaries. The `xattr -dr` is a no-op in the normal path. + +### The headless-Mac gotcha + +A LaunchAgent needs a GUI login session. On a Mac mini with no one logged in, +`launchctl` will refuse to load the agent and the runner shows offline after +every reboot. Two settings fix it permanently: + +1. **System Settings → Users & Groups → Automatic login** → the runner user. + Without this, runners do not come back after a power cut or an OS update. +2. **No sleep.** `RUNNER_KEEP_AWAKE=1` runs + `systemsetup -setcomputersleep Never` and `pmset -a disksleep 0 womp 1` for + you. A sleeping Mac does not pick up queued jobs; the job just sits there + until the Linux box frees up, which hides the problem instead of failing. + +Run the setup script the first time from a Screen Sharing session, not a bare +SSH session. If it prints the "could not start via launchctl" warning, that is +what happened — log in and run `./svc.sh start` in the runner dir. + +### How many runners + +`RUNNER_COUNT` defaults to 4, matching the Linux host. A full `npm run cf:build` +of this site is a real Astro/Rolldown build; four in parallel on a 16 GB Mac +mini is the practical ceiling. Drop to `RUNNER_COUNT=2` if builds start +swapping. + +## 3. The rule: every `run:` block must be portable + +This is the part that bites. A mixed pool means **any** job can land on either +OS, so a shell script that only works on GNU userland fails roughly half the +time, non-deterministically, on PRs that changed nothing. + +macOS ships BSD userland, not GNU. The traps that apply to this repo: + +| Don't | Do | Why | +|-------|----|-----| +| `grep -oP '(?<=x)y'` | `sed -nE 's/.*x(y).*/\1/p'` | BSD grep has no `-P` (PCRE) | +| `sed -i 's/a/b/' f` | `sed -i.bak` then `rm f.bak` | BSD `sed -i` requires a suffix arg | +| `readlink -f p` | `cd "$(dirname p)" && pwd -P` | BSD `readlink` has no `-f` | +| `date -d '1 day ago'` | `date -u +%s` and do maths | BSD `date` uses `-v-1d` | +| `sha256sum f` | `shasum -a 256 f` | `sha256sum` is GNU coreutils | +| `xargs -r` | guard with `[ -s file ]` | BSD `xargs` has no `-r` | +| bash 4+ syntax (`${v,,}`, `declare -A`, `mapfile`) | bash 3.2 equivalents | macOS `/bin/bash` is 3.2 | + +Fixing `grep -oP` in `preview.yml` was the only change this migration needed — +it extracted the Cloudflare preview URL from `wrangler` output and would have +failed every time the deploy job landed on the Mac. + +Two more constraints worth knowing before you add a step: + +- **Docker-based actions cannot run on macOS runners.** Every action we use + today (`actions/checkout`, `actions/setup-node`, `googleapis/release-please-action`, + `marocchino/sticky-pull-request-comment`) is a JavaScript action, so we are + fine — but a container action added later would fail only on the Mac. +- **`npm ci` needs the darwin-arm64 optional deps in the lockfile.** They are + there (`@esbuild/darwin-arm64`, `@rolldown/binding-darwin-arm64`, + `@tailwindcss/oxide-darwin-arm64`, `@resvg/resvg-js-darwin-arm64`, + `@img/sharp-darwin-arm64`, `@cloudflare/workerd-darwin-arm64`, + `@astrojs/compiler-binding-darwin-arm64`). If a dependency bump is ever made + with `--no-optional` or on a platform-filtered install, the lockfile loses + those entries and the Mac builds break while Linux stays green. + +## 4. Operating it + +```sh +# status of one runner +cd ~/actions-runners/-1 && ./svc.sh status + +# live logs +tail -f ~/actions-runners/-1/_diag/Runner_*.log + +# what the org thinks is online +open https://github.com/organizations/wavekat/settings/actions/runners +``` + +Self-hosted runners do **not** get a clean machine per job. `_work` persists +between runs; `actions/checkout` cleans the repo but the npm cache, the +`actions/setup-node` tool cache, and anything a job wrote outside the workspace +do not. That is a feature (fast builds) with one sharp edge: a job that fails +only on one host is usually stale state, not the code. `rm -rf _work` in the +runner dir, with the runner stopped, is the reset. diff --git a/scripts/setup-gha-runners-macos.sh b/scripts/setup-gha-runners-macos.sh new file mode 100755 index 0000000..13f4fe7 --- /dev/null +++ b/scripts/setup-gha-runners-macos.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# +# Install N self-hosted GitHub Actions runners on a single macOS host +# (Apple Silicon or Intel) and register them with the `wavekat` org. +# +# This is the macOS twin of setup-gha-runners.sh. It registers with the +# SAME `wavekat-ci` label, so a Mac mini joins the existing pool and +# `runs-on: [self-hosted, wavekat-ci]` jobs land on whichever host is +# free — no workflow changes needed. +# +# Differences from the Linux script, all forced by the platform: +# * runner package is actions-runner-osx-{arm64,x64} +# * the service is a launchd LaunchAgent (svc.sh, no sudo), not systemd +# * runners live under $HOME by default, since a LaunchAgent runs as you +# * a `.path` file is written so Homebrew binaries are visible to the +# runner (launchd does not source your shell profile) +# +# Usage (run on the Mac, in a logged-in session — see NOTE below): +# +# # Easiest: let the script fetch a registration token via gh CLI. +# # (`gh auth login` once with an account that has wavekat org admin) +# ./setup-gha-runners-macos.sh +# +# # Or pass a token explicitly (valid 1h, can register multiple runners): +# RUNNER_TOKEN=AAAA... ./setup-gha-runners-macos.sh +# +# # Override defaults: +# RUNNER_COUNT=6 RUNNER_PREFIX=mac-mini RUNNER_LABELS=wavekat-ci,macos \ +# ./setup-gha-runners-macos.sh +# +# # Also stop the Mac from sleeping (recommended for a dedicated host): +# RUNNER_KEEP_AWAKE=1 ./setup-gha-runners-macos.sh +# +# NOTE: a LaunchAgent needs a GUI login session. On a headless Mac mini, +# enable automatic login (System Settings -> Users & Groups -> Automatic +# login) so the agents come back after a reboot, and run this script from +# a Screen Sharing session the first time. Over a bare SSH session +# `launchctl` can refuse to load the agent; the script tells you if that +# happens. +# +# Re-running is safe: existing runners with the same name are stopped, +# de-registered and re-registered. + +set -euo pipefail + +ORG="${RUNNER_ORG:-wavekat}" +COUNT="${RUNNER_COUNT:-4}" +PREFIX="${RUNNER_PREFIX:-$(hostname -s)}" +BASE_DIR="${RUNNER_BASE_DIR:-${HOME}/actions-runners}" +EXTRA_LABELS="${RUNNER_LABELS:-wavekat-ci,${PREFIX}}" +RUNNER_VERSION="${RUNNER_VERSION:-}" # empty = latest +KEEP_AWAKE="${RUNNER_KEEP_AWAKE:-0}" + +log() { printf '\033[1;36m==>\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m!!\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1;31mxx\033[0m %s\n' "$*" >&2; exit 1; } + +[[ "$(uname -s)" == "Darwin" ]] || die "this script targets macOS (got $(uname -s)) — use setup-gha-runners.sh on Linux" + +case "$(uname -m)" in + arm64) ARCH=arm64 ;; + x86_64) ARCH=x64 ;; + *) die "unsupported arch $(uname -m)" ;; +esac + +# `git` on a fresh Mac is a stub that prompts for the Command Line Tools. +# actions/checkout shells out to it, so fail loudly here rather than in CI. +if ! /usr/bin/git --version >/dev/null 2>&1; then + die "git is not usable — run: xcode-select --install" +fi + +if [[ -z "${RUNNER_VERSION}" ]]; then + log "resolving latest runner version from github.com/actions/runner" + RUNNER_VERSION="$(curl -fsSL https://api.github.com/repos/actions/runner/releases/latest \ + | grep -oE '"tag_name": *"v[^"]+"' \ + | head -n1 \ + | sed -E 's/.*"v([^"]+)".*/\1/')" + [[ -n "${RUNNER_VERSION}" ]] || die "could not resolve latest runner version" +fi +log "runner version: ${RUNNER_VERSION} arch: osx-${ARCH}" + +missing_gh_help() { + cat >&2 <<'EOF' + +No RUNNER_TOKEN set, and `gh` CLI is not installed. + +Pick one: + + A) Install gh with Homebrew, then re-run: + + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + brew install gh + gh auth login # use an account with wavekat org admin + + B) Fetch a registration token elsewhere and export it (valid 1h, can + register multiple runners during that window): + + # on any machine with gh authed as a wavekat admin: + gh api -X POST /orgs/wavekat/actions/runners/registration-token --jq .token + + # then on this host: + RUNNER_TOKEN= ./setup-gha-runners-macos.sh +EOF +} + +get_token() { + if [[ -n "${RUNNER_TOKEN:-}" ]]; then + printf '%s' "${RUNNER_TOKEN}" + return + fi + if ! command -v gh >/dev/null 2>&1; then + missing_gh_help + exit 1 + fi + gh api -X POST "/orgs/${ORG}/actions/runners/registration-token" --jq .token \ + || die "failed to fetch registration token (is gh authed as a wavekat admin?)" +} + +mkdir -p "${BASE_DIR}/.cache" + +TARBALL="actions-runner-osx-${ARCH}-${RUNNER_VERSION}.tar.gz" +TARBALL_URL="https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/${TARBALL}" +CACHE_TARBALL="${BASE_DIR}/.cache/${TARBALL}" + +if [[ ! -f "${CACHE_TARBALL}" ]]; then + log "downloading ${TARBALL}" + curl -fsSL -o "${CACHE_TARBALL}" "${TARBALL_URL}" +fi + +# launchd starts the runner with a minimal PATH — it does not source +# ~/.zprofile, so Homebrew (and anything installed through it) is invisible +# unless we say so. actions/setup-node injects its own node ahead of this, +# so this is mostly about git/gh/jq and any tool a job shells out to. +BREW_PREFIX="" +for candidate in /opt/homebrew /usr/local; do + if [[ -x "${candidate}/bin/brew" ]]; then BREW_PREFIX="${candidate}"; break; fi +done +RUNNER_PATH="/usr/bin:/bin:/usr/sbin:/sbin" +if [[ -n "${BREW_PREFIX}" ]]; then + RUNNER_PATH="${BREW_PREFIX}/bin:${BREW_PREFIX}/sbin:${RUNNER_PATH}" +else + warn "Homebrew not found — the runner's PATH will be system-only" +fi + +TOKEN="$(get_token)" +[[ -n "${TOKEN}" ]] || die "got empty registration token" + +for i in $(seq 1 "${COUNT}"); do + NAME="${PREFIX}-${i}" + DIR="${BASE_DIR}/${NAME}" + log "configuring runner ${NAME} at ${DIR}" + + # Stop and de-register any previous install of this runner before we + # blow the directory away, otherwise the org is left holding a ghost. + if [[ -d "${DIR}" ]]; then + warn "existing runner dir for ${NAME} found — removing" + ( cd "${DIR}" && ./svc.sh stop >/dev/null 2>&1 || true ) + ( cd "${DIR}" && ./svc.sh uninstall >/dev/null 2>&1 || true ) + ( cd "${DIR}" && ./config.sh remove --token "${TOKEN}" || true ) + fi + + rm -rf "${DIR}" + mkdir -p "${DIR}" + tar -xzf "${CACHE_TARBALL}" -C "${DIR}" + + # The tarball is fetched with curl, which does not set the quarantine + # attribute — but a manually downloaded one would, and Gatekeeper then + # kills the binaries. Clearing it is a no-op in the normal path. + xattr -dr com.apple.quarantine "${DIR}" 2>/dev/null || true + + ( cd "${DIR}" && ./config.sh \ + --unattended \ + --replace \ + --url "https://github.com/${ORG}" \ + --token "${TOKEN}" \ + --name "${NAME}" \ + --runnergroup "Default" \ + --labels "${EXTRA_LABELS}" \ + --work "_work" ) + + printf '%s\n' "${RUNNER_PATH}" > "${DIR}/.path" + + log "installing launchd service for ${NAME}" + ( cd "${DIR}" && ./svc.sh install ) + if ! ( cd "${DIR}" && ./svc.sh start ); then + warn "could not start ${NAME} via launchctl." + warn "This usually means there is no GUI login session. Log in (or" + warn "connect via Screen Sharing) and run: cd ${DIR} && ./svc.sh start" + fi +done + +if [[ "${KEEP_AWAKE}" == "1" ]]; then + log "disabling sleep so queued jobs are picked up" + sudo systemsetup -setcomputersleep Never >/dev/null + sudo pmset -a disksleep 0 womp 1 >/dev/null +fi + +log "done — ${COUNT} runner(s) registered to ${ORG} with labels: ${EXTRA_LABELS}" +log "check status: cd ${BASE_DIR}/${PREFIX}-1 && ./svc.sh status" +log "live logs: tail -f ${BASE_DIR}/${PREFIX}-1/_diag/Runner_*.log" +log "org view: https://github.com/organizations/${ORG}/settings/actions/runners" diff --git a/scripts/uninstall-gha-runners-macos.sh b/scripts/uninstall-gha-runners-macos.sh new file mode 100755 index 0000000..e3a78e3 --- /dev/null +++ b/scripts/uninstall-gha-runners-macos.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# +# Tear down self-hosted GitHub Actions runners installed by +# setup-gha-runners-macos.sh. Stops the launchd agents, removes them, and +# de-registers each runner from the `wavekat` org. +# +# Usage: +# ./uninstall-gha-runners-macos.sh +# RUNNER_TOKEN=AAAA... ./uninstall-gha-runners-macos.sh # uses a remove-token +# +# A *remove* token can be fetched via: +# gh api -X POST /orgs/wavekat/actions/runners/remove-token --jq .token + +set -euo pipefail + +ORG="${RUNNER_ORG:-wavekat}" +COUNT="${RUNNER_COUNT:-4}" +PREFIX="${RUNNER_PREFIX:-$(hostname -s)}" +BASE_DIR="${RUNNER_BASE_DIR:-${HOME}/actions-runners}" + +log() { printf '\033[1;36m==>\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m!!\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1;31mxx\033[0m %s\n' "$*" >&2; exit 1; } + +[[ "$(uname -s)" == "Darwin" ]] || die "this script targets macOS (got $(uname -s))" + +get_token() { + if [[ -n "${RUNNER_TOKEN:-}" ]]; then + printf '%s' "${RUNNER_TOKEN}" + return + fi + if ! command -v gh >/dev/null 2>&1; then + cat >&2 <<'EOF' + +No RUNNER_TOKEN set, and `gh` CLI is not installed. + +Install gh with Homebrew and re-run, or fetch a remove-token elsewhere: + + brew install gh + gh auth login + + # or, from any machine with gh authed as a wavekat admin: + gh api -X POST /orgs/wavekat/actions/runners/remove-token --jq .token + RUNNER_TOKEN= ./uninstall-gha-runners-macos.sh +EOF + exit 1 + fi + gh api -X POST "/orgs/${ORG}/actions/runners/remove-token" --jq .token +} + +TOKEN="$(get_token)" + +for i in $(seq 1 "${COUNT}"); do + NAME="${PREFIX}-${i}" + DIR="${BASE_DIR}/${NAME}" + log "removing runner ${NAME}" + + if [[ -d "${DIR}" ]]; then + ( cd "${DIR}" && ./svc.sh stop >/dev/null 2>&1 || true ) + ( cd "${DIR}" && ./svc.sh uninstall >/dev/null 2>&1 || true ) + ( cd "${DIR}" && ./config.sh remove --token "${TOKEN}" || true ) + rm -rf "${DIR}" + else + warn "no directory at ${DIR} — skipping" + fi +done + +log "done" From 9fc5ff0b482d3ea03c73caee0d36d079dc44be4a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 06:41:57 +0000 Subject: [PATCH 2/2] ci: run the Mac mini's runners as Docker Desktop containers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switches the macOS runners from a native launchd install to Docker Desktop containers, reusing the existing scripts/docker image. Reading the other repos on this label is what settles it: wavekat-voice's ci.yml runs `sudo apt-get install` three times, so every wavekat-ci job in that repo would have failed the moment it landed on a native macOS runner. macOS cannot run macOS containers, so Docker Desktop means the Mac contributes Linux/arm64 capacity — which is the goal, since nothing in the pool's workload needs macOS and a uniform Ubuntu userland removes the BSD-vs-GNU failure mode entirely. The image already resolved its arch via dpkg, so it builds natively on Apple Silicon unchanged. Supervision is Docker's own --restart unless-stopped rather than launchd; the reboot story is Docker Desktop's start-at-login plus auto-login. docs/06 now also records who else rides on this label — seven repos, with wavekat-voice and wavekat-asr building sherpa-onnx native code and so arch-sensitive. No shipped artifact is built on wavekat-ci (installers use GitHub-hosted runners), so a bad host means red CI, not a bad release. Recommends registering the Mac under wavekat-ci-arm64 first and opting repos in, rather than widening the label blind. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TxidDNZBX6a7YqiPdmjf6C --- .github/workflows/preview.yml | 6 +- CLAUDE.md | 12 +- docs/06-self-hosted-runners.md | 279 ++++++++++++++----------- scripts/setup-gha-runners-macos.sh | 258 ++++++++++------------- scripts/uninstall-gha-runners-macos.sh | 52 +++-- 5 files changed, 319 insertions(+), 288 deletions(-) diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index cad5fb5..f6f6f82 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -44,9 +44,9 @@ jobs: run: | OUTPUT=$(npx wrangler pages deploy ./dist --project-name=wavekat-com --branch="${{ github.head_ref || github.ref_name }}" 2>&1) echo "$OUTPUT" - # POSIX sed, not `grep -oP`: this job also runs on the macOS - # runners, whose BSD grep has no -P. Keep every `run:` block in - # this repo portable across the pool — see docs/06. + # POSIX sed rather than `grep -oP`: portable across the runner + # pool, and it does not fail the step under `set -e` when the + # deploy output carries no alias URL. See docs/06. ALIAS=$(printf '%s\n' "$OUTPUT" \ | sed -nE 's|.*Deployment alias URL: (https://[^[:space:]]+).*|\1|p' \ | tail -n 1) diff --git a/CLAUDE.md b/CLAUDE.md index 029e34a..9d3fd65 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -55,13 +55,17 @@ This repo uses **release-please**. Since GitHub squash-merges use the PR title a - **Deployment**: Cloudflare Pages (consistent with rest of org) - **Domain**: `wavekat.com` — DNS to be pointed at Cloudflare Pages once site is ready -## CI runs on a mixed self-hosted pool — keep every `run:` block portable +## CI runs on a shared, org-wide, mixed-arch runner pool -All workflows use `runs-on: [self-hosted, wavekat-ci]`. That label is a **pool**, not a machine: a Linux x86-64 workstation *and* a macOS arm64 Mac mini both carry it, so any job lands on either host non-deterministically. Setup lives in `scripts/setup-gha-runners.sh` (Linux, systemd), `scripts/setup-gha-runners-docker.sh` (Linux, containerised) and `scripts/setup-gha-runners-macos.sh` (macOS, launchd); the full story is `docs/06-self-hosted-runners.md`. +All workflows use `runs-on: [self-hosted, wavekat-ci]`. Two things about that label are easy to get wrong: -The practical consequence: **macOS ships BSD userland, so GNU-only shell in a `run:` block fails about half the time, on PRs that changed nothing.** No `grep -oP`, no bare `sed -i`, no `readlink -f`, no `date -d`, no `sha256sum`, no `xargs -r`, and no bash 4+ syntax (macOS `/bin/bash` is 3.2). Docker-based actions can't run on macOS at all — every action we use must be a JavaScript action. And `npm ci` needs the `darwin-arm64` optional deps to stay in `package-lock.json`, so never regenerate the lockfile with `--no-optional`. +**It is a pool, not a machine.** A Linux x86-64 workstation and a Mac mini (running the runners as Docker containers, so they report `Linux`/`ARM64`) both carry it, and a job lands on either non-deterministically. Setup lives in `scripts/setup-gha-runners.sh` (Linux, systemd), `scripts/setup-gha-runners-docker.sh` (Linux, containers) and `scripts/setup-gha-runners-macos.sh` (macOS, Docker Desktop); the full story — including why the Mac runs Linux containers rather than a native macOS runner — is `docs/06-self-hosted-runners.md`. -To pin a job to one host, add that host's automatic label: `runs-on: [self-hosted, wavekat-ci, macOS]` or `..., Linux]`. +**It is org-wide, and this repo is not its main consumer.** Seven repos ride on `wavekat-ci` (`wavekat.com`, `wavekat-voice`, `wavekat-platform`, `wavekat-asr`, `wavekat-cli`, `wavekat-lab`, `wavekat-platform-client`). Never change what the label points at — or assume a new host is safe — based on this repo's workflows alone; `wavekat-voice` and `wavekat-asr` build sherpa-onnx/ONNX native code and are the arch-sensitive ones. No shipped artifact is built on `wavekat-ci` (installers use GitHub-hosted runners), so the blast radius of a bad host is red CI, not a bad release. + +Since every runner is Ubuntu 24.04, GNU shell is fine — but **arch must never be assumed**. Anything that downloads a prebuilt binary or pins a target triple has to resolve arch at runtime (`uname -m`, `dpkg --print-architecture`), and `npm ci` needs both `linux-x64` and `linux-arm64` optional deps in `package-lock.json`, so never regenerate the lockfile with `--no-optional`. + +To pin a job to one host, add the runner's automatic arch label: `runs-on: [self-hosted, wavekat-ci, X64]` or `..., ARM64]`. ## SEO & GEO — every new page must be both diff --git a/docs/06-self-hosted-runners.md b/docs/06-self-hosted-runners.md index 03ea4ac..57f255f 100644 --- a/docs/06-self-hosted-runners.md +++ b/docs/06-self-hosted-runners.md @@ -1,145 +1,188 @@ # 06 — Self-hosted runners: adding the Mac mini Every workflow in this repo runs on `runs-on: [self-hosted, wavekat-ci]`. Until -now that label existed on exactly one machine — a Linux x86-64 box set up by -`scripts/setup-gha-runners.sh`. This doc adds a Mac mini (Apple Silicon, -arm64) to the same pool, and records the one rule that keeps a mixed-OS pool -from breaking CI at random. +now that label existed on one machine — a Linux x86-64 workstation. This doc +adds a Mac mini (Apple Silicon) to the same pool, running the runners as Docker +Desktop containers rather than natively. -## 1. How the pool actually works +It also records the part that turned out to matter more than the setup itself: +**`wavekat-ci` is an org-wide label, and seven repos ride on it.** Changing what +that label points at is not a wavekat.com decision. -`runs-on` is a **label set**, not a machine. A job runs on any idle runner that -carries *all* the listed labels. So the whole of "how do we run on the Mac too" -is: register the Mac with the `wavekat-ci` label. Nothing in -`.github/workflows/` changes. +## 1. Why containers and not a native macOS runner -Each runner also gets automatic labels it never asked for, which is how you -pin work to one host when you need to: +The first instinct — install the runner natively with `svc.sh` and launchd — is +wrong here, for a reason that only shows up when you read the other repos' +workflows. `wavekat-voice/.github/workflows/ci.yml` does this, three times: -| Host | Automatic labels | Labels we add | -|------|------------------|---------------| -| Linux workstation | `self-hosted`, `Linux`, `X64` | `wavekat-ci`, `` | -| Mac mini | `self-hosted`, `macOS`, `ARM64` | `wavekat-ci`, `` | +```yaml +run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libasound2-dev cmake +``` + +A native macOS runner has no `apt-get`. Every `wavekat-ci` job in the org's most +important repo would have failed the moment it landed on the Mac — not subtly, +immediately. The same jobs also assume GNU userland throughout, and macOS ships +BSD (`grep -oP`, `sed -i`, `readlink -f`, `date -d`, `sha256sum`, `xargs -r`, +and bash 3.2 all differ). + +So the choice isn't "Docker vs raw shell on macOS". It's: -Both scripts default `RUNNER_LABELS` to `wavekat-ci,`, so after setup -you can force a job onto one machine with `runs-on: [self-hosted, wavekat-ci, macOS]` -without disturbing anything else. +> Should the Mac mini be an **ARM Linux** CI host, or a **macOS** CI host? -Jobs are **not** load-balanced by cost or speed. GitHub hands a queued job to -the first idle matching runner, so with both hosts registered a given PR's CI -may land on Linux one run and macOS the next. That is the point — and it is -also why section 3 matters. +macOS cannot run macOS containers — Docker Desktop runs Linux containers in a +VM — so containers mean the Mac contributes *Linux/arm64* capacity. That is what +we want: nothing in the pool's workload needs macOS, and a uniform Ubuntu 24.04 +userland everywhere means a `run:` block can never work on one host and fail on +the other. + +The runner image (`scripts/docker/`) was already arch-portable — +`dpkg --print-architecture` selects the runner tarball, and the `gh` apt line is +arch-templated — so it builds natively on Apple Silicon with **no changes and no +Rosetta**. ## 2. Setting up the Mac mini -`scripts/setup-gha-runners-macos.sh` is the macOS twin of the Linux script and -takes the same environment variables (`RUNNER_ORG`, `RUNNER_COUNT`, -`RUNNER_PREFIX`, `RUNNER_BASE_DIR`, `RUNNER_LABELS`, `RUNNER_VERSION`, -`RUNNER_TOKEN`). On the Mac: +Requires Docker Desktop installed and running. ```sh -xcode-select --install # git — actions/checkout shells out to it -/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" brew install gh && gh auth login # an account with wavekat org admin git clone git@github.com-wavekat:wavekat/wavekat.com.git cd wavekat.com -RUNNER_KEEP_AWAKE=1 ./scripts/setup-gha-runners-macos.sh +./scripts/setup-gha-runners-macos.sh ``` -Tear down with `./scripts/uninstall-gha-runners-macos.sh` (same variables). - -What the script does differently from the Linux one, and why: - -- **`actions-runner-osx-arm64`**, resolved to the latest release. There is a - native Apple Silicon build; do not run the x64 one under Rosetta. -- **launchd, not systemd.** `svc.sh install` on macOS writes a *LaunchAgent* - to `~/Library/LaunchAgents/` and runs as your user — no `sudo`, and runners - therefore live under `$HOME/actions-runners` instead of `/opt`. -- **A `.path` file per runner.** launchd does not source `~/.zprofile`, so - `/opt/homebrew/bin` is invisible to the runner process unless we write it - into the runner's `.path`. (`actions/setup-node` prepends its own Node ahead - of this, so this is about `git`, `gh`, `jq` and anything a job shells out to.) -- **Quarantine clearing.** A curl-fetched tarball carries no - `com.apple.quarantine`, but a browser-downloaded one does and Gatekeeper - then kills the runner binaries. The `xattr -dr` is a no-op in the normal path. - -### The headless-Mac gotcha - -A LaunchAgent needs a GUI login session. On a Mac mini with no one logged in, -`launchctl` will refuse to load the agent and the runner shows offline after -every reboot. Two settings fix it permanently: - -1. **System Settings → Users & Groups → Automatic login** → the runner user. - Without this, runners do not come back after a power cut or an OS update. -2. **No sleep.** `RUNNER_KEEP_AWAKE=1` runs - `systemsetup -setcomputersleep Never` and `pmset -a disksleep 0 womp 1` for - you. A sleeping Mac does not pick up queued jobs; the job just sits there - until the Linux box frees up, which hides the problem instead of failing. - -Run the setup script the first time from a Screen Sharing session, not a bare -SSH session. If it prints the "could not start via launchctl" warning, that is -what happened — log in and run `./svc.sh start` in the runner dir. - -### How many runners - -`RUNNER_COUNT` defaults to 4, matching the Linux host. A full `npm run cf:build` -of this site is a real Astro/Rolldown build; four in parallel on a 16 GB Mac -mini is the practical ceiling. Drop to `RUNNER_COUNT=2` if builds start -swapping. - -## 3. The rule: every `run:` block must be portable - -This is the part that bites. A mixed pool means **any** job can land on either -OS, so a shell script that only works on GNU userland fails roughly half the -time, non-deterministically, on PRs that changed nothing. - -macOS ships BSD userland, not GNU. The traps that apply to this repo: - -| Don't | Do | Why | -|-------|----|-----| -| `grep -oP '(?<=x)y'` | `sed -nE 's/.*x(y).*/\1/p'` | BSD grep has no `-P` (PCRE) | -| `sed -i 's/a/b/' f` | `sed -i.bak` then `rm f.bak` | BSD `sed -i` requires a suffix arg | -| `readlink -f p` | `cd "$(dirname p)" && pwd -P` | BSD `readlink` has no `-f` | -| `date -d '1 day ago'` | `date -u +%s` and do maths | BSD `date` uses `-v-1d` | -| `sha256sum f` | `shasum -a 256 f` | `sha256sum` is GNU coreutils | -| `xargs -r` | guard with `[ -s file ]` | BSD `xargs` has no `-r` | -| bash 4+ syntax (`${v,,}`, `declare -A`, `mapfile`) | bash 3.2 equivalents | macOS `/bin/bash` is 3.2 | - -Fixing `grep -oP` in `preview.yml` was the only change this migration needed — -it extracted the Cloudflare preview URL from `wrangler` output and would have -failed every time the deploy job landed on the Mac. - -Two more constraints worth knowing before you add a step: - -- **Docker-based actions cannot run on macOS runners.** Every action we use - today (`actions/checkout`, `actions/setup-node`, `googleapis/release-please-action`, - `marocchino/sticky-pull-request-comment`) is a JavaScript action, so we are - fine — but a container action added later would fail only on the Mac. -- **`npm ci` needs the darwin-arm64 optional deps in the lockfile.** They are - there (`@esbuild/darwin-arm64`, `@rolldown/binding-darwin-arm64`, - `@tailwindcss/oxide-darwin-arm64`, `@resvg/resvg-js-darwin-arm64`, - `@img/sharp-darwin-arm64`, `@cloudflare/workerd-darwin-arm64`, - `@astrojs/compiler-binding-darwin-arm64`). If a dependency bump is ever made - with `--no-optional` or on a platform-filtered install, the lockfile loses - those entries and the Mac builds break while Linux stays green. - -## 4. Operating it +Tear down with `./scripts/uninstall-gha-runners-macos.sh`. Both take the same +env vars as the Linux scripts (`RUNNER_ORG`, `RUNNER_COUNT`, `RUNNER_PREFIX`, +`RUNNER_LABELS`, `RUNNER_IMAGE`, `RUNNER_TOKEN`). + +Differences from `setup-gha-runners-docker.sh` (the Linux container script): + +- **Supervision is Docker's own `--restart unless-stopped`**, not systemd. + Docker Desktop restarts containers with a restart policy when it starts, so + there is no launchd plist to write. +- **Two manual settings, or the runners die at the next reboot:** + Docker Desktop → Settings → General → *Start Docker Desktop when you sign in*, + and System Settings → Users & Groups → *Automatic login*. Docker Desktop is a + GUI app; without a login session there is no daemon and no runners. +- **The script checks VM memory against `RUNNER_COUNT`** and warns under ~2 GB + per runner. Docker Desktop's default allocation is thin for four parallel + Rust/Astro builds, and the failure mode is an OOM-killed build rather than a + clear error. Raise it in Settings → Resources, or lower `RUNNER_COUNT`. +- **Keep `_work` on the named volume** (the script does). A host bind mount + would put every `npm ci` and `cargo build` through virtiofs — this is the + usual reason people conclude Docker on Mac is slow. + +Docker Desktop's disk image is also a fixed size. Four runners with warm cargo +targets, pnpm stores and Playwright browsers will grow into it; if builds start +failing on "no space left on device", raise the disk limit in Settings → +Resources before blaming anything else. + +## 3. Who else uses `wavekat-ci` — check before you flip the label + +The label is org-wide. As of this change, **7 of the org's 27 repos** run jobs +on it: + +| Repo | Workflows on `wavekat-ci` | What those jobs do | Arch-sensitive? | +|------|---------------------------|--------------------|-----------------| +| `wavekat.com` | 4 (6 jobs) | Astro build, link/meta checks, CF Pages deploy | No | +| `wavekat-voice` | 6 of 7 | Rust check/clippy/test, `make sidecar` (sherpa-onnx via CMake), pnpm + Playwright | **Yes** | +| `wavekat-platform` | 3 | ci, release, db-migrate | Unverified | +| `wavekat-asr` | 2 of 2 | `cargo test --features sherpa-onnx` | **Yes** | +| `wavekat-cli` | 3 | release-plz tail + publish (the cross-target build matrix uses GitHub-hosted runners) | No | +| `wavekat-lab` | 6 of 7 | ci, Common Voice sync/deploy, ONNX publish | Some | +| `wavekat-platform-client` | 2 | ci, release-plz | No | + +Not on the label (they use GitHub-hosted runners): `wavekat-core`, +`wavekat-vad`, `wavekat-turn`, `wavekat-tts`, `wavekat-flow`, `wavekat-brand`. + +One reassurance from that audit: **no shipped artifact is built on `wavekat-ci`.** +`wavekat-voice/release.yml` builds installers on `macos-latest`, +`ubuntu-latest`, `windows-latest` and `windows-11-arm`; its only `wavekat-ci` +job is `trigger-site-rebuild`, which is a `curl -X POST`. Likewise +`wavekat-cli/release.yml` cross-builds its targets on GitHub-hosted runners. So +an arm64 runner joining the pool cannot cause a wrong-architecture binary to +reach users — the blast radius is CI going red, not a bad release. + +### The real risk: ONNX on aarch64 + +`wavekat-asr` runs `cargo test --workspace --features sherpa-onnx`, and +`wavekat-voice`'s `sidecar` job builds the same stack via CMake. +`wavekat-voice/release.yml` describes the daemon as linking "a prebuilt +sherpa-onnx static lib" — and prebuilt native libs are exactly the thing that is +often published for `x86_64-unknown-linux-gnu` and not for +`aarch64-unknown-linux-gnu`. + +**Verify this before putting `wavekat-ci` on the Mac containers**, because a +failure here is silent-until-merge: PRs in two repos start failing about half +the time, on whichever runs happen to land on arm64. + +`wavekat-lab/cv-runner-provision.yml` also hardcodes +`actions-runner-linux-x64-…` in a provisioning script. That one is provisioning +a *different* machine, so it is probably fine — but it is the same class of +assumption and worth a read. + +### Recommended rollout + +Register the Mac with a distinct label first and opt repos in one at a time: ```sh -# status of one runner -cd ~/actions-runners/-1 && ./svc.sh status +RUNNER_LABELS=wavekat-ci-arm64,mac-mini ./scripts/setup-gha-runners-macos.sh +``` + +Point one low-risk repo at it (`wavekat.com` is the obvious candidate — pure +Node, no native deps), let it run for a few days, then either widen the label to +`wavekat-ci` or keep the split permanently and pin the ONNX repos to x86-64 with +the runner's automatic `X64` label: + +```yaml +runs-on: [self-hosted, wavekat-ci, X64] +``` -# live logs -tail -f ~/actions-runners/-1/_diag/Runner_*.log +Both hosts carry automatic labels you can pin against: -# what the org thinks is online +| Host | Automatic labels | Labels we add | +|------|------------------|---------------| +| Linux workstation | `self-hosted`, `Linux`, `X64` | `wavekat-ci`, `` | +| Mac mini (containers) | `self-hosted`, `Linux`, `ARM64` | `wavekat-ci`, `` | + +Note the Mac's containers report `Linux`, not `macOS` — the runner sees the +container, not the host. + +## 4. What still has to stay portable + +Userland is now uniform (Ubuntu 24.04 everywhere), so the BSD-vs-GNU trap list +no longer gates CI. Two things do: + +- **Architecture.** Anything that downloads a prebuilt binary, pins a target + triple, or compiles native code must resolve arch at runtime rather than + assuming x86-64. `uname -m` / `dpkg --print-architecture`, not a hardcoded + `x64` in a URL. +- **Lockfile optional deps.** `npm ci` needs both `linux-x64` and `linux-arm64` + optional packages present. This repo's `package-lock.json` has both for every + native dep (esbuild, rolldown, oxide, resvg, sharp, workerd, and + `@astrojs/compiler-binding-linux-{x64,arm64}-gnu`). Regenerating the lockfile + with `--no-optional`, or on a platform-filtered install, silently drops them + and breaks one host while the other stays green. + +The `sed` in `preview.yml` (replacing a `grep -oP`) is kept: it is correct on +GNU too, and it no longer fails the step under `set -e` when no alias URL is +found. The BSD notes only become load-bearing again if someone adds a *native* +macOS runner later. + +## 5. Operating it + +```sh +docker ps --filter name=gha-runner # what's up +docker logs -f gha-runner-1 # live logs +docker exec -it gha-runner-1 bash # shell inside a runner open https://github.com/organizations/wavekat/settings/actions/runners ``` -Self-hosted runners do **not** get a clean machine per job. `_work` persists -between runs; `actions/checkout` cleans the repo but the npm cache, the -`actions/setup-node` tool cache, and anything a job wrote outside the workspace -do not. That is a feature (fast builds) with one sharp edge: a job that fails -only on one host is usually stale state, not the code. `rm -rf _work` in the -runner dir, with the runner stopped, is the reset. +Self-hosted runners do **not** get a clean machine per job. The named volume +persists between runs: `actions/checkout` cleans the repo, but the cargo target +dir, pnpm store, npm cache and Playwright browsers do not — deliberately, since +that warmth is why these runners are fast. The sharp edge is that a job failing +on one host and not the other is usually stale state, not code. The reset is +`docker rm -f gha-runner-N && docker volume rm gha-runner-N`, then re-run the +setup script. diff --git a/scripts/setup-gha-runners-macos.sh b/scripts/setup-gha-runners-macos.sh index 13f4fe7..debede3 100755 --- a/scripts/setup-gha-runners-macos.sh +++ b/scripts/setup-gha-runners-macos.sh @@ -1,201 +1,161 @@ #!/usr/bin/env bash # -# Install N self-hosted GitHub Actions runners on a single macOS host -# (Apple Silicon or Intel) and register them with the `wavekat` org. +# Install N self-hosted GitHub Actions runners on a macOS host (Apple +# Silicon or Intel) as Docker Desktop containers, and register them with +# the `wavekat` org. # -# This is the macOS twin of setup-gha-runners.sh. It registers with the -# SAME `wavekat-ci` label, so a Mac mini joins the existing pool and -# `runs-on: [self-hosted, wavekat-ci]` jobs land on whichever host is -# free — no workflow changes needed. +# This is the macOS twin of setup-gha-runners-docker.sh. It reuses the +# SAME image (scripts/docker), which is already arch-portable, and +# registers with the SAME `wavekat-ci` label — so the Mac joins the +# existing pool and `runs-on: [self-hosted, wavekat-ci]` jobs land on +# whichever host is idle. No workflow changes needed. # -# Differences from the Linux script, all forced by the platform: -# * runner package is actions-runner-osx-{arm64,x64} -# * the service is a launchd LaunchAgent (svc.sh, no sudo), not systemd -# * runners live under $HOME by default, since a LaunchAgent runs as you -# * a `.path` file is written so Homebrew binaries are visible to the -# runner (launchd does not source your shell profile) +# Note what this means: macOS cannot run macOS containers, so Docker +# Desktop runs these on Linux/arm64 inside its VM. The Mac contributes +# *Linux* CI capacity. That is the point — nothing this repo builds +# needs macOS, and a uniform Ubuntu userland everywhere means a `run:` +# block can never work on one host and fail on the other. # -# Usage (run on the Mac, in a logged-in session — see NOTE below): +# Supervision is Docker's own `--restart unless-stopped`, not launchd: +# Docker Desktop restarts the containers when it starts, so as long as +# Docker Desktop launches at login the runners come back after a reboot. +# +# Usage (run on the Mac, with Docker Desktop running): # # # Easiest: let the script fetch a registration token via gh CLI. -# # (`gh auth login` once with an account that has wavekat org admin) +# # (`brew install gh && gh auth login` as a wavekat org admin) # ./setup-gha-runners-macos.sh # # # Or pass a token explicitly (valid 1h, can register multiple runners): # RUNNER_TOKEN=AAAA... ./setup-gha-runners-macos.sh # # # Override defaults: -# RUNNER_COUNT=6 RUNNER_PREFIX=mac-mini RUNNER_LABELS=wavekat-ci,macos \ +# RUNNER_COUNT=2 RUNNER_PREFIX=mac-mini RUNNER_LABELS=wavekat-ci,mac-mini \ # ./setup-gha-runners-macos.sh # -# # Also stop the Mac from sleeping (recommended for a dedicated host): -# RUNNER_KEEP_AWAKE=1 ./setup-gha-runners-macos.sh -# -# NOTE: a LaunchAgent needs a GUI login session. On a headless Mac mini, -# enable automatic login (System Settings -> Users & Groups -> Automatic -# login) so the agents come back after a reboot, and run this script from -# a Screen Sharing session the first time. Over a bare SSH session -# `launchctl` can refuse to load the agent; the script tells you if that -# happens. -# -# Re-running is safe: existing runners with the same name are stopped, -# de-registered and re-registered. +# Re-running is safe: existing containers are torn down, their volumes +# wiped, and the runners re-registered with a fresh token. set -euo pipefail ORG="${RUNNER_ORG:-wavekat}" COUNT="${RUNNER_COUNT:-4}" PREFIX="${RUNNER_PREFIX:-$(hostname -s)}" -BASE_DIR="${RUNNER_BASE_DIR:-${HOME}/actions-runners}" -EXTRA_LABELS="${RUNNER_LABELS:-wavekat-ci,${PREFIX}}" -RUNNER_VERSION="${RUNNER_VERSION:-}" # empty = latest -KEEP_AWAKE="${RUNNER_KEEP_AWAKE:-0}" +RUNNER_LABELS="${RUNNER_LABELS:-wavekat-ci,${PREFIX}}" +IMAGE="${RUNNER_IMAGE:-wavekat/gha-runner:latest}" +RUNNER_VERSION="${RUNNER_VERSION:-}" # empty = the Dockerfile's default +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DOCKER_CONTEXT="${SCRIPT_DIR}/docker" log() { printf '\033[1;36m==>\033[0m %s\n' "$*"; } warn() { printf '\033[1;33m!!\033[0m %s\n' "$*" >&2; } die() { printf '\033[1;31mxx\033[0m %s\n' "$*" >&2; exit 1; } -[[ "$(uname -s)" == "Darwin" ]] || die "this script targets macOS (got $(uname -s)) — use setup-gha-runners.sh on Linux" +[[ "$(uname -s)" == "Darwin" ]] || die "this script targets macOS (got $(uname -s)) — use setup-gha-runners-docker.sh on Linux" +[[ -d "${DOCKER_CONTEXT}" ]] || die "missing docker context at ${DOCKER_CONTEXT}" -case "$(uname -m)" in - arm64) ARCH=arm64 ;; - x86_64) ARCH=x64 ;; - *) die "unsupported arch $(uname -m)" ;; -esac +# 1. Find the Docker CLI. Docker Desktop symlinks into /usr/local/bin, +# but a non-login shell may not have its own bin dir on PATH. +DOCKER="" +for candidate in docker /usr/local/bin/docker "${HOME}/.docker/bin/docker" /opt/homebrew/bin/docker; do + if command -v "${candidate}" >/dev/null 2>&1; then DOCKER="${candidate}"; break; fi +done +[[ -n "${DOCKER}" ]] || die "docker CLI not found — install Docker Desktop from https://docker.com/products/docker-desktop" -# `git` on a fresh Mac is a stub that prompts for the Command Line Tools. -# actions/checkout shells out to it, so fail loudly here rather than in CI. -if ! /usr/bin/git --version >/dev/null 2>&1; then - die "git is not usable — run: xcode-select --install" +if ! "${DOCKER}" info >/dev/null 2>&1; then + die "Docker Desktop is not running (or the daemon is unreachable). Launch Docker Desktop and re-run." fi -if [[ -z "${RUNNER_VERSION}" ]]; then - log "resolving latest runner version from github.com/actions/runner" - RUNNER_VERSION="$(curl -fsSL https://api.github.com/repos/actions/runner/releases/latest \ - | grep -oE '"tag_name": *"v[^"]+"' \ - | head -n1 \ - | sed -E 's/.*"v([^"]+)".*/\1/')" - [[ -n "${RUNNER_VERSION}" ]] || die "could not resolve latest runner version" +# 2. Sanity-check the VM's memory against the runner count. Docker +# Desktop's default (often 8 GB) is thin for four parallel Astro +# builds; the failure mode is an OOM-killed build, not a clear error. +MEM_BYTES="$("${DOCKER}" info --format '{{.MemTotal}}' 2>/dev/null || echo 0)" +if [[ "${MEM_BYTES}" -gt 0 ]]; then + MEM_GB=$(( MEM_BYTES / 1024 / 1024 / 1024 )) + log "Docker Desktop VM memory: ${MEM_GB} GB for ${COUNT} runner(s)" + if [[ $(( MEM_GB / COUNT )) -lt 2 ]]; then + warn "under ~2 GB per runner — builds may be OOM-killed." + warn "Raise it in Docker Desktop → Settings → Resources → Memory," + warn "or lower RUNNER_COUNT (RUNNER_COUNT=2 ./setup-gha-runners-macos.sh)." + fi fi -log "runner version: ${RUNNER_VERSION} arch: osx-${ARCH}" - -missing_gh_help() { - cat >&2 <<'EOF' - -No RUNNER_TOKEN set, and `gh` CLI is not installed. - -Pick one: - - A) Install gh with Homebrew, then re-run: - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - brew install gh - gh auth login # use an account with wavekat org admin - - B) Fetch a registration token elsewhere and export it (valid 1h, can - register multiple runners during that window): - - # on any machine with gh authed as a wavekat admin: - gh api -X POST /orgs/wavekat/actions/runners/registration-token --jq .token - - # then on this host: - RUNNER_TOKEN= ./setup-gha-runners-macos.sh -EOF -} +# 3. Build the runner image. The Dockerfile resolves the runner tarball +# per-arch (dpkg --print-architecture), so this builds natively on +# Apple Silicon with no changes and no Rosetta. +log "building runner image ${IMAGE} (native $(uname -m))" +if [[ -n "${RUNNER_VERSION}" ]]; then + "${DOCKER}" build --build-arg "RUNNER_VERSION=${RUNNER_VERSION}" -t "${IMAGE}" "${DOCKER_CONTEXT}" +else + "${DOCKER}" build -t "${IMAGE}" "${DOCKER_CONTEXT}" +fi +# 4. Fetch a registration token (one token can register multiple runners +# within its 1h validity window). get_token() { if [[ -n "${RUNNER_TOKEN:-}" ]]; then printf '%s' "${RUNNER_TOKEN}" return fi if ! command -v gh >/dev/null 2>&1; then - missing_gh_help + cat >&2 <<'EOF' + +No RUNNER_TOKEN set, and `gh` CLI is not installed. + +Install gh with Homebrew and re-run, or fetch a token elsewhere: + + brew install gh + gh auth login # use an account with wavekat org admin + + # or, from any machine with gh authed as a wavekat admin: + gh api -X POST /orgs/wavekat/actions/runners/registration-token --jq .token + RUNNER_TOKEN= ./setup-gha-runners-macos.sh +EOF exit 1 fi gh api -X POST "/orgs/${ORG}/actions/runners/registration-token" --jq .token \ || die "failed to fetch registration token (is gh authed as a wavekat admin?)" } -mkdir -p "${BASE_DIR}/.cache" - -TARBALL="actions-runner-osx-${ARCH}-${RUNNER_VERSION}.tar.gz" -TARBALL_URL="https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/${TARBALL}" -CACHE_TARBALL="${BASE_DIR}/.cache/${TARBALL}" - -if [[ ! -f "${CACHE_TARBALL}" ]]; then - log "downloading ${TARBALL}" - curl -fsSL -o "${CACHE_TARBALL}" "${TARBALL_URL}" -fi - -# launchd starts the runner with a minimal PATH — it does not source -# ~/.zprofile, so Homebrew (and anything installed through it) is invisible -# unless we say so. actions/setup-node injects its own node ahead of this, -# so this is mostly about git/gh/jq and any tool a job shells out to. -BREW_PREFIX="" -for candidate in /opt/homebrew /usr/local; do - if [[ -x "${candidate}/bin/brew" ]]; then BREW_PREFIX="${candidate}"; break; fi -done -RUNNER_PATH="/usr/bin:/bin:/usr/sbin:/sbin" -if [[ -n "${BREW_PREFIX}" ]]; then - RUNNER_PATH="${BREW_PREFIX}/bin:${BREW_PREFIX}/sbin:${RUNNER_PATH}" -else - warn "Homebrew not found — the runner's PATH will be system-only" -fi - TOKEN="$(get_token)" [[ -n "${TOKEN}" ]] || die "got empty registration token" +# 5. (Re)create N runners. One container each, with its own named volume +# so registration survives restarts and Docker Desktop upgrades. +# +# `--restart unless-stopped` is the whole supervision story on macOS: +# there is no systemd, and Docker Desktop restores containers with a +# restart policy when it starts. No launchd plist needed. for i in $(seq 1 "${COUNT}"); do NAME="${PREFIX}-${i}" - DIR="${BASE_DIR}/${NAME}" - log "configuring runner ${NAME} at ${DIR}" - - # Stop and de-register any previous install of this runner before we - # blow the directory away, otherwise the org is left holding a ghost. - if [[ -d "${DIR}" ]]; then - warn "existing runner dir for ${NAME} found — removing" - ( cd "${DIR}" && ./svc.sh stop >/dev/null 2>&1 || true ) - ( cd "${DIR}" && ./svc.sh uninstall >/dev/null 2>&1 || true ) - ( cd "${DIR}" && ./config.sh remove --token "${TOKEN}" || true ) - fi - - rm -rf "${DIR}" - mkdir -p "${DIR}" - tar -xzf "${CACHE_TARBALL}" -C "${DIR}" - - # The tarball is fetched with curl, which does not set the quarantine - # attribute — but a manually downloaded one would, and Gatekeeper then - # kills the binaries. Clearing it is a no-op in the normal path. - xattr -dr com.apple.quarantine "${DIR}" 2>/dev/null || true - - ( cd "${DIR}" && ./config.sh \ - --unattended \ - --replace \ - --url "https://github.com/${ORG}" \ - --token "${TOKEN}" \ - --name "${NAME}" \ - --runnergroup "Default" \ - --labels "${EXTRA_LABELS}" \ - --work "_work" ) - - printf '%s\n' "${RUNNER_PATH}" > "${DIR}/.path" - - log "installing launchd service for ${NAME}" - ( cd "${DIR}" && ./svc.sh install ) - if ! ( cd "${DIR}" && ./svc.sh start ); then - warn "could not start ${NAME} via launchctl." - warn "This usually means there is no GUI login session. Log in (or" - warn "connect via Screen Sharing) and run: cd ${DIR} && ./svc.sh start" - fi + CONTAINER="gha-runner-${i}" + log "configuring runner ${NAME} (container ${CONTAINER})" + + # Tear down any previous instance and wipe its volume, so the fresh + # registration token is applied cleanly instead of the entrypoint + # short-circuiting on a stale .runner file. + "${DOCKER}" rm -f "${CONTAINER}" >/dev/null 2>&1 || true + "${DOCKER}" volume rm "${CONTAINER}" >/dev/null 2>&1 || true + + "${DOCKER}" run -d \ + --name "${CONTAINER}" \ + --hostname "${CONTAINER}" \ + --restart unless-stopped \ + -v "${CONTAINER}:/home/runner/runner" \ + -e "RUNNER_ORG=${ORG}" \ + -e "RUNNER_NAME=${NAME}" \ + -e "RUNNER_LABELS=${RUNNER_LABELS}" \ + -e "RUNNER_TOKEN=${TOKEN}" \ + "${IMAGE}" >/dev/null done -if [[ "${KEEP_AWAKE}" == "1" ]]; then - log "disabling sleep so queued jobs are picked up" - sudo systemsetup -setcomputersleep Never >/dev/null - sudo pmset -a disksleep 0 womp 1 >/dev/null -fi - -log "done — ${COUNT} runner(s) registered to ${ORG} with labels: ${EXTRA_LABELS}" -log "check status: cd ${BASE_DIR}/${PREFIX}-1 && ./svc.sh status" -log "live logs: tail -f ${BASE_DIR}/${PREFIX}-1/_diag/Runner_*.log" -log "org view: https://github.com/organizations/${ORG}/settings/actions/runners" +log "done — ${COUNT} runner(s) registered to ${ORG} with labels: ${RUNNER_LABELS}" +echo +warn "One manual step, or the runners will not survive a reboot:" +warn " Docker Desktop → Settings → General → 'Start Docker Desktop when you sign in'" +warn " System Settings → Users & Groups → Automatic login → the runner user" +echo +log "check status: ${DOCKER} ps --filter name=gha-runner" +log "live logs: ${DOCKER} logs -f gha-runner-1" +log "container shell: ${DOCKER} exec -it gha-runner-1 bash" +log "org view: https://github.com/organizations/${ORG}/settings/actions/runners" diff --git a/scripts/uninstall-gha-runners-macos.sh b/scripts/uninstall-gha-runners-macos.sh index e3a78e3..49afc9c 100755 --- a/scripts/uninstall-gha-runners-macos.sh +++ b/scripts/uninstall-gha-runners-macos.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # -# Tear down self-hosted GitHub Actions runners installed by -# setup-gha-runners-macos.sh. Stops the launchd agents, removes them, and -# de-registers each runner from the `wavekat` org. +# Tear down the Docker Desktop self-hosted GitHub Actions runners +# installed by setup-gha-runners-macos.sh. Stops and removes the +# containers, de-registers each runner from the `wavekat` org, and drops +# the persistent volumes. # # Usage: # ./uninstall-gha-runners-macos.sh @@ -10,13 +11,15 @@ # # A *remove* token can be fetched via: # gh api -X POST /orgs/wavekat/actions/runners/remove-token --jq .token +# +# Set RUNNER_KEEP_VOLUMES=1 to leave the volumes in place (keeps the +# warm npm/cargo caches for a later re-register). set -euo pipefail ORG="${RUNNER_ORG:-wavekat}" COUNT="${RUNNER_COUNT:-4}" -PREFIX="${RUNNER_PREFIX:-$(hostname -s)}" -BASE_DIR="${RUNNER_BASE_DIR:-${HOME}/actions-runners}" +KEEP_VOLUMES="${RUNNER_KEEP_VOLUMES:-0}" log() { printf '\033[1;36m==>\033[0m %s\n' "$*"; } warn() { printf '\033[1;33m!!\033[0m %s\n' "$*" >&2; } @@ -24,6 +27,13 @@ die() { printf '\033[1;31mxx\033[0m %s\n' "$*" >&2; exit 1; } [[ "$(uname -s)" == "Darwin" ]] || die "this script targets macOS (got $(uname -s))" +DOCKER="" +for candidate in docker /usr/local/bin/docker "${HOME}/.docker/bin/docker" /opt/homebrew/bin/docker; do + if command -v "${candidate}" >/dev/null 2>&1; then DOCKER="${candidate}"; break; fi +done +[[ -n "${DOCKER}" ]] || die "docker CLI not found" +"${DOCKER}" info >/dev/null 2>&1 || die "Docker Desktop is not running — launch it and re-run" + get_token() { if [[ -n "${RUNNER_TOKEN:-}" ]]; then printf '%s' "${RUNNER_TOKEN}" @@ -51,18 +61,32 @@ EOF TOKEN="$(get_token)" for i in $(seq 1 "${COUNT}"); do - NAME="${PREFIX}-${i}" - DIR="${BASE_DIR}/${NAME}" - log "removing runner ${NAME}" + CONTAINER="gha-runner-${i}" + log "removing ${CONTAINER}" - if [[ -d "${DIR}" ]]; then - ( cd "${DIR}" && ./svc.sh stop >/dev/null 2>&1 || true ) - ( cd "${DIR}" && ./svc.sh uninstall >/dev/null 2>&1 || true ) - ( cd "${DIR}" && ./config.sh remove --token "${TOKEN}" || true ) - rm -rf "${DIR}" + if "${DOCKER}" ps -a --format '{{.Names}}' | grep -qx "${CONTAINER}"; then + # De-register from inside the container while its volume is still + # mounted, so the org isn't left holding an offline ghost runner. + # `docker stop` first would kill run.sh; config.sh remove needs the + # runner idle, so stop it, then run config.sh in a one-shot exec. + "${DOCKER}" stop --time=120 "${CONTAINER}" >/dev/null 2>&1 || true + "${DOCKER}" run --rm \ + -v "${CONTAINER}:/home/runner/runner" \ + -w /home/runner/runner \ + --entrypoint /bin/bash \ + "$("${DOCKER}" inspect --format '{{.Config.Image}}' "${CONTAINER}" 2>/dev/null || echo wavekat/gha-runner:latest)" \ + -c "./config.sh remove --token '${TOKEN}' || true" >/dev/null 2>&1 || \ + warn "could not de-register ${CONTAINER} cleanly — remove it in the org runner settings" + "${DOCKER}" rm -f "${CONTAINER}" >/dev/null 2>&1 || true else - warn "no directory at ${DIR} — skipping" + warn "no container named ${CONTAINER} — skipping" + fi + + if [[ "${KEEP_VOLUMES}" != "1" ]]; then + "${DOCKER}" volume rm "${CONTAINER}" >/dev/null 2>&1 || true fi done log "done" +[[ "${KEEP_VOLUMES}" == "1" ]] && log "volumes kept (RUNNER_KEEP_VOLUMES=1)" +log "verify: https://github.com/organizations/${ORG}/settings/actions/runners"