diff --git a/.github/workflows/mirror-publish.yaml b/.github/workflows/mirror-publish.yaml new file mode 100644 index 00000000..31aa23e1 --- /dev/null +++ b/.github/workflows/mirror-publish.yaml @@ -0,0 +1,501 @@ +# Mirror publish — feed the public, deliverable-only mirror of this repository. +# +# This repository is the DEVELOPMENT repo. Its public face is a separate mirror +# repository that carries only the deliverable: the Helm charts, the installer, +# README / LICENSE / operator docs, the GitHub Pages chart index, and each +# GitHub release with its assets. Nothing reaches the mirror except through this +# workflow, and nothing leaves this workflow except what scripts/publish-guard.sh +# staged and cleared — allowlist (.publish-include), forbidden paths and strings +# (.publish-forbidden), gitleaks. The guard's header states the rules; every +# guard fails closed, and "could not tell" never publishes. +# +# Triggers +# workflow_run after "Release Helm Chart" completes successfully. NOT on +# `release: published`: that event fires when the release +# is created, BEFORE the chart workflow has attached the +# charts, installer and signed manifest and pushed the +# gh-pages index — a mirror cut then would copy a release +# with half its assets. The completed chart workflow is the +# moment every asset exists. head_branch of a release- +# triggered run is the tag (measured on v1.9.109 / -rc.1), +# head_sha the commit it points at; both are checked below. +# workflow_dispatch `dry-run` (default TRUE) runs every guard, prints the +# staged file list and stops. `tag` names a published +# release whose assets are staged too (empty = tree only). +# `dry-run: false` with a tag publishes; it still refuses +# while no mirror is configured. `strict` (default FALSE) +# promotes the guard's report tier to refusal (below). +# +# String tiers +# .publish-forbidden splits its needles in two. [strings-refuse] (mailboxes, +# cloud account identifiers, the private tenant needles) refuses on a hit. +# [strings-report] (internal ticket references, non-production hostnames) is +# COUNTED and printed — per-needle totals, ten most-hit files — but refuses +# only when the guard runs with --strict. This workflow passes --strict when +# the dispatch input `strict` is true OR the repository variable +# PUBLISH_STRICT is "true"; the variable is what arms the workflow_run path, +# which has no inputs. Flip the variable the day the decision to strip the +# report tier from the deliverable is taken. +# +# What runs and what is data +# The guard, the publisher and the policy lists (.publish-include, +# .publish-forbidden) are read from THIS workflow's own commit (github.sha: +# the default branch for workflow_run, the dispatched branch for +# workflow_dispatch) — the one checkout with a trusted ref. The release tag +# is fetched separately as DATA: its tree is staged and scanned, never +# executed, and it is fetched only after two checks — the tag must be the +# release's own tag_name, and it must resolve to the commit the release run +# ran on (workflow_run.head_sha; for a dispatch, the commit GitHub reports +# for the tag). A tag that names a branch, a moved tag, or a release whose +# tag differs from the run's are all refused before anything is read. +# +# Prereleases +# A prerelease (-rc.N) mirrors ONLY its GitHub release, marked prerelease, +# pinned to the mirror's current default-branch head. The default branch and +# the chart index are NOT pushed: they are what a customer installs from, and +# they keep the last stable release, exactly as the chart workflow keeps +# prereleases out of the Helm index. A prerelease therefore cannot be the +# first thing published to an empty mirror — the run refuses and says so. +# +# Older stable releases +# The default branch and the chart index are replaced only by the NEWEST +# stable release of this repository (what GitHub reports as releases/latest). +# A re-run of the chart workflow for an earlier tag, or a dispatch naming one, +# mirrors that release only — otherwise a rebuild of v1.8.0 after v1.9.0 had +# shipped would roll the public README, installer and charts back to v1.8.0 +# and, if v1.8.0 was already mirrored, stop there with the rollback in place. +# When releases/latest cannot be read the run refuses: "cannot tell" does not +# replace what customers install from. +# +# Target +# The mirror is named by the Actions VARIABLE `MIRROR_REPO` (a bare repo name +# in this organisation), or the `mirror-repo` dispatch input. It has NO +# default: until the mirror exists the job refuses to publish, and it always +# refuses a target equal to this repository — publishing onto the source +# would replace the default branch you are standing on. +# +# Credentials +# RELEASE_TRAIN_APP_ID / RELEASE_TRAIN_APP_PRIVATE_KEY mint an installation +# token scoped to the mirror only (permission contents:write); the default +# GITHUB_TOKEN only reads this repo's release. PUBLISH_FORBIDDEN_TENANTS +# carries the private needles for the string scan (one extended regex per +# line — see .publish-forbidden for why they are not committed); the guard +# refuses to run the scan without it. +name: Mirror publish + +on: + workflow_run: + workflows: ["Release Helm Chart"] + types: [completed] + workflow_dispatch: + inputs: + tag: + description: "Published release tag to mirror (vX.Y.Z or vX.Y.Z-rc.N). Empty: stage the tree only." + type: string + default: "" + dry-run: + description: "Run every guard and print the staged file list without publishing" + type: boolean + default: true + mirror-repo: + description: "Mirror repository name in this organisation (overrides the MIRROR_REPO variable)" + type: string + default: "" + strict: + description: "Promote the guard's [strings-report] tier to refusal (--strict); the PUBLISH_STRICT variable does the same for every run" + type: boolean + default: false + +permissions: + contents: read + +jobs: + publish: + name: Guard, then publish to the mirror + # A failed or cancelled release run publishes nothing; there is nothing to + # mirror and a red run here would only point at the wrong workflow. + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + # One publish at a time, never cancelled midway: a half-pushed mirror is + # worse than a late one. + concurrency: + group: mirror-publish + cancel-in-progress: false + env: + # Every event field is read through env, never interpolated into a + # script: a tag or branch name is attacker-shaped input. + EVENT_NAME: ${{ github.event_name }} + INPUT_TAG: ${{ inputs.tag }} + INPUT_DRY_RUN: ${{ inputs.dry-run }} + INPUT_MIRROR: ${{ inputs.mirror-repo }} + INPUT_STRICT: ${{ inputs.strict }} + RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + VAR_MIRROR: ${{ vars.MIRROR_REPO }} + VAR_STRICT: ${{ vars.PUBLISH_STRICT }} + steps: + - name: Check out the publishing tooling (this workflow's own commit) + # No `ref:` — github.sha, the commit this workflow file came from. The + # guard, the publisher and the policy lists run from here and only + # here; the release tag is fetched below into its own directory as + # data. This is also the repository the gh-pages fetch reads. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Resolve what to publish + id: plan + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = "workflow_run" ]; then + TAG="$RUN_HEAD_BRANCH"; DRY_RUN=false + else + TAG="$INPUT_TAG"; DRY_RUN="$INPUT_DRY_RUN" + fi + # Anything that is not exactly "false" is a dry run: fail closed. + [ "$DRY_RUN" = "false" ] || DRY_RUN=true + # --strict from the dispatch input or the repository variable; either + # alone arms it, and only the exact string "true" counts. + STRICT=false + if [ "$INPUT_STRICT" = "true" ] || [ "$VAR_STRICT" = "true" ]; then STRICT=true; fi + PRERELEASE=false; PUBLISH_TREE=true; EXPECT_SHA="" + if [ -n "$TAG" ]; then + if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$ ]]; then + echo "::error::'$TAG' is not a release tag — refusing to mirror it (a workflow_run whose head is a branch, or a mistyped dispatch)." + exit 1 + fi + # The release must be PUBLISHED here before it can be mirrored, and + # it must be the release OF this tag: a run whose head_branch names + # one tag while the release object carries another is refused. + if ! gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json tagName,isDraft,isPrerelease >"$RUNNER_TEMP/release.json"; then + echo "::error::no release '$TAG' on $GITHUB_REPOSITORY — nothing to mirror." + exit 1 + fi + RELEASE_TAG="$(jq -r .tagName "$RUNNER_TEMP/release.json")" + if [ "$RELEASE_TAG" != "$TAG" ]; then + echo "::error::release '$TAG' reports tag_name '$RELEASE_TAG' — the tag and the release disagree, refusing." + exit 1 + fi + if [ "$(jq -r .isDraft "$RUNNER_TEMP/release.json")" != "false" ]; then + echo "::error::release '$TAG' is a draft — only published releases are mirrored." + exit 1 + fi + PRERELEASE="$(jq -r .isPrerelease "$RUNNER_TEMP/release.json")" + # The tree push is armed only by an explicit `false`: a missing or + # malformed isPrerelease is refused, never read as "stable". + case "$PRERELEASE" in + true|false) ;; + *) echo "::error::release '$TAG' reports isPrerelease '$PRERELEASE' — not a boolean, refusing: only an explicit false may replace the mirror's default branch."; exit 1 ;; + esac + # The commit the tag MUST resolve to when it is fetched below. From + # a workflow_run that is the commit the release run ran on; from a + # dispatch it is the commit GitHub reports for the tag right now. + if [ "$EVENT_NAME" = "workflow_run" ]; then + EXPECT_SHA="$RUN_HEAD_SHA" + else + EXPECT_SHA="$(gh api "repos/$GITHUB_REPOSITORY/commits/$TAG" --jq .sha)" || EXPECT_SHA="" + fi + if [[ ! "$EXPECT_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::cannot determine the commit release '$TAG' was cut from (got '${EXPECT_SHA:-}') — refusing to fetch the tag." + exit 1 + fi + if [ "$PRERELEASE" = "true" ]; then + PUBLISH_TREE=false + echo "::notice::'$TAG' is a prerelease: only its GitHub release is mirrored (marked prerelease). The mirror's default branch and chart index are not pushed — they keep the last stable release." + else + # Only the NEWEST stable release replaces the mirror's default + # branch and chart index. A re-run for an older tag (or a dispatch + # naming one) would otherwise roll the public tree back to it. + # releases/latest is GitHub's own answer, not a sort done here. + LATEST="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name)" || LATEST="" + if [ -z "$LATEST" ]; then + echo "::error::cannot read the newest stable release of $GITHUB_REPOSITORY (releases/latest) — refusing: without it this run cannot tell whether '$TAG' may replace the mirror's default branch." + exit 1 + fi + if [ "$LATEST" != "$TAG" ]; then + PUBLISH_TREE=false + echo "::notice::'$TAG' is not the newest stable release ($LATEST is): only its GitHub release is mirrored. The mirror's default branch and chart index are not pushed — they keep the newest stable release." + fi + fi + else + if [ "$DRY_RUN" != "true" ]; then + echo "::error::a real publish needs a release tag; a tree-only run is dry-run only." + exit 1 + fi + fi + { + echo "tag=$TAG" + echo "dry_run=$DRY_RUN" + echo "expect_sha=$EXPECT_SHA" + echo "prerelease=$PRERELEASE" + echo "publish_tree=$PUBLISH_TREE" + echo "strict=$STRICT" + } >>"$GITHUB_OUTPUT" + echo "plan: event=$EVENT_NAME tag='${TAG:-}' expect_sha=${EXPECT_SHA:-} dry_run=$DRY_RUN prerelease=$PRERELEASE publish_tree=$PUBLISH_TREE strict=$STRICT" + + - name: Fetch the release tag as data + # Into a detached worktree under RUNNER_TEMP, outside the tooling + # checkout. Nothing in it is executed: the guard reads it and copies + # the allowlisted files out. The fetched tag must resolve to exactly + # the commit the plan step expects, or the run refuses — a tag moved + # after the release, or a release run on a different commit, does not + # get mirrored. `--no-tags` so only the one named ref arrives. + if: steps.plan.outputs.tag != '' + id: src + env: + TAG: ${{ steps.plan.outputs.tag }} + EXPECT_SHA: ${{ steps.plan.outputs.expect_sha }} + run: | + set -euo pipefail + if ! git fetch --no-tags --depth 1 origin "refs/tags/$TAG"; then + echo "::error::could not fetch tag '$TAG' from origin — refusing to mirror a release whose tag is not here." + exit 1 + fi + SHA="$(git rev-parse 'FETCH_HEAD^{commit}')" + if [ "$SHA" != "$EXPECT_SHA" ]; then + echo "::error::tag '$TAG' resolves to $SHA but the release was cut at $EXPECT_SHA — the tag has moved or the run is not this release's; refusing." + exit 1 + fi + git worktree add --detach "$RUNNER_TEMP/release-src" "$SHA" + echo "dir=$RUNNER_TEMP/release-src" >>"$GITHUB_OUTPUT" + echo "release source: $TAG at $SHA (data only) in $RUNNER_TEMP/release-src" + + - name: Install gitleaks (pinned, checksum-verified) + # Not preinstalled on ubuntu-latest. One pinned release, verified against + # its published checksum before it runs: the guard treats a missing + # scanner as "could not tell", so a failed install here is a red run, + # never a silent skip. + env: + GITLEAKS_VERSION: "8.30.1" + GITLEAKS_SHA256: "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/bin" + curl -fsSL --tlsv1.2 --retry 3 --connect-timeout 10 --max-time 120 \ + -o "$RUNNER_TEMP/gitleaks.tgz" \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + # Assert the digest is a full SHA-256 before trusting it. `sha256sum -c` + # reports a malformed line as "no properly formatted checksum lines + # found", and whether that is a non-zero exit depends on the coreutils + # build (GNU exits 1; the macOS sha256sum exits 0). An empty or + # truncated env var must be a hard failure here, not a verification + # that quietly checks nothing and lets the install proceed. + if [[ ! "${GITLEAKS_SHA256}" =~ ^[0-9a-f]{64}$ ]]; then + echo "::error::GITLEAKS_SHA256 is not a 64-character hex SHA-256 digest." + exit 1 + fi + echo "${GITLEAKS_SHA256} $RUNNER_TEMP/gitleaks.tgz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/gitleaks.tgz" -C "$RUNNER_TEMP/bin" gitleaks + chmod 0755 "$RUNNER_TEMP/bin/gitleaks" + echo "$RUNNER_TEMP/bin" >>"$GITHUB_PATH" + "$RUNNER_TEMP/bin/gitleaks" version + + - name: Write the private needle list + # The secret is written to a file, never echoed. An unset secret yields + # an empty file, which the guard refuses as "could not tell". + env: + PUBLISH_FORBIDDEN_TENANTS: ${{ secrets.PUBLISH_FORBIDDEN_TENANTS }} + run: | + set -euo pipefail + printf '%s\n' "$PUBLISH_FORBIDDEN_TENANTS" | sed '/^[[:space:]]*$/d' >"$RUNNER_TEMP/tenants.txt" + echo "private needle list: $(grep -c . "$RUNNER_TEMP/tenants.txt" || true) entr(y/ies)" + + - name: Download the release assets + if: steps.plan.outputs.tag != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.plan.outputs.tag }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/assets" + gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --dir "$RUNNER_TEMP/assets" + ls -l "$RUNNER_TEMP/assets" + + - name: Guard the tree and the release assets + # --source is the release worktree (the tooling checkout itself for a + # tree-only dry run); the allowlist and the forbidden list are always + # the tooling checkout's, so the policy that runs is the one reviewed + # on this branch, whatever the tag carries. + # Actions runs this body with -e; the guard's exit status is caught + # with `|| rc=$?` so a refusal still reaches the step summary and the + # step then exits with the guard's own status. + id: guard-tree + env: + TAG: ${{ steps.plan.outputs.tag }} + STRICT: ${{ steps.plan.outputs.strict }} + SRC_DIR: ${{ steps.src.outputs.dir }} + run: | + set -uo pipefail + args=(--source "${SRC_DIR:-.}" --include "$GITHUB_WORKSPACE/.publish-include" --forbidden "$GITHUB_WORKSPACE/.publish-forbidden" + --out "$RUNNER_TEMP/stage" --extra-forbidden "$RUNNER_TEMP/tenants.txt") + [ -z "$TAG" ] || args+=(--assets "$RUNNER_TEMP/assets") + [ "$STRICT" != "true" ] || args+=(--strict) + rc=0 + bash scripts/publish-guard.sh "${args[@]}" | tee "$RUNNER_TEMP/guard-tree.log" || rc=$? + { + echo "## Mirror publish — tree${TAG:+ + release $TAG}" + echo + echo '```' + cat "$RUNNER_TEMP/guard-tree.log" + echo '```' + } >>"$GITHUB_STEP_SUMMARY" + exit "$rc" + + - name: Guard the chart index (gh-pages) + # The mirror serves the Helm repository at the same Pages URL, so its + # gh-pages branch is a copy of this repo's. Guarded with the Pages + # allowlist (index + packages, nothing else) and the same forbidden + # lists. A missing gh-pages is not "nothing to do": the chart workflow + # just pushed to it, so its absence means this run is looking at the + # wrong repository. + id: guard-pages + env: + STRICT: ${{ steps.plan.outputs.strict }} + run: | + set -uo pipefail + git fetch --depth 1 origin gh-pages || { echo "::error::could not fetch gh-pages — the chart index is missing, refusing to mirror"; exit 2; } + git worktree add --detach "$RUNNER_TEMP/pages-src" FETCH_HEAD || exit 2 + args=(--source "$RUNNER_TEMP/pages-src" --include .publish-include-pages --forbidden .publish-forbidden + --extra-forbidden "$RUNNER_TEMP/tenants.txt" --out "$RUNNER_TEMP/stage-pages") + [ "$STRICT" != "true" ] || args+=(--strict) + rc=0 + bash scripts/publish-guard.sh "${args[@]}" | tee "$RUNNER_TEMP/guard-pages.log" || rc=$? + { + echo "## Mirror publish — chart index (gh-pages)" + echo + echo '```' + cat "$RUNNER_TEMP/guard-pages.log" + echo '```' + } >>"$GITHUB_STEP_SUMMARY" + exit "$rc" + + - name: Dry run — stop here + if: steps.plan.outputs.dry_run == 'true' + run: | + set -euo pipefail + MIRROR="${INPUT_MIRROR:-$VAR_MIRROR}" + echo "dry run: every guard passed; nothing was published." + if [ -z "$MIRROR" ]; then + echo "::notice::MIRROR_REPO is unset — a real run would refuse at the target check until the mirror repository exists and is named." + else + echo "a real run would publish to: $GITHUB_REPOSITORY_OWNER/$MIRROR" + fi + + - name: Resolve the mirror repository + # Run directly, never through `$(...)`: a refusal is a `::error::` line + # on stdout, and a capture would swallow it before `set -e` exits. The + # result (repo=, name=) is written by the script to $GITHUB_OUTPUT. + if: steps.plan.outputs.dry_run != 'true' + id: target + run: | + set -euo pipefail + bash scripts/publish-mirror.sh target --mirror "${INPUT_MIRROR:-$VAR_MIRROR}" --source-repo "$GITHUB_REPOSITORY" --output "$GITHUB_OUTPUT" + + - name: Mint a token scoped to the mirror + if: steps.plan.outputs.dry_run != 'true' + id: token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.RELEASE_TRAIN_APP_ID }} + private-key: ${{ secrets.RELEASE_TRAIN_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ steps.target.outputs.name }} + permission-contents: write + + - name: Confirm the mirror is a different, existing repository + if: steps.plan.outputs.dry_run != 'true' + id: mirror + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + REPO: ${{ steps.target.outputs.repo }} + run: | + set -euo pipefail + gh api "repos/$REPO" --jq '{full_name, default_branch, visibility}' >"$RUNNER_TEMP/mirror.json" + FULL="$(jq -r .full_name "$RUNNER_TEMP/mirror.json")" + if [ "$(printf '%s' "$FULL" | tr '[:upper:]' '[:lower:]')" = "$(printf '%s' "$GITHUB_REPOSITORY" | tr '[:upper:]' '[:lower:]')" ]; then + echo "::error::mirror '$FULL' resolves to this repository — refusing." + exit 1 + fi + echo "default_branch=$(jq -r .default_branch "$RUNNER_TEMP/mirror.json")" >>"$GITHUB_OUTPUT" + cat "$RUNNER_TEMP/mirror.json" + + - name: Push the tree to the mirror's default branch + # The newest stable release only: neither a prerelease nor an older + # stable tag replaces what customers install from (publish_tree=false, + # see the plan step). + if: steps.plan.outputs.dry_run != 'true' && steps.plan.outputs.publish_tree == 'true' + id: push + env: + MIRROR_TOKEN: ${{ steps.token.outputs.token }} + REPO: ${{ steps.target.outputs.repo }} + BRANCH: ${{ steps.mirror.outputs.default_branch }} + TAG: ${{ steps.plan.outputs.tag }} + run: | + set -euo pipefail + # Credentials come from the helper, read from the environment; the + # token is never part of a URL or a command line. The single quotes + # are the point: $MIRROR_TOKEN expands when git runs the helper. + # shellcheck disable=SC2016 + git config --global credential.helper '!f() { printf "username=x-access-token\npassword=%s\n" "$MIRROR_TOKEN"; }; f' + # Direct, not captured: a rejected push or a refused stage annotates + # the log; the result (result=, sha=) lands in $GITHUB_OUTPUT. + bash scripts/publish-mirror.sh tree --stage "$RUNNER_TEMP/stage/tree" --repo "$REPO" --branch "$BRANCH" --message "Publish $TAG" --output "$GITHUB_OUTPUT" + + - name: Push the chart index to the mirror's gh-pages + if: steps.plan.outputs.dry_run != 'true' && steps.plan.outputs.publish_tree == 'true' + env: + MIRROR_TOKEN: ${{ steps.token.outputs.token }} + REPO: ${{ steps.target.outputs.repo }} + TAG: ${{ steps.plan.outputs.tag }} + run: | + set -euo pipefail + bash scripts/publish-mirror.sh tree --stage "$RUNNER_TEMP/stage-pages/tree" --repo "$REPO" --branch gh-pages --message "Chart index at $TAG" + + - name: Release only — keep the mirror's default branch, pin the release to its head + # A prerelease or an older stable tag: nothing was pushed, so the + # release is created at the commit the mirror's default branch already + # has (the newest stable publish). An empty mirror has no such commit: + # neither can be the first publish, and the run says so instead of + # inventing a target. + if: steps.plan.outputs.dry_run != 'true' && steps.plan.outputs.publish_tree != 'true' + id: keep + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + REPO: ${{ steps.target.outputs.repo }} + BRANCH: ${{ steps.mirror.outputs.default_branch }} + TAG: ${{ steps.plan.outputs.tag }} + run: | + set -euo pipefail + if ! SHA="$(gh api "repos/$REPO/commits/$BRANCH" --jq .sha)" || [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::'$TAG' does not replace the mirror's default branch (a prerelease, or not the newest stable release) and the mirror has no commit on '$BRANCH' to pin its release to — the first publish to an empty mirror must be the newest stable release." + exit 1 + fi + echo "sha=$SHA" >>"$GITHUB_OUTPUT" + echo "release-only $TAG: default branch '$BRANCH' and gh-pages left untouched; release will be pinned to $SHA" + + - name: Create the release on the mirror + if: steps.plan.outputs.dry_run != 'true' + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + REPO: ${{ steps.target.outputs.repo }} + TAG: ${{ steps.plan.outputs.tag }} + SHA: ${{ steps.push.outputs.sha || steps.keep.outputs.sha }} + PRERELEASE: ${{ steps.plan.outputs.prerelease }} + run: | + set -euo pipefail + # Fixed notes, not the source release's generated ones: those list + # merged pull requests by title, which is development history, not + # the deliverable. + { + echo "tracebloc client $TAG." + echo + echo "Install with the one-liner in the README; the assets attached here are the" + echo "installer, the packaged Helm charts and the cosign-signed installer manifest." + echo "Verification recipe: docs/SUPPLY_CHAIN.md." + } >"$RUNNER_TEMP/notes.md" + args=(release --tag "$TAG" --repo "$REPO" --target "$SHA" --assets "$RUNNER_TEMP/stage/assets" --notes "$RUNNER_TEMP/notes.md") + [ "$PRERELEASE" != "true" ] || args+=(--prerelease) + bash scripts/publish-mirror.sh "${args[@]}" diff --git a/.publish-forbidden b/.publish-forbidden new file mode 100644 index 00000000..41f4d35c --- /dev/null +++ b/.publish-forbidden @@ -0,0 +1,79 @@ +# .publish-forbidden — what must never reach the public mirror, even if the +# allowlist (.publish-include) let it through by mistake. +# +# Read by scripts/publish-guard.sh. Four sections; `#` starts a comment. A +# header the guard does not know, a needle listed in both string tiers, or an +# empty [strings-refuse] is refused as "could not tell" (exit 2). +# +# [paths] gitignore-style names. A pattern containing `/` is +# anchored to the staged root; one without matches ANY path +# component; a trailing `/` means "as a directory". +# [strings-refuse] extended regexes, matched case-insensitively against +# every staged TEXT file. A hit REFUSES the publish and +# names the file and line (never the matched text). +# [strings-report] same syntax. Hits are COUNTED and printed (per-needle +# totals, ten most-hit files) but do not refuse — unless +# the guard runs with --strict, which promotes this tier +# to refusal. A needle moves up to [strings-refuse] the +# day it is decided the mirror must never carry it. +# [allow] exact tokens removed from a line before a needle is +# re-tested, so a line is spared only when the allowed +# token was the whole reason it hit. +# +# This file is ONE list read by both the guard and its tests; the tests write +# their own inputs and never iterate this file to check itself. + +[paths] +tests/ +scripts/tests/ +ci/ +.github/ +docs/rfcs/ +docs/migration-tools/ +CLAUDE.md +STYLE.md +Makefile +.cursor/ +*.go +go.mod +go.sum +__pycache__ +.DS_Store +.env* +*.pem +*.key +kubeconfig* + +[strings-refuse] +# Mailboxes (the public support address is spared under [allow]). +[A-Za-z0-9._%+-]+@tracebloc\.io +# AWS account identifiers and ARNs. +arn:aws: +[0-9]{12}\.dkr\.ecr\. +# +# CUSTOMER AND TENANT IDENTIFIERS ARE DELIBERATELY NOT LISTED HERE. This file +# is public, and a list of customer names would itself be the disclosure the +# scan exists to prevent. Those needles are supplied privately at publish time: +# the workflow writes the PUBLISH_FORBIDDEN_TENANTS secret (one needle per line, +# same regex syntax) to a file and passes it as --extra-forbidden; they join +# this tier. The guard refuses to run the scan when that list is missing or +# empty. + +[strings-report] +# Internal tracker and RFC identifiers — a reader of the mirror cannot open +# them. Counted until the decision to strip them from the deliverable (or to +# accept them) is taken; --strict refuses them. +backend# +rfcs# +RFC-0 +RFC-BACKEND +e2e-test-agent# +tracebloc/backend +# Non-production tracebloc hosts; same decision pending. +dev-api\.tracebloc\.io +stg-api\.tracebloc\.io +dev\.tracebloc\.io +stg\.tracebloc\.io + +[allow] +support@tracebloc\.io diff --git a/.publish-include b/.publish-include new file mode 100644 index 00000000..46c27a3b --- /dev/null +++ b/.publish-include @@ -0,0 +1,29 @@ +# .publish-include — what the public mirror of this repo MAY carry. +# +# Read by scripts/publish-guard.sh. One glob per line; `#` starts a comment. +# `*` and `?` do not cross `/`, `**` does; a leading `!` takes matching files +# back out. Only tracked files are considered. Anything not matched here is +# excluded by construction — .publish-forbidden is the second lock, and it +# refuses the excluded directories below even if the `!` lines were deleted. + +# The two Helm charts that `helm repo add tracebloc https://tracebloc.github.io/client` +# serves. Their unit-test suites and CI values never ship (see .helmignore). +client/** +!client/tests/** +!client/ci/** +ingestor/** + +# The installer: the two bootstraps plus every file they fetch and verify +# against scripts/manifest.sha256 (the sub-scripts live under scripts/lib/). +scripts/install.sh +scripts/install.ps1 +scripts/install-k8s.sh +scripts/install-k8s.ps1 +scripts/lib/** +scripts/manifest.sha256 + +# Front matter, and the operator docs README links to. One level only: +# docs/rfcs/ and docs/migration-tools/ stay home. +README.md +LICENSE +docs/*.md diff --git a/.publish-include-pages b/.publish-include-pages new file mode 100644 index 00000000..04c81f04 --- /dev/null +++ b/.publish-include-pages @@ -0,0 +1,7 @@ +# .publish-include-pages — what the mirror's GitHub Pages branch MAY carry. +# +# Read by scripts/publish-guard.sh with `--include` when the gh-pages branch is +# mirrored (it is what `helm repo add tracebloc https://tracebloc.github.io/client` +# reads). The branch holds the chart index and the packaged charts, nothing else. +index.yaml +*.tgz diff --git a/client/Chart.yaml b/client/Chart.yaml index 68ec1618..ecd2e38c 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.113 -appVersion: "1.9.113" +version: 1.9.117 +appVersion: "1.9.117" keywords: - tracebloc - kubernetes diff --git a/client/MIGRATION.md b/client/MIGRATION.md index 07e6d0fc..46111635 100644 --- a/client/MIGRATION.md +++ b/client/MIGRATION.md @@ -2,6 +2,95 @@ This guide explains how to migrate from the legacy per-platform charts (`aks/`, `bm/`, `eks/`, `oc/`) to the unified `client/` chart. +## Upgrading to 1.9.115 — `env.TRACEBLOC_DDP` defaults ON (RFC-0067 D7) + +`env.TRACEBLOC_DDP` now renders as **`"1"`** at the chart default. This is the +ARM step of backend#3147: RFC-0067 D7's precondition bundle holds — the engine +defaults its effective-batch mechanism to `per_rank_split` (tracebloc-engine#1010; +each rank trains on B/N so the effective batch stays B and the numerics are the +one-GPU experiment's, e2e-test-agent#444), a run on which that mechanism cannot +apply falls back to **one GPU** rather than to an uncompensated N×B +(client-runtime#553), and both were verified on the published `:dev` engine and +jobs-manager digests before this default flipped. + +**What changes on upgrade: nothing expands yet.** Multi-GPU needs **both** +switches — `TRACEBLOC_DDP` truthy **and** `env.MULTI_GPU_MIN_PARAMETERS` set +(1.9.103) — and the floor still has no default, so an edge that never set a +floor keeps one GPU per run, with `GPU_COUNT_SIZE_FLOOR_UNSET` in the +jobs-manager log. To arm an edge, set the floor: + +```bash +helm upgrade tracebloc/client --reuse-values --set-string env.MULTI_GPU_MIN_PARAMETERS=1000000 +``` + +(1,000,000 admits ResNet-18-class models and refuses LeNet-class ones; it is the +floor RFC-0067's G6 measured with, not a general recommendation — see +backend#3147 for where the speedup crossover sits on your hardware.) + +**Rollback lever, per edge:** + +```bash +helm upgrade tracebloc/client --reuse-values --set-string env.TRACEBLOC_DDP=0 +``` + +An explicit `"0"` travels to the jobs-manager and wins over the default; the +runtime reads it as OFF (`GPU_COUNT_SWITCH_OFF`) and spawns one GPU per run +whatever the floor says. `TRACEBLOC_AMP` is unchanged (still OFF by default). + +## Upgrading to 1.9.114 — training pods pull from the tracebloc registry (`ghcr.io`) by default + +The training-image host now follows `images.traceblocRegistry`: `JOB_IMAGE_HOST`, +the registry prefix the jobs-manager stamps onto every training image it spawns +(`tracebloc/client--:`), renders as **`ghcr.io/`** at +the chart default instead of `docker.io/`, on both jobs-manager containers. It is +resolved by the same `tracebloc.tbRegistry` helper as the control-plane images, +as ONE precedence chain: a `global.imageRegistry` mirror wins, then +`images.traceblocRegistry`, then the chart default. From this version the +control plane (moved in 1.9.113) and the training pods pull from the same +registry and cannot be pointed at different ones. + +**Why:** every training image is published to GHCR at the same digests as its +Docker Hub copy (the GHCR migration), so this changes where the training pods +pull from, not which bytes run. + +**What you need to do: nothing for most edges.** + +- **Egress.** No new host: `ghcr.io` (and `pkg-containers.githubusercontent.com`, + where GHCR redirects layer downloads) is already required for the + control-plane images since 1.9.113 and for the ingestor image before that. + If your allowlist was built by hand from an older egress table, add both + before upgrading. Docker Hub is still needed for k3s, `tracebloc/mysql-client` + and busybox. +- **One rollout, then one pull per task.** The jobs-manager pod template changes + (`JOB_IMAGE_HOST`), so the upgrade rolls the jobs-manager once. The next + experiment of each task pulls its training image from `ghcr.io` — a full pull + the first time, as after any tag move; the digest-pinned spawn path + (`TRAINING_IMAGE_DIGESTS`) is unaffected, the same digest exists on both + registries. +- **Mirrors.** Edges with `global.imageRegistry` set are unaffected: the mirror + re-homes every image, `JOB_IMAGE_HOST` included, and always wins — exactly as + before. +- **Runtime default.** The chart always sets `JOB_IMAGE_HOST`, so the + client-runtime's own fallback for an *unset* variable (changed separately, in + that project) only ever applies to installs from before the chart carried the + key. + +**Rollback (per edge):** the same knob as 1.9.113 — it moves the control plane +AND the training-image host back to Docker Hub together, and, being +user-supplied, it persists across the fleet auto-upgrade until you clear it: + +```bash +helm upgrade tracebloc/client -n \ + --reset-then-reuse-values --set images.traceblocRegistry=docker.io +``` + +Confirm which host the training pods will pull from: + +```bash +kubectl get deploy -n -jobs-manager \ + -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="JOB_IMAGE_HOST")].value}{"\n"}' +``` + ## Upgrading to 1.9.113 — the control-plane images pull from `ghcr.io` by default `images.traceblocRegistry` now defaults to **`ghcr.io`**: the four diff --git a/client/templates/NOTES.txt b/client/templates/NOTES.txt index c06bfad3..87543597 100644 --- a/client/templates/NOTES.txt +++ b/client/templates/NOTES.txt @@ -5,7 +5,7 @@ {{ "\033[1;35m" }}Components:{{ "\033[0m" }} - {{ "\033[1;34m" }}Jobs Manager:{{ "\033[0m" }} {{ "\033[0;33m" }}{{ include "tracebloc.fullname" . }}-jobs-manager{{ "\033[0m" }} - {{ "\033[1;34m" }}MySQL Host:{{ "\033[0m" }} {{ "\033[0;33m" }}mysql-client{{ "\033[0m" }} -{{- if ne .Values.resourceMonitor false }} +{{- if (include "tracebloc.resourceMonitorEnabled" .) }} - {{ "\033[1;34m" }}Resource Monitor (DS):{{ "\033[0m" }} {{ "\033[0;33m" }}{{ include "tracebloc.resourceMonitorName" . }}{{ "\033[0m" }} {{- end }} @@ -13,7 +13,7 @@ {{ "\033[1;34m" }}Service Account:{{ "\033[0m" }} {{ "\033[0;33m" }}{{ include "tracebloc.serviceAccountName" . }}{{ "\033[0m" }} {{ "\033[1;34m" }}Secret:{{ "\033[0m" }} {{ "\033[0;33m" }}{{ include "tracebloc.secretName" . }}{{ "\033[0m" }} {{ "\033[1;34m" }}RBAC scope:{{ "\033[0m" }} {{ "\033[0;33m" }}{{ if ne .Values.clusterScope false }}Cluster{{ else }}Namespace{{ end }}{{ "\033[0m" }} - {{ "\033[1;34m" }}Image registry:{{ "\033[0m" }} {{ "\033[1;32m" }}{{ include "tracebloc.tbRegistry" . }}{{ "\033[0m" }} (tracebloc control-plane images) + {{ "\033[1;34m" }}Image registry:{{ "\033[0m" }} {{ "\033[1;32m" }}{{ include "tracebloc.tbRegistry" . }}{{ "\033[0m" }} (tracebloc control-plane + training images) {{- if (default dict .Values.hostPath).enabled }} {{ "\033[1;34m" }}Storage:{{ "\033[0m" }} {{ "\033[0;33m" }}hostPath (bare-metal){{ "\033[0m" }} {{ "\033[1;34m" }}Host dirs:{{ "\033[0m" }} {{ "\033[0;33m" }}/tracebloc/data, /tracebloc/logs, /tracebloc/mysql (on the node){{ "\033[0m" }} diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index aaf9346c..0f7360b3 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -126,6 +126,42 @@ tracebloc.io/seal-check-name: {{ .name | quote }} {{ include "tracebloc.fullname" . }}-resource-monitor {{- end }} +{{/* + tracebloc.resourceMonitorEnabled — the SINGLE reader of "is the resource-monitor + on", coalescing the two value shapes during the RFC-0076 alias window + (remove_by: 2026-12-31, client#1009): + + legacy scalar resourceMonitor: + new object resourceMonitor.enabled: (D2: .enabled) + + This is a bool→object rename, so a stored values.yaml or a bare + `--set resourceMonitor=true` still arrives as a SCALAR. Reading + `.Values.resourceMonitor.enabled` blindly would `fail` with "can't evaluate + field enabled in interface {}" on the scalar and, on a `--reuse-values` + upgrade that carries the scalar forward, silently drop the setting. So decide + the shape with kindIs and prefer the new `.enabled` form: + + map -> .enabled, defaulting to true when the key is absent + bool -> the scalar itself + absent -> enabled (the historical default: `ne false` was true) + + Effective behaviour is unchanged: resourceMonitor.enabled=true does exactly + what resourceMonitor=true did. Emits "true" or nothing, so callers use + `(include "tracebloc.resourceMonitorEnabled" .)` in an `and`/`or` and + `not (include ...)` for the disabled case — the same idiom as + tracebloc.nodeAgentsInUse. +*/}} +{{- define "tracebloc.resourceMonitorEnabled" -}} +{{- $rm := .Values.resourceMonitor -}} +{{- if kindIs "map" $rm -}} +{{- if ne (dig "enabled" true $rm) false -}}true{{- end -}} +{{- else if kindIs "invalid" $rm -}} +{{- "true" -}} +{{- else -}} +{{- if ne $rm false -}}true{{- end -}} +{{- end -}} +{{- end }} + {{- define "tracebloc.rbacName" -}} {{ include "tracebloc.fullname" . }}-jobs-manager-rbac {{- end }} @@ -384,11 +420,12 @@ nvidia-device-plugin-daemonset * `resourceMonitor: false` — there is no DaemonSet at all, so there is nothing to reconcile and a cross-namespace `set image` would just fail. - Nil-safe: `.Values.resourceMonitor` absent reads as enabled, matching the - `ne .Values.resourceMonitor false` gate on the DaemonSet itself. + Nil-safe via tracebloc.resourceMonitorEnabled, which absent reads as enabled, + matching the gate on the DaemonSet itself and honouring both the legacy scalar + and the new resourceMonitor.enabled object form. */}} {{- define "tracebloc.resourceMonitorRefreshPinned" -}} -{{- if eq .Values.resourceMonitor false -}} +{{- if not (include "tracebloc.resourceMonitorEnabled" .) -}} true {{- else if (default dict (default dict .Values.images).resourceMonitor).digest -}} true @@ -603,21 +640,26 @@ docker.io ghcr.io {{- end -}} {{/* -tracebloc.tbRegistry — the registry the tracebloc-PUBLISHED control-plane images -(tracebloc/jobs-manager, tracebloc/pods-monitor, tracebloc/resource-monitor, and -the requests-proxy, which runs the jobs-manager image) are pulled from. +tracebloc.tbRegistry — the registry the tracebloc-PUBLISHED images are pulled +from: the control-plane images (tracebloc/jobs-manager, tracebloc/pods-monitor, +tracebloc/resource-monitor, and the requests-proxy, which runs the jobs-manager +image) AND the host jobs-manager stamps onto every training image it spawns +(JOB_IMAGE_HOST, rendered as "/" on both jobs-manager containers). -ONE precedence chain, so the four call sites, the image-refresh CronJob and -NOTES.txt cannot disagree about where those images live: +ONE precedence chain, so the four control-plane call sites, the two +JOB_IMAGE_HOST sites, the image-refresh CronJob and NOTES.txt cannot disagree +about where those images live: 1. `global.imageRegistry` — a private mirror re-homes EVERY image the chart pulls (#585), tracebloc/* included. It always wins. - 2. `images.traceblocRegistry` — the tracebloc-only knob: moves just the - tracebloc-published images, leaving busybox, - squid, alpine/*, the device plugins and the - ingestor where they are. Also the per-edge - rollback: set it to the previous registry. + 2. `images.traceblocRegistry` — the tracebloc-only knob: moves the + tracebloc-published images -- control plane + and training-image host TOGETHER -- leaving + busybox, squid, alpine/*, the device plugins + and the ingestor where they are. Also the + per-edge rollback: set it to the previous + registry. 3. "ghcr.io" — the chart default since the GHCR migration. The images are still dual-published to Docker Hub at the same digests, so @@ -625,8 +667,9 @@ NOTES.txt cannot disagree about where those images live: NOT routed through here, on purpose: `tracebloc/mysql-client` (frozen, digest-pinned, published only to Docker Hub — see images.mysqlClient), the -third-party images (each has its own `registry` key), and — for now — the -training-image host JOB_IMAGE_HOST, which moves in its own step. +third-party images (each has its own `registry` key), and the ingestor, which is +named by full repository (images.ingestor.repository, already on ghcr.io) and +follows only the global mirror. Every read is nil-guarded and `| default`-chained: values.yaml ships `global.imageRegistry: ""` (the key EXISTS, so `dig`'s own fallback never @@ -1550,7 +1593,7 @@ https://api.tracebloc.io/ became a second tenant, two of them were widened and the rest were not." The tri-state made `enabled` a second copy of the answer for a third time. */ -}} -{{- if or (ne .Values.resourceMonitor false) (eq (include "tracebloc.telemetryCollectorState" .) "enabled") }}true{{ end -}} +{{- if or (include "tracebloc.resourceMonitorEnabled" .) (eq (include "tracebloc.telemetryCollectorState" .) "enabled") }}true{{ end -}} {{- end -}} {{/* diff --git a/client/templates/image-refresh-cronjob.yaml b/client/templates/image-refresh-cronjob.yaml index 352c0921..c57cf07a 100644 --- a/client/templates/image-refresh-cronjob.yaml +++ b/client/templates/image-refresh-cronjob.yaml @@ -500,6 +500,13 @@ data: restart_needed=0 annotate_args="" + # Pass-0 diagnostic annotations -- the `stale-pin-` clears and + # finding-sets -- accumulate SEPARATELY from annotate_args and are written in + # their own bounded annotate BEFORE the restart block (#1008 item 1). + # They touch only the stale-pin keys, never `last-refreshed--digest`, + # so they are safe to land before the rollout; keeping them out of the final + # annotate is what lets them survive a latched-flap tick (see the write site). + stale_pin_args="" # `kubectl set image` argument lists, accumulated per WORKLOAD (a single # `set image` call can carry several container=ref pairs, so the # two-container jobs-manager Deployment is re-imaged in one patch and @@ -585,7 +592,7 @@ data: disabled_stale_key="tracebloc.io/stale-pin-${repo#*/}" if [ -n "$(get_annotation "$disabled_stale_key" || true)" ]; then log " clearing ${disabled_stale_key}: this image is no longer pinned by a digest" - annotate_args="$annotate_args ${disabled_stale_key}-" + stale_pin_args="$stale_pin_args ${disabled_stale_key}-" fi continue fi @@ -613,7 +620,7 @@ data: # (@saqlainsyed007 + Bugbot on client#824.) if [ -n "$(get_annotation "$stale_key" || true)" ]; then log " clearing a previous ${stale_key}: the pin is current again" - annotate_args="$annotate_args ${stale_key}-" + stale_pin_args="$stale_pin_args ${stale_key}-" fi else log " WARN: PIN IS STALE. values pin ${pin_digest}" @@ -624,7 +631,7 @@ data: log " in values (backend#2458)." # Queryable after the log ages out, on the same object the refresh # annotations use, so `kubectl describe` shows pin state beside refresh state. - annotate_args="$annotate_args ${stale_key}=${pin_latest}" + stale_pin_args="$stale_pin_args ${stale_key}=${pin_latest}" fi continue fi @@ -637,7 +644,7 @@ data: unpinned_stale_key="tracebloc.io/stale-pin-${repo#*/}" if [ -n "$(get_annotation "$unpinned_stale_key" || true)" ]; then log " clearing ${unpinned_stale_key}: this image is no longer pinned" - annotate_args="$annotate_args ${unpinned_stale_key}-" + stale_pin_args="$stale_pin_args ${unpinned_stale_key}-" fi latest="$(get_latest_digest "$repo" "$IMAGE_TAG" "$IMAGE_REGISTRY" || true)" @@ -881,6 +888,32 @@ data: esac done + # Pass-0 annotations land HERE, before the restart block (#1008 item 1). + # The restart block's #563 flap guard does `WARN + FLAP_KEY + exit 0` + # once refresh-attempt >= MAX_REFRESH_ATTEMPTS -- BEFORE the digest-record + # annotate at the end of the tick. So on a tick that is both off-digest + # (restart_needed=1) and latched, batching the stale-pin writes into that + # final annotate dropped them: a stale-pin CLEAR that never landed leaves a + # FALSE "pin is stale" finding to persist forever (a write-only annotation + # outliving its problem -- the class client#824's clear paths fixed), and it + # is dropped on the exact tick refresh is dead, when the finding matters most. + # These touch only the `stale-pin-` keys, never + # `last-refreshed--digest`, so writing them before the rollout cannot + # affect the `recorded == latest` skip logic -- unlike the digest record, + # which MUST stay after a successful `rollout status`, since annotating the + # digest before a failed rollout would freeze the workload on the old image + # (@shujaatTracebloc on #1008). NON-FATAL, like the SKIP_KEY clear above + # (backend#2007): a transient failure on a diagnostic annotation must not + # abort the tick before the re-image; a stale value is re-reconciled next tick. + if [ -n "$stale_pin_args" ]; then + log "updating stale-pin annotations:$stale_pin_args" + # shellcheck disable=SC2086 # word-split stale_pin_args intentional + if ! sp_err="$(kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ + $stale_pin_args --overwrite --request-timeout=15s 2>&1 >/dev/null)"; then + log " WARNING: could not update stale-pin annotations on deployment/${DEPLOYMENT_NAME}: ${sp_err:-unknown error}. Continuing -- these are diagnostic bookkeeping and a stale value is re-reconciled on the next tick." + fi + fi + # Order matters: rollout FIRST, annotate AFTER `rollout status` # succeeds. Annotating first would let a failed rollout silently # freeze the deployment on the old image (next tick sees diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index ac24ffcc..4a016941 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -672,13 +672,18 @@ spec: value: "http://egress-proxy-service:{{ (default dict .Values.egressProxy).port | default 3128 }}" {{- end }} # JOB_IMAGE_HOST is the registry prefix jobs-manager stamps onto every - # training-job image it spawns. Honour a private mirror set via - # global.imageRegistry (#585) so an air-gapped/mirrored install pulls job - # images from the same registry as the chart's own images; defaults to - # docker.io/ when no mirror is configured. Nil-guarded for pre-this-key - # --reset-then-reuse-values upgrades. + # training-job image it spawns. Rendered from tracebloc.tbRegistry -- the + # SAME precedence chain as the control-plane images, so the two cannot + # point at different registries: a private mirror set via + # global.imageRegistry (#585) wins, so an air-gapped/mirrored install + # pulls job images from the same registry as the chart's own images; + # else images.traceblocRegistry (the per-edge rollback knob); else the + # chart default, ghcr.io since the GHCR migration (the training images + # are published there at the same digests as on Docker Hub). The helper + # is nil-guarded for pre-this-key --reset-then-reuse-values upgrades. + # Trailing slash: the runtime concatenates : verbatim. - name: JOB_IMAGE_HOST - value: {{ printf "%s/" (dig "imageRegistry" "docker.io" (.Values.global | default dict) | default "docker.io") | quote }} + value: {{ printf "%s/" (include "tracebloc.tbRegistry" .) | quote }} {{- if include "tracebloc.useImagePullSecrets" . }} # IMAGE_PULL_SECRET_NAME is the pull Secret jobs-manager puts on the # TRAINING pods it spawns (backend#2119). It has to be injected rather @@ -745,7 +750,7 @@ spec: # populated would list a namespace that isn't rendered — a 403 every # heartbeat for a workload the operator turned off. - name: NODE_AGENTS_NAMESPACE - value: {{ if ne .Values.resourceMonitor false }}{{ dig "namespace" "name" "" (.Values.nodeAgents | default dict) | quote }}{{ else }}""{{ end }} + value: {{ if (include "tracebloc.resourceMonitorEnabled" .) }}{{ dig "namespace" "name" "" (.Values.nodeAgents | default dict) | quote }}{{ else }}""{{ end }} # backend#664 (Utilization Ladder L0): with NEITHER env.RESOURCE_REQUESTS # nor env.RESOURCE_LIMITS set, BOTH vars are omitted and jobs-manager # sizes the envelope from node allocatable — BUT ONLY IF @@ -952,10 +957,11 @@ spec: key: CLIENT_PASSWORD - name: CLIENT_LOGS_PVC value: {{ include "tracebloc.clientLogsPvc" . | quote }} - # See the api container's JOB_IMAGE_HOST note (#585): honour a private - # mirror set via global.imageRegistry, defaulting to docker.io/. + # See the api container's JOB_IMAGE_HOST note (#585): rendered from + # tracebloc.tbRegistry, so a global.imageRegistry mirror wins, then + # images.traceblocRegistry, then the chart default (ghcr.io). - name: JOB_IMAGE_HOST - value: {{ printf "%s/" (dig "imageRegistry" "docker.io" (.Values.global | default dict) | default "docker.io") | quote }} + value: {{ printf "%s/" (include "tracebloc.tbRegistry" .) | quote }} - name: CLIENT_ENV value: {{ include "tracebloc.clientEnv" . | quote }} # See the api container's RESOURCE_REQUESTS note (backend#664, diff --git a/client/templates/rbac.yaml b/client/templates/rbac.yaml index 07765305..32341007 100644 --- a/client/templates/rbac.yaml +++ b/client/templates/rbac.yaml @@ -232,7 +232,7 @@ roleRef: {{- end }} {{- $nodeAgentsNs := dig "namespace" "name" "" (.Values.nodeAgents | default dict) }} -{{- if and (ne .Values.resourceMonitor false) $nodeAgentsNs (ne $nodeAgentsNs .Release.Namespace) }} +{{- if and (include "tracebloc.resourceMonitorEnabled" .) $nodeAgentsNs (ne $nodeAgentsNs .Release.Namespace) }} --- {{/* jobs-manager reads the resource-monitor DaemonSet for the heartbeat version diff --git a/client/templates/resource-monitor-daemonset.yaml b/client/templates/resource-monitor-daemonset.yaml index 81bf11c4..08cec0eb 100644 --- a/client/templates/resource-monitor-daemonset.yaml +++ b/client/templates/resource-monitor-daemonset.yaml @@ -1,4 +1,4 @@ -{{- if ne .Values.resourceMonitor false }} +{{- if (include "tracebloc.resourceMonitorEnabled" .) }} {{/* Pre-flight: resource-monitor polls the metrics.k8s.io API, so metrics-server must be registered. We probe kube-system via `lookup` first — that returns @@ -66,7 +66,7 @@ {{- $preflight = "skipped-by-values" -}} {{- else -}} {{- if not (lookup "apiregistration.k8s.io/v1" "APIService" "" "v1beta1.metrics.k8s.io") -}} - {{- fail "resourceMonitor is enabled but the metrics.k8s.io/v1beta1 API is not registered. Install metrics-server (https://github.com/kubernetes-sigs/metrics-server) or set resourceMonitor: false. See SECURITY.md.\n\nIf THIS line was instead an `apiservices ... is forbidden` error, the problem is the caller's RBAC, not metrics-server: APIService is cluster-scoped and the built-in `admin` ClusterRole excludes it. Set nodeAgents.metricsServerPreflight: false to skip this check, or run the upgrade with cluster-scope read on apiservices (backend#2469)." -}} + {{- fail "resourceMonitor is enabled but the metrics.k8s.io/v1beta1 API is not registered. Install metrics-server (https://github.com/kubernetes-sigs/metrics-server) or set resourceMonitor.enabled: false. See SECURITY.md.\n\nIf THIS line was instead an `apiservices ... is forbidden` error, the problem is the caller's RBAC, not metrics-server: APIService is cluster-scoped and the built-in `admin` ClusterRole excludes it. Set nodeAgents.metricsServerPreflight: false to skip this check, or run the upgrade with cluster-scope read on apiservices (backend#2469)." -}} {{- end -}} {{- $preflight = "satisfied-by-apiservice" -}} {{- end -}} diff --git a/client/templates/resource-monitor-rbac.yaml b/client/templates/resource-monitor-rbac.yaml index da0339dc..5a68bf2c 100644 --- a/client/templates/resource-monitor-rbac.yaml +++ b/client/templates/resource-monitor-rbac.yaml @@ -1,4 +1,4 @@ -{{- if ne .Values.resourceMonitor false }} +{{- if (include "tracebloc.resourceMonitorEnabled" .) }} --- apiVersion: v1 kind: ServiceAccount @@ -27,7 +27,7 @@ metadata: the training/jobs isolation footprint elsewhere -- it must not cripple node telemetry by leaving the DaemonSet without the permissions it cannot run without. If a deployment genuinely cannot allow any cluster-scoped read, disable the - monitor entirely via .Values.resourceMonitor=false rather than deploying it broken. + monitor entirely via .Values.resourceMonitor.enabled=false rather than deploying it broken. */}} --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/client/templates/resource-monitor-scc.yaml b/client/templates/resource-monitor-scc.yaml index f9fba66c..b6b0f1f5 100644 --- a/client/templates/resource-monitor-scc.yaml +++ b/client/templates/resource-monitor-scc.yaml @@ -1,4 +1,4 @@ -{{- if and (ne .Values.resourceMonitor false) .Values.openshift.scc.enabled }} +{{- if and (include "tracebloc.resourceMonitorEnabled" .) .Values.openshift.scc.enabled }} --- apiVersion: security.openshift.io/v1 kind: SecurityContextConstraints diff --git a/client/templates/secrets.yaml b/client/templates/secrets.yaml index 474fc71c..cecfbeb3 100644 --- a/client/templates/secrets.yaml +++ b/client/templates/secrets.yaml @@ -510,7 +510,7 @@ data: {{- if (include "tracebloc.bootstrapDbReparent" .) }} DB_BOOTSTRAP_PASSWORD: {{ $bootstrapDbPassword | b64enc | quote }} {{- end }} -{{- if and (ne .Values.resourceMonitor false) (ne .Values.nodeAgents.namespace.name .Release.Namespace) }} +{{- if and (include "tracebloc.resourceMonitorEnabled" .) (ne .Values.nodeAgents.namespace.name .Release.Namespace) }} --- # Mirrored into the node-agents namespace so the resource-monitor DaemonSet # can read CLIENT_ID / CLIENT_PASSWORD via secretKeyRef. Secrets are diff --git a/client/tests/global_image_registry_test.yaml b/client/tests/global_image_registry_test.yaml index 19c17915..a0c4e2d8 100644 --- a/client/tests/global_image_registry_test.yaml +++ b/client/tests/global_image_registry_test.yaml @@ -5,9 +5,10 @@ suite: global.imageRegistry private-mirror re-homing (#585) # convention. Two invariants are pinned here: # 1. Set -> every image (tracebloc/*, the spawned ingestor + training jobs, # and the alpine/*, ubuntu/squid utility images) carries the mirror prefix. -# 2. Unset -> the chart's own defaults, untouched: tracebloc/* on -# images.traceblocRegistry (ghcr.io since the GHCR migration), squid on -# docker.io, ingestor on ghcr.io, alpine/* unprefixed (docker.io implicit). +# 2. Unset -> the chart's own defaults, untouched: tracebloc/* AND the +# training-image host JOB_IMAGE_HOST on images.traceblocRegistry (ghcr.io +# since the GHCR migration), squid on docker.io, ingestor on ghcr.io, +# alpine/* unprefixed (docker.io implicit). # Precedence guards: an explicit images.ingestor.repository still wins over the # mirror (someone who names a full repo means it); an explicit per-image # registry (egressProxy.image.registry) is overridden by the global mirror. @@ -105,6 +106,7 @@ tests: value: "mirror.corp.example/tracebloc/ingestor" - it: re-homes the training-job image host (JOB_IMAGE_HOST) onto the mirror + # Both jobs-manager containers carry the variable; both read tracebloc.tbRegistry. template: templates/jobs-manager-deployment.yaml set: global: @@ -115,6 +117,11 @@ tests: content: name: JOB_IMAGE_HOST value: "mirror.corp.example/" + - contains: + path: spec.template.spec.containers[1].env + content: + name: JOB_IMAGE_HOST + value: "mirror.corp.example/" # --------------------------------------------------------------------------- # 2. Mirror UNSET -> the chart's own defaults, untouched @@ -135,14 +142,19 @@ tests: name: INGESTOR_IMAGE_REPOSITORY value: "ghcr.io/tracebloc/ingestor" - - it: leaves JOB_IMAGE_HOST on docker.io when no mirror is set + - it: leaves JOB_IMAGE_HOST on the tracebloc registry default (ghcr.io/) when no mirror is set template: templates/jobs-manager-deployment.yaml asserts: - contains: path: spec.template.spec.containers[0].env content: name: JOB_IMAGE_HOST - value: "docker.io/" + value: "ghcr.io/" + - contains: + path: spec.template.spec.containers[1].env + content: + name: JOB_IMAGE_HOST + value: "ghcr.io/" - it: leaves the alpine/helm auto-upgrade image unprefixed when no mirror is set template: templates/auto-upgrade-cronjob.yaml diff --git a/client/tests/image_refresh_test.yaml b/client/tests/image_refresh_test.yaml index ad457ebf..488314d3 100644 --- a/client/tests/image_refresh_test.yaml +++ b/client/tests/image_refresh_test.yaml @@ -1020,3 +1020,49 @@ tests: - notMatchRegex: path: data["image-refresh.sh"] pattern: 'recorded="\$\(get_annotation "\$key" \|\| true\)"' + + - it: stale-pin annotations are written BEFORE the restart block, so they survive a latched flap + # Guards #1008 item 1. The #563 flap guard does WARN + FLAP_KEY + + # exit 0 once refresh-attempt >= MAX, BEFORE the final digest-record annotate. + # Batching the stale-pin CLEARS into that final annotate dropped them on a + # tick that is both off-digest and latched -- leaving a FALSE stale-pin + # finding to persist. They now accumulate in their own list and are annotated + # above the restart block. image-refresh-latched-annotate.bats asserts the + # BEHAVIOUR; these lock the code shapes. + template: templates/image-refresh-cronjob.yaml + documentIndex: 0 + asserts: + # ALL FOUR stale-pin writes go to their own accumulator, not annotate_args. + # Every key gets a positive (goes to stale_pin_args) AND a negative (does NOT + # ride annotate_args) -- Bugbot Low + @saadqbal on #1039: a key covered by + # neither could be re-batched into annotate_args and dropped on a latched + # tick while both suites stayed green (this guard's own thesis, one level up). + # The two clears: + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'stale_pin_args="\$stale_pin_args \$\{stale_key\}-"' + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'stale_pin_args="\$stale_pin_args \$\{unpinned_stale_key\}-"' + # the disabled-monitor clear (the third clear -- was uncovered): + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'stale_pin_args="\$stale_pin_args \$\{disabled_stale_key\}-"' + # the finding-SET (a lost set means a real staleness goes unreported): + - matchRegex: + path: data["image-refresh.sh"] + pattern: 'stale_pin_args="\$stale_pin_args \$\{stale_key\}=\$\{pin_latest\}"' + # and that accumulator is annotated BEFORE the restart block (ordering) + - matchRegex: + path: data["image-refresh.sh"] + pattern: '(?s)if \[ -n "\$stale_pin_args" \]; then.*if \[ "\$restart_needed" -eq 1 \]; then' + # NONE of the stale-pin keys may ride the final digest-record annotate_args + - notMatchRegex: + path: data["image-refresh.sh"] + pattern: 'annotate_args="\$annotate_args \$\{stale_key\}' + - notMatchRegex: + path: data["image-refresh.sh"] + pattern: 'annotate_args="\$annotate_args \$\{unpinned_stale_key\}' + - notMatchRegex: + path: data["image-refresh.sh"] + pattern: 'annotate_args="\$annotate_args \$\{disabled_stale_key\}' diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 1c4d4234..5a20b412 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -106,13 +106,22 @@ tests: path: spec.template.spec.containers[0].image pattern: "^ghcr\\.io/tracebloc/" - - it: should set JOB_IMAGE_HOST to docker.io + - it: should set JOB_IMAGE_HOST to the tracebloc registry default (ghcr.io/) on both containers + # tracebloc.tbRegistry's chart default with the trailing slash the runtime + # concatenates onto :. The api and pods-monitor containers read + # the same helper, so both are pinned; tests/tracebloc_registry_test.yaml + # covers the knob and the mirror. asserts: - contains: path: spec.template.spec.containers[0].env content: name: JOB_IMAGE_HOST - value: "docker.io/" + value: "ghcr.io/" + - contains: + path: spec.template.spec.containers[1].env + content: + name: JOB_IMAGE_HOST + value: "ghcr.io/" # --------------------------------------------------------------------------- # Ingestor image distribution (backend#1028 pin, backend#1245 distribution). diff --git a/client/tests/resource_monitor_test.yaml b/client/tests/resource_monitor_test.yaml index 2e9c7d24..4f163a85 100644 --- a/client/tests/resource_monitor_test.yaml +++ b/client/tests/resource_monitor_test.yaml @@ -251,3 +251,99 @@ tests: - equal: path: metadata.namespace value: tracebloc-node-agents + + # ── RFC-0076 bool→object alias window (client#1009) ─────────────────────────── + # `resourceMonitor: ` is being renamed to `resourceMonitor.enabled: ` + # (D2). This is a bool→object change, so a stored values.yaml or a bare + # `--set resourceMonitor=true` still arrives as a SCALAR; the templates route the + # gate through tracebloc.resourceMonitorEnabled, which must honour BOTH shapes + # for the whole window (remove_by: 2026-12-31). There are two crash directions + # the helper prevents: a blind `ne .Values.resourceMonitor false` gate fails + # "incompatible types for comparison: map and bool" on the new object, and a + # blind `.Values.resourceMonitor.enabled` read fails "can't evaluate field + # enabled in interface {}" on the legacy scalar. The three canonical inputs are + # legacy scalar, new object, and unset; null (a --reuse-values upgrade from a + # chart that predates the key) and `{}` (object with .enabled absent) are the + # two edges of "unset". + - it: renders the DaemonSet for the legacy scalar resourceMonitor=true + template: templates/resource-monitor-daemonset.yaml + set: + resourceMonitor: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: DaemonSet + + - it: renders the DaemonSet for the new object resourceMonitor.enabled=true + template: templates/resource-monitor-daemonset.yaml + set: + resourceMonitor: + enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: DaemonSet + + - it: renders the DaemonSet on the chart default (unset — values.yaml object form) + template: templates/resource-monitor-daemonset.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: DaemonSet + + - it: treats an absent key (null, --reuse-values from a pre-key chart) as enabled + template: templates/resource-monitor-daemonset.yaml + set: + resourceMonitor: null + asserts: + - hasDocuments: + count: 1 + - isKind: + of: DaemonSet + + - it: treats an object with .enabled absent as enabled (default true) + template: templates/resource-monitor-daemonset.yaml + set: + resourceMonitor: {} + asserts: + - hasDocuments: + count: 1 + - isKind: + of: DaemonSet + + - it: renders nothing for the legacy scalar resourceMonitor=false + template: templates/resource-monitor-daemonset.yaml + set: + resourceMonitor: false + asserts: + - hasDocuments: + count: 0 + + - it: renders nothing for the new object resourceMonitor.enabled=false + template: templates/resource-monitor-daemonset.yaml + set: + resourceMonitor: + enabled: false + asserts: + - hasDocuments: + count: 0 + + # The object form CLOSES its keys (values.schema.json additionalProperties: + # false), so a mistyped `enabled` (enable / Enabled / disabled) is REFUSED at + # chart load rather than silently `dig`-defaulting back to true and turning the + # monitor back on — a silent-misconfiguration surface the scalar had no sub-key + # to expose. BARE failedTemplate, per the auto-upgrade schema tests: 0.5.2 + # reports a schema rejection as an errored test with no matchable render error, + # and errorPattern is silently ignored (backend#2606). To pin the specific + # "additional properties 'enable' not allowed" text, assert it from outside the + # plugin with `helm template` + grep. + - it: refuses a mistyped enabled sub-key instead of silently staying enabled + template: templates/resource-monitor-daemonset.yaml + set: + resourceMonitor: + enable: false + asserts: + - failedTemplate: {} diff --git a/client/tests/tracebloc_registry_test.yaml b/client/tests/tracebloc_registry_test.yaml index a42aeef4..a05c08e0 100644 --- a/client/tests/tracebloc_registry_test.yaml +++ b/client/tests/tracebloc_registry_test.yaml @@ -1,11 +1,14 @@ suite: images.traceblocRegistry — the tracebloc-only registry knob (tracebloc.tbRegistry) -# The tracebloc-PUBLISHED control-plane images (jobs-manager, pods-monitor, -# resource-monitor, and the requests-proxy that runs the jobs-manager image) are +# The tracebloc-PUBLISHED images -- the control-plane images (jobs-manager, +# pods-monitor, resource-monitor, and the requests-proxy that runs the +# jobs-manager image) and the training images jobs-manager spawns -- are # published to both docker.io and ghcr.io at the same digests. ONE helper, -# tracebloc.tbRegistry, decides which registry the chart pulls them from, as one -# precedence chain: global.imageRegistry (a mirror re-homes everything) wins, -# then images.traceblocRegistry, then the chart default -- ghcr.io since the -# GHCR migration, with docker.io the documented per-edge rollback. +# tracebloc.tbRegistry, decides which registry the chart pulls them from, and +# which host jobs-manager stamps onto the training images (JOB_IMAGE_HOST, on +# both of its containers), as one precedence chain: global.imageRegistry (a +# mirror re-homes everything) wins, then images.traceblocRegistry, then the +# chart default -- ghcr.io since the GHCR migration, with docker.io the +# documented per-edge rollback. # # Because ghcr.io IS the default, a knob set to ghcr.io exercises nothing: every # override test below sets the knob to docker.io (the rollback) or quay.example @@ -13,9 +16,11 @@ suite: images.traceblocRegistry — the tracebloc-only registry knob (tracebloc. # That is what lets a helper that ignores the knob go red here. # # Pinned here, each of them a machine check for a sentence in values.yaml: -# 1. The knob moves EXACTLY the tracebloc control-plane sites: busybox, the -# ingestor repo and JOB_IMAGE_HOST do not follow it, and mysql-client stays -# on docker.io (frozen, digest-pinned, published nowhere else). +# 1. The knob moves EXACTLY the tracebloc-published sites -- the control-plane +# images AND the training-image host JOB_IMAGE_HOST, together, so one +# rollback moves both: busybox and the ingestor repo do not follow it, and +# mysql-client stays on docker.io (frozen, digest-pinned, published nowhere +# else). # 2. The mirror wins over the knob; an empty knob renders the default, not "". # 3. The image-refresh CronJob resolves digests on the same registry the pods # pull from, and its reconcile verdict (IMAGE_REGISTRY_RESOLVABLE) and the @@ -169,9 +174,9 @@ tests: path: spec.template.spec.initContainers[0].image pattern: "^docker\\.io/library/busybox:" - - it: the knob does NOT move the ingestor repository or JOB_IMAGE_HOST - # The ingestor is already on ghcr.io by repository; the training-image host - # moves in its own step, so it must still read docker.io/ here. + - it: the knob does NOT move the ingestor repository + # The ingestor is named by full repository (images.ingestor.repository), + # already on ghcr.io, and follows only the global mirror. template: templates/jobs-manager-deployment.yaml set: images: @@ -182,11 +187,90 @@ tests: content: name: INGESTOR_IMAGE_REPOSITORY value: "ghcr.io/tracebloc/ingestor" + + - it: images.traceblocRegistry moves JOB_IMAGE_HOST on both containers -- the docker.io rollback + # The training-image host reads the same helper as the control-plane + # images, so ONE knob rolls the control plane and the training pods back + # together. A site that hardcodes the default, or reads only the mirror, + # reddens here. Trailing slash: the runtime concatenates :. + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: docker.io + asserts: - contains: path: spec.template.spec.containers[0].env content: name: JOB_IMAGE_HOST value: "docker.io/" + - contains: + path: spec.template.spec.containers[1].env + content: + name: JOB_IMAGE_HOST + value: "docker.io/" + + - it: images.traceblocRegistry moves JOB_IMAGE_HOST to a registry the script has no token arm for + # The knob is honoured verbatim; whether the refresh script can resolve + # there is a separate verdict (the IMAGE_REGISTRY_RESOLVABLE tests below). + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: quay.example + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: JOB_IMAGE_HOST + value: "quay.example/" + - contains: + path: spec.template.spec.containers[1].env + content: + name: JOB_IMAGE_HOST + value: "quay.example/" + + - it: an EMPTY knob renders the default registry into JOB_IMAGE_HOST, not a bare "/" (the dig-empty trap) + # Same trap as the pods' image above: the key EXISTS in values.yaml, so + # `dig`'s own fallback never applies, and an edge that clears it to "" (or + # a --reuse-values replay from before it existed) must still get a host. A + # site that read the knob without the helper's `| default` chain would + # stamp a bare "/" onto every training image. + template: templates/jobs-manager-deployment.yaml + set: + images: + traceblocRegistry: "" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: JOB_IMAGE_HOST + value: "ghcr.io/" + - contains: + path: spec.template.spec.containers[1].env + content: + name: JOB_IMAGE_HOST + value: "ghcr.io/" + + - it: global.imageRegistry wins over images.traceblocRegistry in JOB_IMAGE_HOST + # A mirror re-homes EVERY image, the training images included: the knob + # must not be able to send training pods past the mirror to a public + # registry. + template: templates/jobs-manager-deployment.yaml + set: + global: + imageRegistry: mirror.corp.example + images: + traceblocRegistry: docker.io + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: JOB_IMAGE_HOST + value: "mirror.corp.example/" + - contains: + path: spec.template.spec.containers[1].env + content: + name: JOB_IMAGE_HOST + value: "mirror.corp.example/" - it: global.imageRegistry wins over images.traceblocRegistry on the jobs-manager template: templates/jobs-manager-deployment.yaml @@ -428,7 +512,7 @@ tests: template: templates/NOTES.txt asserts: - matchRegexRaw: - pattern: "Image registry:.*ghcr\\.io.*\\(tracebloc control-plane images\\)" + pattern: "Image registry:.*ghcr\\.io.*\\(tracebloc control-plane \\+ training images\\)" - it: NOTES follows images.traceblocRegistry -- the docker.io rollback template: templates/NOTES.txt diff --git a/client/tests/training_pod_switches_test.yaml b/client/tests/training_pod_switches_test.yaml index 247dad7e..b73fd8d0 100644 --- a/client/tests/training_pod_switches_test.yaml +++ b/client/tests/training_pod_switches_test.yaml @@ -62,35 +62,61 @@ tests: name: TRACEBLOC_AMP value: "True" - - it: an absent key emits nothing -- unset is OFF on the runtime side, and the pod spec stays byte-identical for every edge that never set it + - it: an absent AMP key emits nothing -- unset is OFF on the runtime side, and the pod spec stays byte-identical for every edge that never set it + # TRACEBLOC_DDP left this case in 1.9.115: it now has a rendered default, so + # "absent" means "the default travels" for it (pinned two cases below). The + # three-state rule -- set / explicitly empty / absent -- is still exercised + # for DDP by the explicit-empty case further down. asserts: - notContains: path: spec.template.spec.containers[0].env content: - name: TRACEBLOC_DDP + name: TRACEBLOC_AMP any: true + + - it: the default values.yaml ships TRACEBLOC_DDP ON (RFC-0067 D7 precondition bundle met, chart 1.9.115) and TRACEBLOC_AMP OFF (absent) + # THE FLIP THIS SUITE USED TO GUARD AGAINST (client#975's "no edge is armed + # by a chart upgrade"). It is deliberate now: D7's preconditions hold -- + # per_rank_split is the engine default (tracebloc-engine#1010), a decline + # falls back to one GPU (client-runtime#553), both on the published :dev + # digests -- so the chart default records the decision (backend#3147). + # DDP ON alone still arms nothing: the size floor below has no default and + # the runtime refuses every expansion while it is unset, which the next + # case pins. AMP keeps its OFF default (backend#2327). + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: TRACEBLOC_DDP + value: "1" - notContains: path: spec.template.spec.containers[0].env content: name: TRACEBLOC_AMP any: true - - it: the default values.yaml ships both switches OFF (absent), so no edge is armed by a chart upgrade - # The keys exist in values.yaml only as documentation comments. If someone - # uncomments a default, every auto-upgrading edge turns DDP (or AMP) on - # overnight with none of RFC-0067 D7's preconditions checked -- this is the - # guard, and it covers BOTH switches (Bugbot, client#975). + - it: the default values.yaml ships NO size floor, so the DDP default arms no edge on its own + # Two-switch arming (client-runtime#513, values.yaml): expansion needs + # TRACEBLOC_DDP truthy AND MULTI_GPU_MIN_PARAMETERS set. With the first now + # defaulted ON, this is the line that keeps a chart upgrade from expanding + # anyone: the floor stays absent until an operator sets one. asserts: - notContains: path: spec.template.spec.containers[0].env content: - name: TRACEBLOC_DDP + name: MULTI_GPU_MIN_PARAMETERS any: true - - notContains: + + - it: an operator's explicit OFF still wins over the ON default -- the kill switch + set: + env: + TRACEBLOC_DDP: "0" + asserts: + - contains: path: spec.template.spec.containers[0].env content: - name: TRACEBLOC_AMP - any: true + name: TRACEBLOC_DDP + value: "0" - it: an EXPLICIT empty string is the third state -- unset -- and must not travel either # "" is how a values file says "I have no opinion"; the runtime reads an diff --git a/client/values.schema.json b/client/values.schema.json index c50422fc..63dd3783 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -333,9 +333,19 @@ "description": "Use ClusterRole (true) or namespace-scoped Role (false). Defaults to true." }, "resourceMonitor": { - "type": "boolean", - "default": true, - "description": "Deploy resource-monitor DaemonSet. Defaults to true." + "type": [ + "boolean", + "object" + ], + "description": "Deploy the resource-monitor DaemonSet. RFC-0076 (D2): prefer the object form `resourceMonitor.enabled: `; the legacy scalar `resourceMonitor: ` (and `--set resourceMonitor=true`) is still accepted through the alias window (remove_by: 2026-12-31) — see tracebloc.resourceMonitorEnabled. Defaults to enabled.", + "properties": { + "enabled": { + "type": "boolean", + "default": true, + "description": "Deploy the resource-monitor DaemonSet. Defaults to true." + } + }, + "additionalProperties": false }, "nodeAgents": { "type": "object", @@ -794,7 +804,7 @@ "traceblocRegistry": { "type": "string", "pattern": "^([A-Za-z0-9.-]+(:[0-9]+)?)?$", - "description": "Registry the tracebloc-published control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) are pulled from. A bare host, no scheme; empty renders the chart default. global.imageRegistry always wins when set. Moves ONLY the tracebloc images (mysql-client stays on docker.io; third-party images keep their own registry keys; JOB_IMAGE_HOST is not routed through it yet). Image-refresh resolves digests on this registry and can only do so anonymously on docker.io and ghcr.io -- any other value makes the reconcile inert and the pods imagePullPolicy=Always, like a mirror." + "description": "Registry the tracebloc-published images are pulled from: the control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) and the training-image host JOB_IMAGE_HOST (the '/' prefix jobs-manager stamps onto every training image it spawns) -- both move together. A bare host, no scheme; empty renders the chart default. global.imageRegistry always wins when set. Moves ONLY the tracebloc images (mysql-client stays on docker.io; third-party images and the ingestor keep their own registry keys). Image-refresh resolves digests on this registry and can only do so anonymously on docker.io and ghcr.io -- any other value makes the reconcile inert and the pods imagePullPolicy=Always, like a mirror." }, "jobsManager": { "type": "object", diff --git a/client/values.yaml b/client/values.yaml index d1e33b0c..f52fb307 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -152,10 +152,16 @@ env: { # construction. # # TRACEBLOC_DDP -- DistributedDataParallel across the GPUs a training pod is - # given. Default OFF until RFC-0067 D7's precondition bundle - # holds (backend#3147 tracks it); flipping it on an edge - # ahead of that is the operator's call, and this is the - # lever that flips it back. + # given. DEFAULT ON since chart 1.9.115: RFC-0067 D7's + # precondition bundle holds (backend#3147 -- the engine + # defaults to per_rank_split, tracebloc-engine#1010; a + # decline falls back to ONE GPU, client-runtime#553; both + # verified on the published :dev digests 2026-09-11). ON + # alone arms nothing: expansion also needs the size floor + # below (MULTI_GPU_MIN_PARAMETERS), which has no default, + # so an edge that never set a floor keeps one GPU per run. + # This key is the kill switch: `--set-string + # env.TRACEBLOC_DDP=0` on an edge is the rollback lever. # TRACEBLOC_AMP -- automatic mixed precision. The ENGINE still chooses # bf16-vs-fp16 from the device in front of it (RFC-0067 D4); # this only says whether it may. Default OFF (backend#2327). @@ -163,7 +169,7 @@ env: { # Reaches the training pods only through a jobs-manager that forwards it # (client-runtime#480); on an older runtime the value lands on the # jobs-manager container and goes no further -- inert, never harmful. - # TRACEBLOC_DDP: "0" + TRACEBLOC_DDP: "1" # TRACEBLOC_AMP: "0" # -- Per-edge OPERATOR switches for what the training pod REPORTS (same # mechanism, same vocabulary). Both default OFF because the destination @@ -370,8 +376,14 @@ clusterScope: true # OC: uses the built-in OpenShift metrics stack. # kubeadm / bare-metal: install metrics-server manually; add # --kubelet-insecure-tls on clusters with self-signed kubelet certs. -# Set to false on clusters where metrics-server cannot be installed. -resourceMonitor: true +# Set enabled to false on clusters where metrics-server cannot be installed. +# +# RFC-0076 (D2): this was the scalar `resourceMonitor: ` and is now +# `resourceMonitor.enabled: `. The chart still honours the legacy scalar +# (and `--set resourceMonitor=true|false`) through the alias window +# (remove_by: 2026-12-31) — see tracebloc.resourceMonitorEnabled in _helpers.tpl. +resourceMonitor: + enabled: true # -- Node-level agents (currently: tracebloc-resource-monitor DaemonSet). # The resource-monitor needs hostPath /proc and /sys to read node metrics, @@ -920,27 +932,32 @@ telemetryCollector: # Setting `digest` here is still meaningful: it is an explicit operator pin # that also opts the image OUT of auto-refresh. images: - # Registry the tracebloc-PUBLISHED control-plane images are pulled from: - # tracebloc/jobs-manager, tracebloc/pods-monitor, tracebloc/resource-monitor - # and the requests-proxy (which runs the jobs-manager image). A bare host, no + # Registry the tracebloc-PUBLISHED images are pulled from: the control-plane + # images (tracebloc/jobs-manager, tracebloc/pods-monitor, + # tracebloc/resource-monitor and the requests-proxy, which runs the + # jobs-manager image) AND the training-image host -- JOB_IMAGE_HOST, the + # "/" prefix jobs-manager stamps onto every training image it + # spawns (tracebloc/client--:). A bare host, no # scheme. Resolved by the tracebloc.tbRegistry helper as ONE precedence chain # -- `global.imageRegistry` (a mirror re-homes everything) always wins, then - # this, then the chart default -- so the pods, the image-refresh CronJob and - # NOTES.txt cannot disagree about where those images live. + # this, then the chart default -- so the pods, the training pods they spawn, + # the image-refresh CronJob and NOTES.txt cannot disagree about where those + # images live. # - # This knob moves ONLY the tracebloc images: busybox, squid, alpine/*, the - # device plugins and the ingestor keep their own registries, and + # This knob moves ONLY the tracebloc images, and it moves the control plane + # and the training-image host TOGETHER: busybox, squid, alpine/*, the device + # plugins and the ingestor keep their own registries, and # tracebloc/mysql-client stays on docker.io (frozen, digest-pinned, published - # nowhere else -- see images.mysqlClient). The training-image host - # (JOB_IMAGE_HOST) is not routed through it yet; that moves in its own step. + # nowhere else -- see images.mysqlClient). # - # ghcr.io is where the control-plane images are published since the GHCR - # migration, and it is the chart default. They are still dual-published to - # docker.io at the same digests for now, so "docker.io" remains a valid value - # and is the per-edge rollback (`--set images.traceblocRegistry=docker.io`; - # user-supplied, so it survives the fleet auto-upgrade until cleared). Digest - # pins (images.*.digest) are registry-agnostic: the same digest exists on - # both. No new egress is needed: ghcr.io (+ pkg-containers.githubusercontent.com + # ghcr.io is where the control-plane and training images are published since + # the GHCR migration, and it is the chart default. They are still + # dual-published to docker.io at the same digests for now, so "docker.io" + # remains a valid value and is the per-edge rollback + # (`--set images.traceblocRegistry=docker.io`; user-supplied, so it survives + # the fleet auto-upgrade until cleared). Digest pins (images.*.digest, and + # the training-image digest pins below) are registry-agnostic: the same digest + # exists on both. No new egress is needed: ghcr.io (+ pkg-containers.githubusercontent.com # for layer redirects) is already required for the ingestor image and probed # by the installer preflight. Image-refresh follows it: the CronJob resolves # digests on this registry (it can do so anonymously on ghcr.io and docker.io; diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 7b43ad6d..dee11d1c 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -33,8 +33,8 @@ The standalone installer runs a **preflight** check that verifies this connectiv | Host | Why | |---|---| -| `registry-1.docker.io` (Docker Hub) | k3s, mysql-client, busybox + the tracebloc training images; the control-plane images too, only if you roll them back with `images.traceblocRegistry=docker.io` | -| `ghcr.io` (+ `pkg-containers.githubusercontent.com`, where GHCR redirects layer downloads) | the tracebloc control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy — the default since the GHCR migration) + the ingestor image + k3d node images | +| `registry-1.docker.io` (Docker Hub) | k3s, mysql-client, busybox; the tracebloc control-plane and training images too, only if you roll them back with `images.traceblocRegistry=docker.io` | +| `ghcr.io` (+ `pkg-containers.githubusercontent.com`, where GHCR redirects layer downloads) | the tracebloc control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) and the training images jobs-manager spawns — both the default since the GHCR migration — + the ingestor image + k3d node images | | `api.tracebloc.io` (`dev-api`/`stg-api` for non-prod) | client credential check + the running client's platform connection | | `tracebloc.github.io` | the tracebloc Helm chart repository | @@ -55,7 +55,7 @@ Some sites hard-block Docker Hub / GHCR outright — the images aren't reachable The chart follows the **`global.imageRegistry`** convention: set it once and **every** image the chart pulls — the tracebloc services, the spawned ingestor, the training-job images, and the `alpine/*`, `ubuntu/squid`, `busybox`, `curl` helper images — is re-homed onto your registry. No per-image overrides. -**Moving only the tracebloc images.** `global.imageRegistry` re-homes *everything*. The tracebloc-published control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) pull from `ghcr.io` by default and are also published to `docker.io` at the same digests. To point just those images at a different registry, set `images.traceblocRegistry` (a bare host) and leave `global.imageRegistry` unset; `global.imageRegistry` always wins when both are set. Rolling the control plane back to Docker Hub is one flag — `--set images.traceblocRegistry=docker.io` — and, being user-supplied, it persists across the fleet auto-upgrade until you clear it. The image-refresh CronJob follows the same value and can resolve digests anonymously on `ghcr.io` and `docker.io` only; any other registry makes the reconcile inert, exactly as a mirror does. `tracebloc/mysql-client` is not moved by this knob (it is frozen, digest-pinned and published only to Docker Hub), and neither is the training-image host yet. +**Moving only the tracebloc images.** `global.imageRegistry` re-homes *everything*. The tracebloc-published images — the control-plane images (jobs-manager, pods-monitor, resource-monitor, requests-proxy) and the training images jobs-manager spawns (through `JOB_IMAGE_HOST`, the registry prefix it stamps onto every training image) — pull from `ghcr.io` by default and are also published to `docker.io` at the same digests. To point just those images at a different registry, set `images.traceblocRegistry` (a bare host) and leave `global.imageRegistry` unset; `global.imageRegistry` always wins when both are set. Rolling them back to Docker Hub is one flag — `--set images.traceblocRegistry=docker.io` — which moves the control plane and the training-image host together, and, being user-supplied, persists across the fleet auto-upgrade until you clear it. The image-refresh CronJob follows the same value and can resolve digests anonymously on `ghcr.io` and `docker.io` only; any other registry makes the reconcile inert, exactly as a mirror does. `tracebloc/mysql-client` is not moved by this knob (it is frozen, digest-pinned and published only to Docker Hub), and neither is the ingestor (named by full repository, already on `ghcr.io`). **1. A private/mirror registry your site *can* reach.** diff --git a/scripts/list-images.sh b/scripts/list-images.sh index e729ea86..018671d4 100755 --- a/scripts/list-images.sh +++ b/scripts/list-images.sh @@ -40,7 +40,11 @@ # chart images <- `helm template` on YOUR values, so conditionals resolve # mirror prefix <- the rendered JOB_IMAGE_HOST # ingestor <- the rendered INGESTOR_IMAGE_REPOSITORY + TAG/DIGEST -# training tasks <- the registry's own `client-*` repository list +# training tasks <- Docker Hub's `client-*` repository list for the namespace +# (the NAMES; the host prefix is the rendered JOB_IMAGE_HOST, +# ghcr.io by chart default since the GHCR migration -- the +# images are dual-published under the same names, and GHCR +# has no anonymous repository-list endpoint to derive from) # # FAILS CLOSED (rule 3). A failed render, an unreadable registry, or a task # enumeration of zero is an ERROR, not an empty section. "We could not tell" @@ -262,6 +266,11 @@ else # refusal could only be reasoned about, never exercised -- and an unexercised # guard is indistinguishable from one that does not work (rule 5). Note that # common.sh prepends the system PATH, so stubbing `curl` is not an option. + # Docker Hub is queried for the repository NAMES whatever host the render + # stamps (ghcr.io by chart default): GHCR exposes no anonymous repository-list + # endpoint, and the training images are dual-published under the same names, + # so the names agree. A site that blocks Docker Hub but reaches the rendered + # host still enumerates via TRACEBLOC_REGISTRY_URL or TRACEBLOC_TASK_REPOS. page="${TRACEBLOC_REGISTRY_URL:-https://hub.docker.com/v2/repositories/${REGISTRY_NAMESPACE}/?page_size=100}" while [ -n "$page" ]; do # curl's OWN stderr is kept and shown. Bugbot, medium, and it matters here diff --git a/scripts/publish-guard.sh b/scripts/publish-guard.sh new file mode 100755 index 00000000..ef1aae27 --- /dev/null +++ b/scripts/publish-guard.sh @@ -0,0 +1,471 @@ +#!/usr/bin/env bash +# ============================================================================= +# publish-guard.sh — stage the public deliverable of this repo and refuse +# anything else. +# +# The public mirror of this repo carries a DELIVERABLE, not the source tree. +# This script builds that deliverable in a clean directory from an explicit +# allowlist, then runs four guards over what it staged. Nothing outside the +# allowlist can be staged (exclusion by construction), and four independent +# scans stand between the staged tree and the push: +# +# 1. [allowlist] .publish-include names what MAY ship. Tracked files +# only (`git ls-files`), matched by glob; a `!glob` +# line takes files back out again. +# 2. [forbidden-paths] .publish-forbidden `[paths]`: names that must never +# be in the staged tree even if allowlisted by +# mistake (gitignore-style matching). +# 3. [forbidden-strings] .publish-forbidden needles (extended regex, +# case-insensitive) scanned over every staged text +# file, in two tiers: +# [strings-refuse] a hit refuses the publish +# (mailboxes, cloud account +# identifiers; the private +# needles from --extra-forbidden +# join this tier and are named +# `private needle #N` in every +# line this script prints or +# writes — the pattern itself +# never reaches a log). +# [strings-report] hits are COUNTED and printed — +# per-needle totals and the ten +# most-hit files — but refuse +# only under --strict. Internal +# ticket references and +# non-production hostnames live +# here until the decision to +# strip them is taken; --strict +# arms that decision. +# `[allow]` entries are exact tokens spared before a +# needle is re-tested (a public support mailbox +# beside a rule that bans every other mailbox): a +# token is stripped only as a whole word, case- +# insensitively like the scan — `devsupport@…` is +# not spared by `support@…`. +# A needle may sit in one tier only, [strings-refuse] +# may not be empty, and a section header the guard +# does not know is refused: each of those is a list +# the guard cannot vouch for (exit 2). +# 4. [gitleaks] gitleaks detect --no-git --redact over everything +# staged, default rules. +# +# FAIL CLOSED. Exit 0 only when every guard RAN and every guard PASSED. +# exit 1 a guard REFUSED — the message names the guard and the rule. +# exit 2 COULD NOT TELL — unreadable or empty allowlist / forbidden list, +# a malformed forbidden list (unknown section, a needle in both +# tiers, an empty refuse tier), zero tracked files, an allowlist +# that matched nothing, a symlink in the allowlisted set, a missing +# or erroring scanner, a guard that did not run, a non-empty --out. +# "Cannot tell" is never clean. +# Every guard runs even after an earlier one has refused, so one run reports +# everything; the exit status is the worst verdict seen. +# +# Usage: +# publish-guard.sh --source DIR --out DIR +# [--include FILE] default DIR/.publish-include +# [--forbidden FILE] default DIR/.publish-forbidden +# [--extra-forbidden FILE] more refuse-tier needles (repeat +# as needed); must be readable +# and non-empty +# [--assets DIR] release assets to publish next +# to the tree; guards 2–4 scan +# them too +# [--strict] a [strings-report] hit refuses +# instead of being counted +# +# Output: one line per guard, the staged file list, a final verdict. +# OUT/tree holds the staged tree, OUT/assets the assets; OUT must not exist or +# must be empty (a stale staging directory could carry a file no guard read). +# A full findings report is written to OUT/publish-guard-report.txt. +# +# Environment (tests only): PUBLISH_GUARD_GITLEAKS names the gitleaks binary. +# ============================================================================= +set -uo pipefail + +SOURCE=""; OUT=""; INCLUDE=""; FORBIDDEN=""; ASSETS=""; STRICT=0 +EXTRA_FORBIDDEN=() +while [ "$#" -gt 0 ]; do + case "$1" in + --source) SOURCE="${2:-}"; shift 2 ;; + --out) OUT="${2:-}"; shift 2 ;; + --include) INCLUDE="${2:-}"; shift 2 ;; + --forbidden) FORBIDDEN="${2:-}"; shift 2 ;; + --extra-forbidden) EXTRA_FORBIDDEN+=("${2:-}"); shift 2 ;; + --assets) ASSETS="${2:-}"; shift 2 ;; + --strict) STRICT=1; shift ;; + -h|--help) sed -n '2,/^# ====/p' "$0" | sed 's/^# \{0,2\}//'; exit 0 ;; + *) echo "publish-guard: unknown argument '$1'" >&2; exit 2 ;; + esac +done + +# ---- verdict bookkeeping ----------------------------------------------------- +# WORST is the exit status: 0 clean, 1 refused, 2 could not tell. RAN counts the +# guards that reached a verdict; the final check refuses to report green unless +# all four did — a refactor that drops a stage must not look like a clean run. +WORST=0 +RAN=0 +GUARDS_EXPECTED=4 +worsen() { [ "$1" -gt "$WORST" ] && WORST="$1"; return 0; } +# The workflow-command prefix goes to STDOUT: Actions reads ::error:: from +# stdout only. Plain lines are the guard's narration. +refuse() { echo "::error::publish-guard: [$1] REFUSED — $2"; worsen 1; } +cant_tell(){ echo "::error::publish-guard: [$1] COULD NOT TELL — $2 (never reported as clean)"; worsen 2; } +note() { echo "publish-guard: [$1] $2"; } +# A guard error before any guard can run: nothing to stage, nothing to report. +die2() { echo "::error::publish-guard: COULD NOT TELL — $1 (never reported as clean)"; exit 2; } + +[ -n "$SOURCE" ] || die2 "--source is required" +[ -n "$OUT" ] || die2 "--out is required" +[ -d "$SOURCE" ] || die2 "--source '$SOURCE' is not a directory" +SOURCE="$(cd "$SOURCE" && pwd)" +[ -n "$INCLUDE" ] || INCLUDE="$SOURCE/.publish-include" +[ -n "$FORBIDDEN" ] || FORBIDDEN="$SOURCE/.publish-forbidden" +if [ -e "$OUT" ]; then + [ -d "$OUT" ] || die2 "--out '$OUT' exists and is not a directory" + [ -z "$(ls -A "$OUT")" ] || die2 "--out '$OUT' is not empty; a stale staging directory could carry a file no guard read" +fi +mkdir -p "$OUT/tree" || die2 "cannot create '$OUT/tree'" +OUT="$(cd "$OUT" && pwd)" +TREE="$OUT/tree" + +# Scratch, armed only once it exists (a failed mktemp must not make the trap +# expand to `rm -rf /*`). +TMP="$(mktemp -d "${TMPDIR:-/tmp}/publish-guard.XXXXXX")" && [ -d "$TMP" ] || die2 "could not create a scratch directory" +trap 'rm -rf "$TMP"' EXIT +REPORT="$TMP/report.txt" +: >"$REPORT" + +# ---- list files: strip comments and blanks, keep order ------------------------ +# A section header is a line that is nothing but one bracketed token. The match +# is deliberately loose (`[strings refuse]`, `[Strings-Refuse]` are headers too) +# so a misspelt header is refused by name below instead of being read as a +# needle of the section before it. +SECTION_RE='^[[][^]]*[]][[:space:]]*$' # bracket expressions, so no awk escape processing applies +# read_list FILE SECTION — print the entries of SECTION ([paths] / +# [strings-refuse] / [strings-report] / [allow]) from a sectioned list file; +# SECTION "" prints every entry of a file that has no section headers (the +# allowlist, an --extra-forbidden list). +read_list() { + awk -v want="$2" -v hdr="$SECTION_RE" ' + /^[[:space:]]*(#|$)/ { next } + $0 ~ hdr { sec = $0; sub(/^\[/, "", sec); sub(/\].*$/, "", sec); next } + { line = $0; sub(/[[:space:]]+$/, "", line) + if (want == "" || sec == want) print line } + ' "$1" +} + +# The sections the forbidden list may declare. Both guards that read the list +# check every header against this set: a header the guard does not read would +# silently orphan the rules under it. +FORBIDDEN_SECTIONS="paths strings-refuse strings-report allow" +# forbidden_sections_ok GUARD — could-not-tell (and return 1) on the first +# header of $FORBIDDEN that is not one of FORBIDDEN_SECTIONS. +forbidden_sections_ok() { + local sec + while IFS= read -r sec; do + case " $FORBIDDEN_SECTIONS " in + *" $sec "*) ;; + *) cant_tell "$1" "'$FORBIDDEN' has an unknown section [$sec] — the guard reads only [${FORBIDDEN_SECTIONS// /] [}]"; return 1 ;; + esac + done < <(awk -v hdr="$SECTION_RE" '$0 ~ hdr { sec = $0; sub(/^\[/, "", sec); sub(/\].*$/, "", sec); print sec }' "$FORBIDDEN") + return 0 +} + +# glob_to_ere GLOB — an anchored extended regex for a path glob: `*` and `?` do +# not cross `/`, `**` does (`**/` also matches zero directories). Every other +# regex metacharacter in the glob is escaped, so a `.` in `*.go` is a dot. +glob_to_ere() { + local g="$1" out="" i c n + n=${#g} + for ((i = 0; i < n; i++)); do + c="${g:i:1}" + case "$c" in + '*') + if [ "${g:i+1:1}" = '*' ]; then + if [ "${g:i+2:1}" = '/' ]; then out+='(.*/)?'; i=$((i + 2)); else out+='.*'; i=$((i + 1)); fi + else + out+='[^/]*' + fi ;; + '?') out+='[^/]' ;; + '['|']'|'.'|'^'|'$'|'+'|'('|')'|'{'|'}'|'|'|'\') out+="\\$c" ;; + *) out+="$c" ;; + esac + done + printf '^%s$' "$out" +} + +# ---- guard 1: allowlist --------------------------------------------------------- +guard_allowlist() { + local g="allowlist" n_inc=0 n_exc=0 line re + local -a inc_re=() exc_re=() + if [ ! -r "$INCLUDE" ]; then cant_tell "$g" "allowlist '$INCLUDE' is missing or unreadable"; RAN=$((RAN + 1)); return; fi + while IFS= read -r line; do + case "$line" in + '!'*) exc_re+=("$(glob_to_ere "${line#!}")"); n_exc=$((n_exc + 1)) ;; + *) inc_re+=("$(glob_to_ere "$line")"); n_inc=$((n_inc + 1)) ;; + esac + done < <(read_list "$INCLUDE" "") + if [ "$n_inc" -eq 0 ]; then cant_tell "$g" "allowlist '$INCLUDE' lists no include entries — nothing may ship, so nothing can be vouched for"; RAN=$((RAN + 1)); return; fi + + # Tracked files only: an untracked file in the checkout is never a deliverable. + # A path containing a newline is unrepresentable in the line-oriented list + # below, so the NUL-separated count must equal the line count. + local listed nul_count line_count + listed="$TMP/tracked.txt" + if ! git -C "$SOURCE" -c core.quotePath=false ls-files >"$listed" 2>"$TMP/git.err"; then + cant_tell "$g" "git ls-files failed in '$SOURCE': $(tr '\n' ' ' <"$TMP/git.err")"; RAN=$((RAN + 1)); return + fi + nul_count="$(git -C "$SOURCE" ls-files -z | tr -cd '\0' | wc -c | tr -d ' ')" + line_count="$(wc -l <"$listed" | tr -d ' ')" + if [ "$line_count" -eq 0 ]; then cant_tell "$g" "'$SOURCE' has zero tracked files"; RAN=$((RAN + 1)); return; fi + if [ "$nul_count" != "$line_count" ]; then cant_tell "$g" "a tracked path contains a newline ($nul_count entries, $line_count lines) — cannot match it safely"; RAN=$((RAN + 1)); return; fi + + local staged=0 f matched + : >"$TMP/staged.txt" + while IFS= read -r f; do + matched=0 + for re in "${inc_re[@]}"; do [[ "$f" =~ $re ]] && { matched=1; break; }; done + [ "$matched" -eq 1 ] || continue + for re in "${exc_re[@]+"${exc_re[@]}"}"; do [[ "$f" =~ $re ]] && { matched=0; break; }; done + [ "$matched" -eq 1 ] || continue + if [ -L "$SOURCE/$f" ]; then cant_tell "$g" "'$f' is a symlink — a link can point outside the tree, so it is not staged"; RAN=$((RAN + 1)); return; fi + [ -f "$SOURCE/$f" ] || { cant_tell "$g" "tracked file '$f' is missing from the checkout"; RAN=$((RAN + 1)); return; } + mkdir -p "$TREE/$(dirname "$f")" || { cant_tell "$g" "cannot create '$TREE/$(dirname "$f")'"; RAN=$((RAN + 1)); return; } + cp -p "$SOURCE/$f" "$TREE/$f" || { cant_tell "$g" "cannot copy '$f'"; RAN=$((RAN + 1)); return; } + printf '%s\n' "$f" >>"$TMP/staged.txt" + staged=$((staged + 1)) + done <"$listed" + if [ "$staged" -eq 0 ]; then cant_tell "$g" "the allowlist matched none of the $line_count tracked files — a mirror with nothing in it is not a deliverable"; RAN=$((RAN + 1)); return; fi + note "$g" "staged $staged of $line_count tracked file(s) ($n_inc include, $n_exc exclude pattern(s)):" + sort "$TMP/staged.txt" | sed 's/^/ /' + RAN=$((RAN + 1)) +} + +# ---- assets ---------------------------------------------------------------------- +stage_assets() { + [ -n "$ASSETS" ] || return 0 + [ -d "$ASSETS" ] || die2 "--assets '$ASSETS' is not a directory" + local n + n="$(find "$ASSETS" -mindepth 1 -maxdepth 1 -type f | wc -l | tr -d ' ')" + [ "$n" -gt 0 ] || die2 "--assets '$ASSETS' holds no files — a release with no assets is not what a customer downloads" + [ "$(find "$ASSETS" -mindepth 1 -maxdepth 1 ! -type f | wc -l | tr -d ' ')" -eq 0 ] || die2 "--assets '$ASSETS' holds something other than plain files (a directory or a symlink)" + mkdir -p "$OUT/assets" && cp -p "$ASSETS"/* "$OUT/assets"/ || die2 "cannot copy assets from '$ASSETS'" + note "assets" "staged $n release asset(s):" + find "$OUT/assets" -mindepth 1 -maxdepth 1 -type f | sed "s|^$OUT/assets/||" | sort | sed 's/^/ /' +} + +# staged_paths — every staged path as `:` (area = tree or +# assets), one per line. +staged_paths() { + ( cd "$OUT" && find tree assets -type f 2>/dev/null ) | sed -E 's#^(tree|assets)/#\1:#' | sort +} + +# ---- guard 2: forbidden paths ---------------------------------------------------- +# gitignore-style: a pattern with a `/` inside it is anchored to the staged root +# (`scripts/tests/` matches only that directory); one without matches ANY path +# component (`tests/` matches `client/tests/x`, `*.go` matches `a/b/c.go`); a +# trailing `/` means "as a directory" (`tests/` does not match a file named +# tests). The staged area prefix (tree/, assets/) is not part of the path. +path_pattern_hits() { # $1 = pattern, reads staged paths on stdin, prints hits + local pat="$1" dir_only=0 anchored=0 re + case "$pat" in */) dir_only=1; pat="${pat%/}" ;; esac + pat="${pat#/}" + case "$pat" in */*) anchored=1 ;; esac + re="$(glob_to_ere "$pat")" + local entry p comp + local -a comps + while IFS= read -r entry; do + p="${entry#*:}" + if [ "$anchored" -eq 1 ]; then + if [ "$dir_only" -eq 0 ] && [[ "$p" =~ $re ]]; then printf '%s\n' "$entry"; continue; fi + [[ "$p/" == "${pat}/"* ]] && printf '%s\n' "$entry" + continue + fi + IFS='/' read -r -a comps <<<"$p" + local i last=$(( ${#comps[@]} - 1 )) + for i in "${!comps[@]}"; do + comp="${comps[$i]}" + [ "$dir_only" -eq 1 ] && [ "$i" -eq "$last" ] && continue + if [[ "$comp" =~ $re ]]; then printf '%s\n' "$entry"; break; fi + done + done +} + +guard_forbidden_paths() { + local g="forbidden-paths" n=0 pat hits total=0 + if [ ! -r "$FORBIDDEN" ]; then cant_tell "$g" "forbidden list '$FORBIDDEN' is missing or unreadable"; RAN=$((RAN + 1)); return; fi + forbidden_sections_ok "$g" || { RAN=$((RAN + 1)); return; } + read_list "$FORBIDDEN" paths >"$TMP/paths.txt" + n="$(grep -c . "$TMP/paths.txt" || true)" + if [ "$n" -eq 0 ]; then cant_tell "$g" "'$FORBIDDEN' has no [paths] entries — a scan with no rules proves nothing"; RAN=$((RAN + 1)); return; fi + staged_paths >"$TMP/all.txt" + while IFS= read -r pat; do + hits="$(path_pattern_hits "$pat" <"$TMP/all.txt")" + [ -n "$hits" ] || continue + total=$((total + $(printf '%s\n' "$hits" | grep -c .))) + refuse "$g" "forbidden path pattern '$pat' matched:" + printf '%s\n' "$hits" | sed 's/^/ /' | tee -a "$REPORT" + done <"$TMP/paths.txt" + [ "$total" -gt 0 ] || note "$g" "clean ($n pattern(s) against $(grep -c . "$TMP/all.txt") staged path(s))" + RAN=$((RAN + 1)) +} + +# ---- guard 3: forbidden strings -------------------------------------------------- +# Two tiers over the same scan. A [strings-refuse] needle (or any needle from +# --extra-forbidden) refuses on a hit. A [strings-report] needle is counted and +# printed — per-needle totals and the ten most-hit files — and refuses only +# under --strict: the tier can be measured on the real deliverable before the +# decision to strip it is taken, and one flag arms that decision. +# Text files only (`grep -I`): a binary asset is opaque to a string scan; its +# integrity is the release's own SHA256SUMS + signature. The count of binaries +# skipped is printed so "scanned everything" and "skipped half" read differently. +guard_forbidden_strings() { + local g="forbidden-strings" needle rc hits n_refuse n_report n_allow=0 extra dup + if [ ! -r "$FORBIDDEN" ]; then cant_tell "$g" "forbidden list '$FORBIDDEN' is missing or unreadable"; RAN=$((RAN + 1)); return; fi + forbidden_sections_ok "$g" || { RAN=$((RAN + 1)); return; } + read_list "$FORBIDDEN" strings-refuse >"$TMP/needles-refuse.txt" + read_list "$FORBIDDEN" strings-report >"$TMP/needles-report.txt" + read_list "$FORBIDDEN" allow >"$TMP/allow.txt" + # One tier per needle: the same text in both would be refused by one loop and + # counted by the other, and whichever the reader saw first would be the rule. + dup="$(comm -12 <(sort -u "$TMP/needles-refuse.txt") <(sort -u "$TMP/needles-report.txt") | grep . | head -1)" + if [ -n "$dup" ]; then cant_tell "$g" "'$FORBIDDEN' lists needle '$dup' in both [strings-refuse] and [strings-report] — a needle has one tier"; RAN=$((RAN + 1)); return; fi + # The committed refuse tier is judged BEFORE the private needles join it: a + # list whose only hard rules arrive from a secret is misconfigured. + n_refuse="$(grep -c . "$TMP/needles-refuse.txt" || true)" + if [ "$n_refuse" -eq 0 ]; then cant_tell "$g" "'$FORBIDDEN' has no [strings-refuse] entries — a guard with nothing to refuse is misconfigured"; RAN=$((RAN + 1)); return; fi + for extra in "${EXTRA_FORBIDDEN[@]+"${EXTRA_FORBIDDEN[@]}"}"; do + if [ ! -r "$extra" ]; then cant_tell "$g" "extra forbidden list '$extra' is missing or unreadable"; RAN=$((RAN + 1)); return; fi + if [ "$(read_list "$extra" "" | grep -c .)" -eq 0 ]; then cant_tell "$g" "extra forbidden list '$extra' is empty — the private needles were not supplied, so this scan cannot vouch for them"; RAN=$((RAN + 1)); return; fi + read_list "$extra" "" >>"$TMP/needles-private.txt" + done + : >>"$TMP/needles-private.txt" + n_refuse="$(( $(grep -c . "$TMP/needles-refuse.txt" || true) + $(grep -c . "$TMP/needles-private.txt" || true) ))" + n_report="$(grep -c . "$TMP/needles-report.txt" || true)" + n_allow="$(grep -c . "$TMP/allow.txt" || true)" + + # Census of what the scan can and cannot see. + local n_text=0 n_bin=0 f + while IFS= read -r f; do + if [ "$(tr -d -c '\000' <"$f" | wc -c | tr -d ' ')" -gt 0 ]; then n_bin=$((n_bin + 1)); else n_text=$((n_text + 1)); fi + done < <(find "$OUT/tree" "$OUT/assets" -type f 2>/dev/null) + if [ "$n_text" -eq 0 ]; then cant_tell "$g" "no text file staged — nothing this scan can read"; RAN=$((RAN + 1)); return; fi + + local -a scan_dirs=("$OUT/tree") + [ -d "$OUT/assets" ] && scan_dirs+=("$OUT/assets") + local allow_expr + allow_expr="$(paste -sd'|' "$TMP/allow.txt")" + # needle_hits NEEDLE SHOWN — write the `area/file:line` locations NEEDLE + # matches, after [allow] stripping, to $TMP/hits.txt. Returns 2 when grep + # itself failed, with the reason in $GREP_ERR; the caller reports + # could-not-tell. SHOWN is how the needle is named in any message: the + # pattern for a committed needle, `private needle #N` for one that came from + # --extra-forbidden — those are the identifiers kept out of the public list, + # and this log is public too. + needle_hits() { + local needle="$1" shown="$2" rc + # Hits go through a FILE, never `producer | grep -q`: a closed pipe would + # turn a real finding into "clean" via SIGPIPE. + grep -rIinE -e "$needle" "${scan_dirs[@]}" >"$TMP/hits.txt" 2>"$TMP/grep.err"; rc=$? + if [ "$rc" -ge 2 ]; then GREP_ERR="grep exited $rc on $shown: $(tr '\n' ' ' <"$TMP/grep.err")"; return 2; fi + if [ "$rc" -ne 0 ]; then : >"$TMP/hits.txt"; return 0; fi + # [allow] tokens are removed from each hit line and the needle re-tested, so + # a line is spared only when the allowed token was the whole reason it hit. + # A token is removed only as a WHOLE word — not when it is the tail of a + # longer mailbox (`devsupport@…`) or the head of a longer domain — and + # case-insensitively, as the scan itself matches. A sentence-ending `.` + # after the token is still a boundary. + # Split each hit into its location and its text; only the TEXT is re-tested, + # so the `file:line` prefix can never be what matches. + awk -F: '{ print $1 ":" $2 }' "$TMP/hits.txt" >"$TMP/locs.txt" + sed -E 's/^[^:]*:[^:]*://' "$TMP/hits.txt" >"$TMP/texts.txt" + if [ "$n_allow" -gt 0 ]; then + sed -E "s#(^|[^[:alnum:]._%+-])($allow_expr)($|[^[:alnum:]._%+-]|\.([^[:alnum:]]|$))#\1 \3#gI" "$TMP/texts.txt" >"$TMP/texts2.txt" && mv "$TMP/texts2.txt" "$TMP/texts.txt" + fi + grep -inE -e "$needle" "$TMP/texts.txt" | cut -d: -f1 >"$TMP/kept.txt"; rc=${PIPESTATUS[0]} + if [ "$rc" -ge 2 ]; then GREP_ERR="re-test after [allow] stripping exited $rc on $shown"; return 2; fi + awk 'NR == FNR { keep[$1] = 1; next } (FNR in keep)' "$TMP/kept.txt" "$TMP/locs.txt" | sed "s|^$OUT/||" >"$TMP/hits.txt" + return 0 + } + + # Three passes: the committed refuse tier, the private needles (refuse tier, + # named by number only), the report tier. + local tier label shown k n_refused=0 n_reported=0 + : >"$TMP/report-locs.txt" + for tier in refuse private report; do + k=0 + while IFS= read -r needle; do + k=$((k + 1)) + if [ "$tier" = private ]; then shown="private needle #$k"; else shown="needle '$needle'"; fi + needle_hits "$needle" "$shown" || { cant_tell "$g" "$GREP_ERR"; RAN=$((RAN + 1)); return; } + hits="$(grep -c . "$TMP/hits.txt" || true)" + [ "$hits" -gt 0 ] || continue + if [ "$tier" != report ]; then + { echo "[strings-refuse] $shown:"; cat "$TMP/hits.txt"; } >>"$REPORT" + n_refused=$((n_refused + hits)); label="strings-refuse" + else + { echo "[strings-report] $shown:"; cat "$TMP/hits.txt"; } >>"$REPORT" + n_reported=$((n_reported + hits)); cat "$TMP/hits.txt" >>"$TMP/report-locs.txt" + if [ "$STRICT" -eq 1 ]; then + label="strings-report (strict)" + else + note "$g" "[strings-report] $shown found in $hits staged line(s) — counted, not refused (--strict refuses)" + continue + fi + fi + refuse "$g" "[$label] $shown found in $hits staged line(s):" + head -20 "$TMP/hits.txt" | sed 's/^/ /' + [ "$hits" -le 20 ] || echo " … and $((hits - 20)) more (full list in publish-guard-report.txt)" + done <"$TMP/needles-$tier.txt" + done + if [ "$n_reported" -gt 0 ]; then + # Where the report tier lands, so the clean-up (or the decision not to) has + # a map: count per file, ten most-hit first. + sed 's/:[0-9]*$//' "$TMP/report-locs.txt" | sort | uniq -c | sort -rn >"$TMP/report-files.txt" + note "$g" "[strings-report] $n_reported hit(s) in $(grep -c . "$TMP/report-files.txt") file(s); most-hit files:" + head -10 "$TMP/report-files.txt" | awk '{ n = $1; sub(/^ *[0-9]+ /, ""); printf " %6d %s\n", n, $0 }' + fi + local tally="$n_refuse refuse + $n_report report needle(s), $n_allow allow token(s); $n_text text file(s) scanned, $n_bin binary file(s) opaque to this scan" + if [ "$n_refused" -eq 0 ] && [ "$n_reported" -eq 0 ]; then + note "$g" "clean ($tally)" + elif [ "$STRICT" -eq 1 ]; then + note "$g" "$n_refused refuse-tier hit(s), $n_reported report-tier hit(s) refused under --strict ($tally)" + else + note "$g" "$n_refused refuse-tier hit(s), $n_reported report-tier hit(s) counted ($tally)" + fi + RAN=$((RAN + 1)) +} + +# ---- guard 4: gitleaks ----------------------------------------------------------- +guard_gitleaks() { + local g="gitleaks" bin="${PUBLISH_GUARD_GITLEAKS:-gitleaks}" rc + if ! command -v "$bin" >/dev/null 2>&1; then cant_tell "$g" "scanner '$bin' is not on PATH — a scan that did not run is not a clean scan"; RAN=$((RAN + 1)); return; fi + # Leaks exit with a code no crash uses (default 1 is also "something broke"), + # so a scanner failure cannot be misread as either verdict. + "$bin" detect --no-git --redact --no-banner --exit-code 9 --source "$OUT" >"$TMP/gitleaks.out" 2>&1; rc=$? + case "$rc" in + 0) note "$g" "clean ($("$bin" version 2>/dev/null | head -1 || echo 'version unknown'), default rules, $(staged_paths | grep -c .) staged file(s))" ;; + 9) refuse "$g" "secrets detected in the staged tree:"; grep -vE '^[0-9]+:[0-9]+[AP]M' "$TMP/gitleaks.out" | sed 's/^/ /' | tee -a "$REPORT" ;; + *) cant_tell "$g" "scanner exited $rc: $(tail -3 "$TMP/gitleaks.out" | tr '\n' ' ')" ;; + esac + RAN=$((RAN + 1)) +} + +# ---- run everything, then judge --------------------------------------------------- +guard_allowlist +stage_assets +guard_forbidden_paths +guard_forbidden_strings +guard_gitleaks + +cp "$REPORT" "$OUT/publish-guard-report.txt" 2>/dev/null || true + +if [ "$RAN" -ne "$GUARDS_EXPECTED" ]; then + cant_tell "self-check" "$RAN of $GUARDS_EXPECTED guards reached a verdict" +fi +case "$WORST" in + 0) echo "publish-guard: OK — all $GUARDS_EXPECTED guards ran and passed; $OUT/tree is the deliverable." ;; + 1) echo "::error::publish-guard: REFUSED — do not publish $OUT (see the [guard] lines above)." ;; + *) echo "::error::publish-guard: COULD NOT TELL — do not publish $OUT (see the [guard] lines above)." ;; +esac +exit "$WORST" diff --git a/scripts/publish-mirror.sh b/scripts/publish-mirror.sh new file mode 100755 index 00000000..c15d6be9 --- /dev/null +++ b/scripts/publish-mirror.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# ============================================================================= +# publish-mirror.sh — the publish half of the mirror pipeline: push what +# scripts/publish-guard.sh staged and cleared to the public mirror repository. +# +# Three subcommands, each one step of the workflow, each refusing on its own: +# +# target --mirror NAME --source-repo OWNER/REPO [--owner OWNER] +# [--output FILE] +# Validate the mirror name and print OWNER/NAME. Refuses an empty +# name (the mirror is unset until it exists — there is no default), +# a name with characters GitHub does not allow, and a target equal +# to the source repository: publishing onto the source would +# replace the default branch of the repo you are standing in. +# --output appends `repo=OWNER/NAME` and `name=NAME` to FILE (the +# workflow passes $GITHUB_OUTPUT). +# +# tree --stage DIR --repo OWNER/NAME --branch NAME --message TEXT +# [--remote URL] [--output FILE] +# Clone the mirror branch (or start it when the mirror has none), +# replace its content with DIR, commit, PLAIN push. A diverged +# remote rejects the push; nothing here ever forces. Prints +# `pushed ` or `unchanged `; --output appends +# `result=pushed|unchanged` and `sha=` to FILE. +# +# Results go to --output, refusals go to stdout: a caller that captured stdout +# with `$(...)` to read the result would swallow the `::error::` line of a +# refusal, so the workflow runs these commands directly and reads the file. +# Nothing is written to --output on a refusal. +# +# release --tag TAG --repo OWNER/NAME --target SHA --assets DIR +# --notes FILE [--prerelease] +# Create TAG on the mirror at SHA with every file in DIR attached. +# Refuses when TAG already exists on the mirror: a published release +# is never overwritten, and a re-run of a mirrored release is a +# human decision. +# +# Exit 0 done; 1 refused (the message says why); 2 could not tell (an input +# missing or unreadable, a remote that did not answer). "Cannot tell" never +# publishes. +# +# Authentication is the caller's: git reads its credential helper, `gh` reads +# GH_TOKEN. Nothing here takes a token argument, so no token can land on a +# command line. Commits are authored as PUBLISH_MIRROR_GIT_NAME / +# PUBLISH_MIRROR_GIT_EMAIL (default: github-actions[bot]). +# ============================================================================= +set -uo pipefail + +die1() { echo "::error::publish-mirror: REFUSED — $1"; exit 1; } +die2() { echo "::error::publish-mirror: COULD NOT TELL — $1 (never publishes)"; exit 2; } + +REPO_RE='^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$' +NAME_RE='^[A-Za-z0-9_.-]+$' + +# emit_output FILE KEY=VALUE... — append results for the caller (the workflow's +# $GITHUB_OUTPUT). An unwritable file is "could not tell": a result the caller +# never receives is a publish it cannot finish or account for. +emit_output() { + local file="$1"; shift + [ -n "$file" ] || return 0 + printf '%s\n' "$@" >>"$file" || die2 "could not write results to '$file'" +} + +cmd_target() { + local mirror="" source_repo="" owner="" output="" + while [ "$#" -gt 0 ]; do + case "$1" in + --mirror) mirror="${2:-}"; shift 2 ;; + --source-repo) source_repo="${2:-}"; shift 2 ;; + --owner) owner="${2:-}"; shift 2 ;; + --output) output="${2:-}"; shift 2 ;; + *) die2 "target: unknown argument '$1'" ;; + esac + done + [ -n "$source_repo" ] || die2 "target: --source-repo is required" + [[ "$source_repo" =~ $REPO_RE ]] || die2 "target: --source-repo '$source_repo' is not OWNER/REPO" + [ -n "$owner" ] || owner="${source_repo%%/*}" + [ -n "$mirror" ] || die1 "no mirror repository is configured (MIRROR_REPO is unset) — the mirror has no default, so nothing is published until one is named" + case "$mirror" in */*) die1 "mirror name '$mirror' must be a bare repository name in the '$owner' organisation, not OWNER/NAME" ;; esac + [[ "$mirror" =~ $NAME_RE ]] || die1 "mirror name '$mirror' contains characters a repository name cannot" + local full="$owner/$mirror" + if [ "$(printf '%s' "$full" | tr '[:upper:]' '[:lower:]')" = "$(printf '%s' "$source_repo" | tr '[:upper:]' '[:lower:]')" ]; then + die1 "mirror '$full' is this repository — publishing onto the source would replace its default branch" + fi + emit_output "$output" "repo=$full" "name=$mirror" + printf '%s\n' "$full" +} + +cmd_tree() { + local stage="" repo="" branch="" message="" remote="" output="" + while [ "$#" -gt 0 ]; do + case "$1" in + --stage) stage="${2:-}"; shift 2 ;; + --repo) repo="${2:-}"; shift 2 ;; + --branch) branch="${2:-}"; shift 2 ;; + --message) message="${2:-}"; shift 2 ;; + --remote) remote="${2:-}"; shift 2 ;; + --output) output="${2:-}"; shift 2 ;; + *) die2 "tree: unknown argument '$1'" ;; + esac + done + [ -n "$stage" ] && [ -n "$repo" ] && [ -n "$branch" ] && [ -n "$message" ] || die2 "tree: --stage, --repo, --branch and --message are all required" + [[ "$repo" =~ $REPO_RE ]] || die2 "tree: --repo '$repo' is not OWNER/NAME" + [[ "$branch" =~ ^[A-Za-z0-9_./-]+$ ]] || die2 "tree: --branch '$branch' is not a branch name" + [ -d "$stage" ] || die2 "tree: stage '$stage' is not a directory" + [ -n "$(find "$stage" -type f | head -1)" ] || die2 "tree: stage '$stage' holds no files — an empty deliverable is not published" + [ ! -e "$stage/.git" ] || die2 "tree: stage '$stage' contains a .git entry — that is a checkout, not a staged deliverable" + [ -n "$remote" ] || remote="https://github.com/$repo.git" + + local name="${PUBLISH_MIRROR_GIT_NAME:-github-actions[bot]}" + local email="${PUBLISH_MIRROR_GIT_EMAIL:-github-actions[bot]@users.noreply.github.com}" + # The checkout lives in its own subdirectory of the scratch dir; error + # captures live BESIDE it, never inside it, or they would be committed. + local scratch work + scratch="$(mktemp -d "${TMPDIR:-/tmp}/publish-mirror.XXXXXX")" && [ -d "$scratch" ] || die2 "tree: could not create a scratch directory" + trap 'rm -rf "$scratch"' EXIT + work="$scratch/work" + mkdir -p "$work" || die2 "tree: could not create the checkout directory" + + git -C "$work" init -q || die2 "tree: git init failed" + git -C "$work" remote add origin "$remote" || die2 "tree: could not add remote" + # Absent-vs-unreachable are different answers: ls-remote's own status says + # whether the remote answered; an empty answer says the branch is not there. + local heads rc existed=0 + heads="$(git -C "$work" ls-remote --heads origin "refs/heads/$branch" 2>"$scratch/lsr.err")"; rc=$? + [ "$rc" -eq 0 ] || die2 "tree: the mirror remote did not answer (git ls-remote exited $rc: $(tr '\n' ' ' <"$scratch/lsr.err"))" + if [ -n "$heads" ]; then + existed=1 + git -C "$work" fetch -q --depth 1 origin "refs/heads/$branch" || die2 "tree: could not fetch '$branch' from the mirror" + git -C "$work" checkout -q -B "$branch" FETCH_HEAD || die2 "tree: could not check out '$branch'" + find "$work" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + || die2 "tree: could not clear the checkout" + else + git -C "$work" checkout -q --orphan "$branch" || die2 "tree: could not start branch '$branch'" + fi + cp -Rp "$stage"/. "$work"/ || die2 "tree: could not copy the stage into the checkout" + git -C "$work" add -A || die2 "tree: git add failed" + local sha + if [ "$existed" -eq 1 ] && git -C "$work" diff --cached --quiet; then + sha="$(git -C "$work" rev-parse HEAD)" + emit_output "$output" "result=unchanged" "sha=$sha" + echo "unchanged $sha" + return 0 + fi + git -C "$work" -c user.name="$name" -c user.email="$email" commit -q -m "$message" || die2 "tree: git commit failed" + # A PLAIN push. If the mirror moved underneath us the push is rejected and + # this exits 2; the answer is to re-run, never to force. + git -C "$work" push -q origin "HEAD:refs/heads/$branch" 2>"$scratch/push.err" || die2 "tree: push to '$repo' '$branch' was rejected: $(tr '\n' ' ' <"$scratch/push.err")" + sha="$(git -C "$work" rev-parse HEAD)" + emit_output "$output" "result=pushed" "sha=$sha" + echo "pushed $sha" +} + +cmd_release() { + local tag="" repo="" target="" assets="" notes="" prerelease=0 + while [ "$#" -gt 0 ]; do + case "$1" in + --tag) tag="${2:-}"; shift 2 ;; + --repo) repo="${2:-}"; shift 2 ;; + --target) target="${2:-}"; shift 2 ;; + --assets) assets="${2:-}"; shift 2 ;; + --notes) notes="${2:-}"; shift 2 ;; + --prerelease) prerelease=1; shift ;; + *) die2 "release: unknown argument '$1'" ;; + esac + done + [ -n "$tag" ] && [ -n "$repo" ] && [ -n "$target" ] && [ -n "$assets" ] && [ -n "$notes" ] || die2 "release: --tag, --repo, --target, --assets and --notes are all required" + [[ "$repo" =~ $REPO_RE ]] || die2 "release: --repo '$repo' is not OWNER/NAME" + [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$ ]] || die1 "release: '$tag' is not a release tag (vX.Y.Z or vX.Y.Z-
)"
+  [[ "$target" =~ ^[0-9a-f]{40}$ ]] || die2 "release: --target '$target' is not a full commit sha"
+  [ -d "$assets" ] || die2 "release: assets '$assets' is not a directory"
+  [ -s "$notes" ] || die2 "release: notes file '$notes' is missing or empty"
+  local -a files=()
+  while IFS= read -r f; do files+=("$f"); done < <(find "$assets" -mindepth 1 -maxdepth 1 -type f | sort)
+  [ "${#files[@]}" -gt 0 ] || die2 "release: '$assets' holds no files — a release with no assets is not what a customer downloads"
+  command -v gh >/dev/null 2>&1 || die2 "release: gh is not on PATH"
+
+  # Existing release → refuse. `gh release view` exits 1 for "not found" AND for
+  # auth or network failure, so the text decides which it was; anything that is
+  # not a clear "not found" is "cannot tell".
+  local err rc
+  err="$(gh release view "$tag" --repo "$repo" 2>&1 >/dev/null)"; rc=$?
+  if [ "$rc" -eq 0 ]; then die1 "release: '$tag' already exists on '$repo' — a mirrored release is never overwritten"; fi
+  printf '%s' "$err" | grep -qi 'release not found' || die2 "release: could not read releases of '$repo' (gh exited $rc: $(printf '%s' "$err" | tr '\n' ' '))"
+
+  local -a args=(release create "$tag" --repo "$repo" --target "$target" --title "$tag" --notes-file "$notes")
+  [ "$prerelease" -eq 1 ] && args+=(--prerelease)
+  gh "${args[@]}" "${files[@]}" || die2 "release: gh release create exited $?"
+  echo "released $tag on $repo at $target with ${#files[@]} asset(s)"
+}
+
+[ "$#" -ge 1 ] || die2 "a subcommand is required: target | tree | release"
+sub="$1"; shift
+case "$sub" in
+  target)  cmd_target "$@" ;;
+  tree)    cmd_tree "$@" ;;
+  release) cmd_release "$@" ;;
+  -h|--help) sed -n '2,/^# ====/p' "$0" | sed 's/^# \{0,2\}//' ;;
+  *) die2 "unknown subcommand '$sub' (target | tree | release)" ;;
+esac
diff --git a/scripts/tests/image-refresh-latched-annotate.bats b/scripts/tests/image-refresh-latched-annotate.bats
new file mode 100644
index 00000000..c7a54ba4
--- /dev/null
+++ b/scripts/tests/image-refresh-latched-annotate.bats
@@ -0,0 +1,164 @@
+#!/usr/bin/env bats
+# image-refresh writes the Pass-0 stale-pin annotations BEFORE the restart block,
+# so they survive a tick that is both off-digest (restart_needed=1) and LATCHED
+# (refresh-attempt >= MAX_REFRESH_ATTEMPTS).
+#
+# #1008 item 1. The #563 flap guard does `WARN + FLAP_KEY + exit 0` once
+# the attempt counter reaches MAX -- BEFORE the digest-record annotate at the end
+# of the tick. When the stale-pin CLEARS were batched into that final annotate,
+# a latched tick dropped them, leaving a FALSE "pin is stale" finding to persist
+# forever -- and on the exact tick refresh is dead, when the finding matters most.
+# The fix moves the stale-pin writes into their own bounded annotate above the
+# restart block; the `last-refreshed` digest record deliberately stays BELOW,
+# after a successful rollout (@shujaatTracebloc on #1008).
+#
+# This asserts BEHAVIOUR: it extracts the shipped tail (the stale-pin annotate +
+# the restart block + the final digest annotate) from the RENDERED chart and
+# drives it with kubectl and the attempt-counter read stubbed, so re-batching the
+# stale-pin writes back into the final annotate reddens.
+
+setup() {
+  TMP="$(mktemp -d)"
+  CHART="${BATS_TEST_DIRNAME}/../../client"
+  helm template t "$CHART" --set clientId=x --set clientPassword=y \
+    --set storageClass.create=false > "$TMP/rendered.yaml"
+  python3 - "$TMP/rendered.yaml" "$TMP/tail.sh" <<'PYX'
+import sys
+
+try:
+    import yaml
+except ImportError:
+    sys.exit("[ERROR] PyYAML required (pip install pyyaml)")
+
+MARKER = "already on the pinned digest; no-op"
+
+def walk(o):
+    if isinstance(o, str) and MARKER in o:
+        return o
+    if isinstance(o, dict):
+        for v in o.values():
+            r = walk(v)
+            if r:
+                return r
+    if isinstance(o, list):
+        for v in o:
+            r = walk(v)
+            if r:
+                return r
+
+script = None
+for d in yaml.safe_load_all(open(sys.argv[1])):
+    if not d:
+        continue
+    script = walk(d)
+    if script:
+        break
+assert script, "no rendered image-refresh script found"
+
+lines = script.splitlines()
+start = next(i for i, l in enumerate(lines)
+            if l.strip() == 'if [ -n "$stale_pin_args" ]; then')
+# the LAST `log "tick complete"` -- the flap-guard early exits use the same line,
+# so the first match would truncate the region mid-restart-block.
+end = max(i for i in range(start, len(lines))
+         if lines[i].strip() == 'log "tick complete"')
+region = lines[start:end + 1]
+indent = min(len(l) - len(l.lstrip()) for l in region if l.strip())
+open(sys.argv[2], "w").write("\n".join(l[indent:] for l in region))
+PYX
+}
+teardown() { rm -rf "$TMP"; }
+
+# Drives the shipped tail with kubectl + the ATTEMPT_KEY read stubbed.
+#   $1 = STUB_ATTEMPT  what get_annotation returns for ATTEMPT_KEY (the flap count)
+#   $2 = JM_SET_ARGS   `set image` args (non-empty => a rollout runs, stubbed OK)
+# stale_pin_args and annotate_args are always populated so the test can assert
+# which of the two landed.
+#
+# The kubectl stub records EVERY call to "$TMP/calls.log" rather than stdout,
+# because the stale-pin annotate is wrapped in a non-fatal handler that discards
+# its stdout (`2>&1 >/dev/null`) -- exactly as a real silent-success annotate
+# would. The file captures the call regardless of the caller's redirections;
+# assert kubectl invocations against "$TMP/calls.log" and log lines against stdout.
+run_tail() {
+  : > "$TMP/calls.log"
+  cat > "$TMP/harness.sh" <> "\$CALLS"; }
+get_annotation() { case "\$1" in "\$ATTEMPT_KEY") printf '%s' "\$STUB_ATTEMPT" ;; esac; }
+$(cat "$TMP/tail.sh")
+EOF
+  sh "$TMP/harness.sh" "${1:-0}" "${2:-}"
+}
+
+@test "the harness really extracted the shipped tail (not an empty file)" {
+  [ -s "$TMP/tail.sh" ] || return 1
+  grep -q 'stale_pin_args' "$TMP/tail.sh" || return 1
+  grep -q 'restart_needed' "$TMP/tail.sh" || return 1
+}
+
+@test "LATCHED tick (restart_needed=1, attempt>=MAX): stale-pin clear LANDS, digest record does NOT" {
+  # The acceptance case (#1008 item 1). attempt=3, MAX=3 -> the flap guard
+  # WARNs, annotates FLAP_KEY, and exit 0s. The stale-pin clear must already have
+  # been written (before the restart block); the last-refreshed digest record
+  # must NOT be (its annotate is after the guard and never runs).
+  run run_tail "3"
+  [ "$status" -eq 0 ] || return 1
+  calls="$(cat "$TMP/calls.log")"
+  # stale-pin clear landed, above the restart block
+  [[ "$calls" == *"annotate deployment"*"tracebloc.io/stale-pin-jobs-manager-"* ]] || return 1
+  # the flap guard fired
+  [[ "$output" == *"FLAP DETECTED"* ]] || return 1
+  [[ "$calls" == *"tracebloc.io/refresh-flap-detected=3"* ]] || return 1
+  # the digest record did NOT land (dropped by the exit 0, as designed)
+  [[ "$calls" != *"last-refreshed-jobs-manager-digest=sha256:beef"* ]] || return 1
+}
+
+@test "NON-latched tick (attempt/dev/null || { echo "[ERROR] PyYAML required (pip install pyyaml)"; return 1; }
+  SHIM="$BATS_TEST_TMPDIR/shim"
+  WORK="$BATS_TEST_TMPDIR/work"
+  mkdir -p "$SHIM" "$WORK" "$BATS_TEST_TMPDIR/runner-temp"
+  # gh shim: `release view` prints GH_RELEASE_JSON (or fails with GH_RELEASE_RC);
+  # `api .../releases/latest` prints GH_LATEST_TAG (or fails with GH_LATEST_RC);
+  # any other `api ... --jq .sha` prints GH_API_SHA (or fails with GH_API_RC).
+  # Every call is logged so a test can assert WHICH question the step asked.
+  cat >"$SHIM/gh" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' "$*" >>"${GH_LOG:?}"
+case "${1:-} ${2:-}" in
+  "release view")
+    [ "${GH_RELEASE_RC:-0}" -eq 0 ] || { echo "release not found" >&2; exit "$GH_RELEASE_RC"; }
+    printf '%s\n' "${GH_RELEASE_JSON:?}" ;;
+  "api "*"/releases/latest")
+    [ "${GH_LATEST_RC:-0}" -eq 0 ] || { echo "HTTP 404: Not Found" >&2; exit "$GH_LATEST_RC"; }
+    printf '%s\n' "${GH_LATEST_TAG:?}" ;;
+  "api "*)
+    [ "${GH_API_RC:-0}" -eq 0 ] || { echo "HTTP 409: Git Repository is empty" >&2; exit "$GH_API_RC"; }
+    printf '%s\n' "${GH_API_SHA:?}" ;;
+esac
+exit 0
+EOF
+  chmod +x "$SHIM/gh"
+  export GH_LOG="$BATS_TEST_TMPDIR/gh.log"
+  : >"$GH_LOG"
+  export GITHUB_OUTPUT="$BATS_TEST_TMPDIR/github-output"
+  : >"$GITHUB_OUTPUT"
+  export RUNNER_TEMP="$BATS_TEST_TMPDIR/runner-temp"
+  export GITHUB_REPOSITORY="example/source"
+  export GITHUB_WORKSPACE="$REPO_ROOT"
+  # The plan step's job-level env, every field set (the body runs under set -u).
+  export EVENT_NAME=workflow_run INPUT_TAG="" INPUT_DRY_RUN="" INPUT_MIRROR="" INPUT_STRICT=""
+  export RUN_HEAD_BRANCH="" RUN_HEAD_SHA="" VAR_MIRROR="" VAR_STRICT=""
+  export TAG="" EXPECT_SHA="" BRANCH="" REPO=""
+}
+
+SHA_A=1111111111111111111111111111111111111111
+SHA_B=2222222222222222222222222222222222222222
+
+# step_run   — print that step's `run:` body; refuse when absent.
+step_run() {
+  python3 - "$1" "$2" <<'PY'
+import sys
+try:
+    import yaml
+except ImportError:
+    sys.exit("[ERROR] PyYAML required (pip install pyyaml)")
+path, want = sys.argv[1], sys.argv[2]
+try:
+    with open(path) as fh:
+        doc = yaml.safe_load(fh)
+except (OSError, yaml.YAMLError) as e:
+    sys.exit("FAIL: cannot read or parse workflow %s: %s" % (path, e))
+steps = ((doc.get("jobs") or {}).get("publish") or {}).get("steps") or []
+for s in steps:
+    if isinstance(s, dict) and s.get("id") == want:
+        if "run" not in s:
+            sys.exit("FAIL: step %r has no run: body" % want)
+        sys.stdout.write(s["run"])
+        sys.exit(0)
+sys.exit("FAIL: no step with id %r in %s" % (want, path))
+PY
+}
+
+# run_step  [cwd] — execute the step body as Actions would: its own
+# bash, the exported env, the gh shim first on PATH. RUN_WF names the workflow
+# to read the body from (default: the real one; a mutation test points it at a
+# mutated copy so the same step body runs with one decision removed).
+run_step() {
+  local body="$BATS_TEST_TMPDIR/step-$1.sh"
+  step_run "${RUN_WF:-$WF}" "$1" >"$body" || { cat "$body"; return 1; }
+  local dir="${2:-$WORK}"
+  # `bash -e`: what Actions runs a `run:` body with. A body that relies on
+  # surviving a failing command (the guard steps' tee pipeline) is tested under
+  # the same errexit it gets in CI, or the test proves nothing about the step.
+  run env PATH="$SHIM:$PATH" bash -c "cd '$dir' && bash -e '$body'"
+}
+
+out() { grep -E "^$1=" "$GITHUB_OUTPUT" | tail -1 | cut -d= -f2-; }
+
+release_json() { #  
+  printf '{"tagName":"%s","isDraft":false,"isPrerelease":%s}' "$1" "$2"
+}
+
+# ── plan ──────────────────────────────────────────────────────────────────────
+
+@test "plan: the newest stable release from workflow_run publishes tree and release, pinned to head_sha" {
+  export RUN_HEAD_BRANCH=v1.2.3 RUN_HEAD_SHA="$SHA_A" GH_RELEASE_JSON GH_LATEST_TAG=v1.2.3
+  GH_RELEASE_JSON="$(release_json v1.2.3 false)"
+  run_step plan
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$(out tag)" = "v1.2.3" ] || return 1
+  [ "$(out dry_run)" = "false" ] || return 1
+  [ "$(out prerelease)" = "false" ] || return 1
+  [ "$(out publish_tree)" = "true" ] || return 1
+  [ "$(out expect_sha)" = "$SHA_A" ] || return 1
+  grep -q '^release view v1.2.3 --repo example/source --json tagName,isDraft,isPrerelease$' "$GH_LOG" || return 1
+  # The newest-stable answer is GitHub's, asked of the SOURCE repository.
+  grep -q '^api repos/example/source/releases/latest --jq .tag_name$' "$GH_LOG" || { cat "$GH_LOG"; return 1; }
+  [[ "$output" != *"::notice::"* ]] || return 1
+}
+
+@test "plan: a prerelease mirrors only its release — publish_tree=false, and the log says why" {
+  export RUN_HEAD_BRANCH=v1.2.3-rc.1 RUN_HEAD_SHA="$SHA_A" GH_RELEASE_JSON
+  GH_RELEASE_JSON="$(release_json v1.2.3-rc.1 true)"
+  run_step plan
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$(out prerelease)" = "true" ] || return 1
+  [ "$(out publish_tree)" = "false" ] || return 1
+  [ "$(out expect_sha)" = "$SHA_A" ] || return 1
+  [[ "$output" == *"::notice::'v1.2.3-rc.1' is a prerelease: only its GitHub release is mirrored (marked prerelease)."*"default branch and chart index are not pushed"* ]] || { echo "$output"; return 1; }
+  # A prerelease is never the newest stable release; the question is not asked.
+  ! grep -q 'releases/latest' "$GH_LOG" || return 1
+}
+
+@test "plan: a release whose isPrerelease is not a boolean is refused — the tree push is armed only by an explicit false" {
+  export RUN_HEAD_BRANCH=v1.2.3 RUN_HEAD_SHA="$SHA_A" GH_RELEASE_JSON GH_LATEST_TAG=v1.2.3
+  GH_RELEASE_JSON='{"tagName":"v1.2.3","isDraft":false}'
+  run_step plan
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"::error::release 'v1.2.3' reports isPrerelease 'null' — not a boolean, refusing"* ]] || { echo "$output"; return 1; }
+  [ ! -s "$GITHUB_OUTPUT" ] || return 1
+}
+
+@test "plan: a stable release that is not the newest one mirrors only its release — publish_tree=false, not marked prerelease" {
+  export RUN_HEAD_BRANCH=v1.2.3 RUN_HEAD_SHA="$SHA_A" GH_RELEASE_JSON GH_LATEST_TAG=v1.3.0
+  GH_RELEASE_JSON="$(release_json v1.2.3 false)"
+  run_step plan
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$(out tag)" = "v1.2.3" ] || return 1
+  [ "$(out prerelease)" = "false" ] || return 1
+  [ "$(out publish_tree)" = "false" ] || return 1
+  [ "$(out expect_sha)" = "$SHA_A" ] || return 1
+  [[ "$output" == *"::notice::'v1.2.3' is not the newest stable release (v1.3.0 is): only its GitHub release is mirrored."*"they keep the newest stable release"* ]] || { echo "$output"; return 1; }
+}
+
+@test "plan: an unreadable releases/latest is refused — 'cannot tell' does not replace the default branch" {
+  export RUN_HEAD_BRANCH=v1.2.3 RUN_HEAD_SHA="$SHA_A" GH_RELEASE_JSON GH_LATEST_TAG="" GH_LATEST_RC=1
+  GH_RELEASE_JSON="$(release_json v1.2.3 false)"
+  run_step plan
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"::error::cannot read the newest stable release of example/source (releases/latest) — refusing"* ]] || { echo "$output"; return 1; }
+  [ ! -s "$GITHUB_OUTPUT" ] || return 1
+}
+
+@test "plan mutation: with the newest-stable comparison removed, an older tag publishes the tree — the test above catches it" {
+  local m
+  m="$(mutate "s = [s for s in steps if s.get('id') == 'plan'][0]; s['run'] = s['run'].replace('if [ \"\$LATEST\" != \"\$TAG\" ]; then', 'if false; then')")" || return 1
+  export RUN_HEAD_BRANCH=v1.2.3 RUN_HEAD_SHA="$SHA_A" GH_RELEASE_JSON GH_LATEST_TAG=v1.3.0
+  GH_RELEASE_JSON="$(release_json v1.2.3 false)"
+  RUN_WF="$m" run_step plan
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  # The mutated body lets the older tag through: this is the outcome the real
+  # test refuses, so the assertion there is live, not vacuous.
+  [ "$(out publish_tree)" = "true" ] || { echo "$output"; return 1; }
+}
+
+@test "plan: a workflow_run whose head is a branch, not a tag, is refused before anything is read" {
+  export RUN_HEAD_BRANCH=develop RUN_HEAD_SHA="$SHA_A"
+  run_step plan
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"::error::'develop' is not a release tag"* ]] || return 1
+  [ ! -s "$GITHUB_OUTPUT" ] || return 1
+  [ ! -s "$GH_LOG" ] || return 1
+}
+
+@test "plan: a release whose tag_name is not the run's tag is refused" {
+  export RUN_HEAD_BRANCH=v1.2.3 RUN_HEAD_SHA="$SHA_A" GH_RELEASE_JSON
+  GH_RELEASE_JSON="$(release_json v9.9.9 false)"
+  run_step plan
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"::error::release 'v1.2.3' reports tag_name 'v9.9.9' — the tag and the release disagree, refusing."* ]] || return 1
+  [ ! -s "$GITHUB_OUTPUT" ] || return 1
+}
+
+@test "plan: a workflow_run without a full head_sha to pin the tag to is refused" {
+  export RUN_HEAD_BRANCH=v1.2.3 RUN_HEAD_SHA=abc123 GH_RELEASE_JSON
+  GH_RELEASE_JSON="$(release_json v1.2.3 false)"
+  run_step plan
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"::error::cannot determine the commit release 'v1.2.3' was cut from (got 'abc123')"* ]] || return 1
+  [ ! -s "$GITHUB_OUTPUT" ] || return 1
+}
+
+@test "plan: a dispatch takes the expected commit from the API and is a dry run unless told 'false'" {
+  export EVENT_NAME=workflow_dispatch INPUT_TAG=v1.2.3 INPUT_DRY_RUN=true GH_RELEASE_JSON GH_API_SHA="$SHA_B" GH_LATEST_TAG=v1.2.3
+  GH_RELEASE_JSON="$(release_json v1.2.3 false)"
+  run_step plan
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$(out dry_run)" = "true" ] || return 1
+  [ "$(out publish_tree)" = "true" ] || return 1
+  [ "$(out expect_sha)" = "$SHA_B" ] || return 1
+  grep -q '^api repos/example/source/commits/v1.2.3 --jq .sha$' "$GH_LOG" || return 1
+  # The API not answering is "cannot tell", never an unpinned fetch.
+  : >"$GITHUB_OUTPUT"
+  GH_API_RC=1 run_step plan
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"::error::cannot determine the commit release 'v1.2.3' was cut from (got '')"* ]] || return 1
+}
+
+# ── src: the release tag is data, fetched only at the expected commit ─────────
+
+make_origin() { # a bare origin with one commit tagged v1.2.3 (annotated); WORK becomes its clone; prints the commit
+  local seed="$BATS_TEST_TMPDIR/seed" bare="$BATS_TEST_TMPDIR/origin.git"
+  git init -q --bare "$bare"
+  # The bare HEAD is pinned to `main` explicitly: with init.defaultBranch unset
+  # (a fresh runner) it would point at a `master` that never receives a push,
+  # the clone would have an unborn HEAD, and `rev-parse HEAD` would print the
+  # literal word HEAD as the expected sha (measured on the first CI run).
+  git -C "$bare" symbolic-ref HEAD refs/heads/main
+  git init -q "$seed"
+  printf 'readme\n' >"$seed/README.md"
+  git -C "$seed" -c user.name=t -c user.email=t@example.invalid add README.md
+  git -C "$seed" -c user.name=t -c user.email=t@example.invalid commit -q -m one
+  git -C "$seed" -c user.name=t -c user.email=t@example.invalid tag -a v1.2.3 -m v1.2.3
+  git -C "$seed" push -q "file://$bare" HEAD:refs/heads/main refs/tags/v1.2.3
+  rm -rf "$WORK"
+  git clone -q "file://$bare" "$WORK" 2>/dev/null
+  git -C "$seed" rev-parse --verify HEAD
+}
+
+@test "src: the tag is fetched into a detached worktree outside the checkout, only at the expected commit" {
+  local sha
+  sha="$(make_origin)"
+  export TAG=v1.2.3 EXPECT_SHA="$sha"
+  run_step src
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$(out dir)" = "$RUNNER_TEMP/release-src" ] || return 1
+  [ "$(git -C "$RUNNER_TEMP/release-src" rev-parse HEAD)" = "$sha" ] || return 1
+  [ -f "$RUNNER_TEMP/release-src/README.md" ] || return 1
+  [[ "$output" == *"release source: v1.2.3 at $sha (data only)"* ]] || return 1
+}
+
+@test "src: a tag that does not resolve to the expected commit is refused and nothing is checked out" {
+  make_origin >/dev/null
+  export TAG=v1.2.3 EXPECT_SHA="$SHA_B"
+  run_step src
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"::error::tag 'v1.2.3' resolves to "*" but the release was cut at $SHA_B — the tag has moved or the run is not this release's; refusing."* ]] || return 1
+  [ ! -e "$RUNNER_TEMP/release-src" ] || return 1
+  [ ! -s "$GITHUB_OUTPUT" ] || return 1
+}
+
+@test "src: a tag origin does not have is refused" {
+  make_origin >/dev/null
+  export TAG=v9.9.9 EXPECT_SHA="$SHA_A"
+  run_step src
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"::error::could not fetch tag 'v9.9.9' from origin"* ]] || return 1
+  [ ! -e "$RUNNER_TEMP/release-src" ] || return 1
+}
+
+# ── target / keep: refusals annotate, results go to GITHUB_OUTPUT ─────────────
+
+@test "target: an unset MIRROR_REPO is refused with the ::error:: line IN THE STEP LOG, nothing captured" {
+  export VAR_MIRROR="" INPUT_MIRROR=""
+  run_step target "$REPO_ROOT"
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"::error::publish-mirror: REFUSED — no mirror repository is configured (MIRROR_REPO is unset)"* ]] || { echo "$output"; return 1; }
+  [ ! -s "$GITHUB_OUTPUT" ] || return 1
+}
+
+@test "target: a configured mirror lands in GITHUB_OUTPUT as repo= and name=" {
+  export VAR_MIRROR=source-public INPUT_MIRROR=""
+  run_step target "$REPO_ROOT"
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$(out repo)" = "example/source-public" ] || return 1
+  [ "$(out name)" = "source-public" ] || return 1
+}
+
+@test "keep: a release-only publish is pinned to the mirror's default-branch head; an empty mirror is refused" {
+  export REPO=example/source-public BRANCH=main TAG=v1.2.3-rc.1 GH_API_SHA="$SHA_B"
+  run_step keep
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$(out sha)" = "$SHA_B" ] || return 1
+  grep -q '^api repos/example/source-public/commits/main --jq .sha$' "$GH_LOG" || return 1
+  [[ "$output" == *"release-only v1.2.3-rc.1: default branch 'main' and gh-pages left untouched"* ]] || { echo "$output"; return 1; }
+  : >"$GITHUB_OUTPUT"
+  GH_API_RC=1 run_step keep
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"::error::'v1.2.3-rc.1' does not replace the mirror's default branch (a prerelease, or not the newest stable release) and the mirror has no commit on 'main' to pin its release to — the first publish to an empty mirror must be the newest stable release."* ]] || { echo "$output"; return 1; }
+  [ ! -s "$GITHUB_OUTPUT" ] || return 1
+}
+
+# ── guard steps: a refusal reaches the step summary, the step exits with it ────
+
+# fake_guard  — a cwd holding a scripts/publish-guard.sh that refuses
+# (prints a guard line, exits 1) whatever it is asked; the guard itself has its
+# own suite, this is about what the STEP does with a refusal.
+fake_guard() {
+  mkdir -p "$1/scripts"
+  cat >"$1/scripts/publish-guard.sh" <<'EOF'
+#!/usr/bin/env bash
+echo "::error::publish-guard: [forbidden-strings] REFUSED — planted refusal"
+exit 1
+EOF
+}
+
+@test "guard-tree: a guard refusal is written to the step summary and the step exits with the guard's status" {
+  fake_guard "$WORK"
+  export GITHUB_STEP_SUMMARY="$BATS_TEST_TMPDIR/summary.md" TAG="" STRICT="" SRC_DIR=""
+  : >"$GITHUB_STEP_SUMMARY"
+  run_step guard-tree
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"REFUSED — planted refusal"* ]] || { echo "$output"; return 1; }
+  grep -q '^## Mirror publish — tree$' "$GITHUB_STEP_SUMMARY" || { cat "$GITHUB_STEP_SUMMARY"; return 1; }
+  grep -q 'REFUSED — planted refusal' "$GITHUB_STEP_SUMMARY" || { cat "$GITHUB_STEP_SUMMARY"; return 1; }
+}
+
+@test "guard-tree mutation: without catching the guard's status, errexit skips the summary — the test above catches it" {
+  local m
+  m="$(mutate "s = [s for s in steps if s.get('id') == 'guard-tree'][0]; s['run'] = s['run'].replace(' || rc=\$?', '')")" || return 1
+  fake_guard "$WORK"
+  export GITHUB_STEP_SUMMARY="$BATS_TEST_TMPDIR/summary.md" TAG="" STRICT="" SRC_DIR=""
+  : >"$GITHUB_STEP_SUMMARY"
+  RUN_WF="$m" run_step guard-tree
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  # The refusal happened, but the summary is empty: the finding, reproduced.
+  [ ! -s "$GITHUB_STEP_SUMMARY" ] || { cat "$GITHUB_STEP_SUMMARY"; return 1; }
+}
+
+# ── shape: derived from the workflow, one implementation for real and mutated ──
+
+# shape  — OK lines / one FAIL line. Every rule is derived from
+# the steps themselves (which steps check out, which steps invoke the
+# publisher), never from a list of step names held here.
+shape() {
+  run python3 - "$1" <<'PY'
+import re, sys
+try:
+    import yaml
+except ImportError:
+    sys.exit("[ERROR] PyYAML required (pip install pyyaml)")
+
+path = sys.argv[1]
+
+
+def fail(msg):
+    print("FAIL: " + msg)
+    sys.exit(1)
+
+
+try:
+    with open(path) as fh:
+        doc = yaml.safe_load(fh)
+except (OSError, yaml.YAMLError) as e:
+    fail("cannot read or parse workflow %s: %s" % (path, e))
+publish = ((doc or {}).get("jobs") or {}).get("publish")
+if not isinstance(publish, dict):
+    fail("no `publish` job in %s" % path)
+steps = [s for s in (publish.get("steps") or []) if isinstance(s, dict)]
+if not steps:
+    fail("`publish` has no steps")
+
+GATE = "steps.plan.outputs.publish_tree == 'true'"
+
+checkouts = [s for s in steps if str(s.get("uses", "")).startswith("actions/checkout")]
+if not checkouts:
+    fail("no actions/checkout step — the tooling has to come from somewhere")
+for s in checkouts:
+    with_ = s.get("with") or {}
+    if "ref" in with_:
+        fail("checkout step %r takes a ref (%r): the tooling must come from this workflow's own commit, the release tag is data" % (s.get("name"), with_["ref"]))
+print("OK: %d checkout step(s), none with a ref" % len(checkouts))
+
+tree_pushes = [s for s in steps if re.search(r"publish-mirror\.sh\s+tree\b", str(s.get("run", "")))]
+if not tree_pushes:
+    fail("no step invokes `publish-mirror.sh tree` — nothing to gate")
+for s in tree_pushes:
+    if GATE not in str(s.get("if", "")):
+        fail("step %r pushes a tree without `if: ... %s` — a prerelease would replace the mirror's branch" % (s.get("name"), GATE))
+print("OK: %d tree push step(s), each gated on publish_tree" % len(tree_pushes))
+
+releases = [s for s in steps if re.search(r"publish-mirror\.sh\s+\"?\$\{?args|publish-mirror\.sh\s+release\b", str(s.get("run", "")))]
+if len(releases) != 1:
+    fail("expected exactly one release step, found %d" % len(releases))
+if "publish_tree" in str(releases[0].get("if", "")):
+    fail("the release step is gated on publish_tree — a prerelease must still get its release")
+print("OK: the release step is not gated on publish_tree")
+
+captured = [s for s in steps if re.search(r"\$\(\s*bash\s+scripts/publish-mirror\.sh", str(s.get("run", "")))]
+if captured:
+    fail("step %r captures publish-mirror.sh through $(...) — a refusal's ::error:: line would never reach the log" % captured[0].get("name"))
+print("OK: no step captures the publisher's output")
+
+fetches = [s for s in steps if re.search(r"git fetch[^\n]*refs/tags/", str(s.get("run", "")))]
+if len(fetches) != 1:
+    fail("expected exactly one step fetching a tag, found %d" % len(fetches))
+if "EXPECT_SHA" not in str(fetches[0].get("run", "")):
+    fail("the tag fetch step does not compare against EXPECT_SHA")
+print("OK: the one tag fetch compares against the expected commit")
+
+# The step that DECIDES publish_tree (writes it to GITHUB_OUTPUT) must ask
+# GitHub which release is the newest stable one; a decision that never asks
+# would let a rebuild of an older tag replace the mirror's default branch.
+deciders = [s for s in steps if re.search(r"publish_tree=", str(s.get("run", "")))]
+if len(deciders) != 1:
+    fail("expected exactly one step writing publish_tree=, found %d" % len(deciders))
+if "releases/latest" not in str(deciders[0].get("run", "")):
+    fail("step %r decides publish_tree without reading releases/latest — an older stable tag would replace the mirror's default branch" % deciders[0].get("name"))
+print("OK: the publish_tree decision reads releases/latest")
+PY
+}
+
+# mutate  — write a mutated copy of the real
+# workflow and print its path. The mutation is applied to the PARSED document,
+# and asserted to have changed it, so an inert edit cannot pass as coverage.
+mutate() {
+  local out="$BATS_TEST_TMPDIR/mutated-$BATS_TEST_NUMBER.yaml"
+  python3 - "$WF" "$out" "$1" <<'PY' || return 1
+import copy, sys
+try:
+    import yaml
+except ImportError:
+    sys.exit("[ERROR] PyYAML required (pip install pyyaml)")
+src, dst, expr = sys.argv[1], sys.argv[2], sys.argv[3]
+with open(src) as fh:
+    doc = yaml.safe_load(fh)
+before = copy.deepcopy(doc)
+steps = doc["jobs"]["publish"]["steps"]
+exec(expr, {"doc": doc, "steps": steps})
+if doc == before:
+    sys.exit("mutation did not change the document: " + expr)
+with open(dst, "w") as fh:
+    yaml.safe_dump(doc, fh, sort_keys=False)
+print(dst)
+PY
+}
+
+@test "shape: the real workflow — no checkout ref, tree pushes gated, release ungated, nothing captured, one pinned tag fetch" {
+  shape "$WF"
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"OK: 1 checkout step(s), none with a ref"* ]] || { echo "$output"; return 1; }
+  [[ "$output" == *"OK: 2 tree push step(s), each gated on publish_tree"* ]] || { echo "$output"; return 1; }
+  [[ "$output" == *"OK: the release step is not gated on publish_tree"* ]] || return 1
+  [[ "$output" == *"OK: no step captures the publisher's output"* ]] || return 1
+  [[ "$output" == *"OK: the one tag fetch compares against the expected commit"* ]] || return 1
+  [[ "$output" == *"OK: the publish_tree decision reads releases/latest"* ]] || { echo "$output"; return 1; }
+}
+
+@test "shape mutation: a publish_tree decision that never reads releases/latest reddens" {
+  local m
+  m="$(mutate "s = [s for s in steps if s.get('id') == 'plan'][0]; s['run'] = s['run'].replace('releases/latest', 'releases/tags/latest')")" || return 1
+  shape "$m"
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"FAIL: step "*"decides publish_tree without reading releases/latest"* ]] || { echo "$output"; return 1; }
+}
+
+@test "shape mutation: a checkout that takes a ref reddens" {
+  local m
+  m="$(mutate "[s for s in steps if str(s.get('uses','')).startswith('actions/checkout')][0]['with'] = {'ref': '\${{ steps.plan.outputs.tag }}'}")" || return 1
+  shape "$m"
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"FAIL: checkout step "*"takes a ref"* ]] || { echo "$output"; return 1; }
+}
+
+@test "shape mutation: a tree push without the publish_tree gate reddens" {
+  local m
+  m="$(mutate "s = [s for s in steps if s.get('id') == 'push'][0]; s['if'] = \"steps.plan.outputs.dry_run != 'true'\"")" || return 1
+  shape "$m"
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"FAIL: step "*"pushes a tree without"* ]] || { echo "$output"; return 1; }
+}
+
+@test "shape mutation: capturing the publisher through \$(...) reddens" {
+  local m
+  m="$(mutate "s = [s for s in steps if s.get('id') == 'target'][0]; s['run'] = 'REPO=\"\$(bash scripts/publish-mirror.sh target --mirror x --source-repo a/b)\"\n'")" || return 1
+  shape "$m"
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"FAIL: step "*"captures publish-mirror.sh through"* ]] || { echo "$output"; return 1; }
+}
+
+@test "shape mutation: a tag fetch that skips the EXPECT_SHA comparison reddens" {
+  local m
+  m="$(mutate "s = [s for s in steps if s.get('id') == 'src'][0]; s['run'] = s['run'].replace('EXPECT_SHA', 'IGNORED')")" || return 1
+  shape "$m"
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"FAIL: the tag fetch step does not compare against EXPECT_SHA"* ]] || { echo "$output"; return 1; }
+}
diff --git a/scripts/tests/publish-guard.bats b/scripts/tests/publish-guard.bats
new file mode 100644
index 00000000..dbb95033
--- /dev/null
+++ b/scripts/tests/publish-guard.bats
@@ -0,0 +1,638 @@
+#!/usr/bin/env bats
+# scripts/publish-guard.sh — stage the public deliverable and refuse anything
+# else. Every test drives the REAL script against a fixture git repository it
+# builds itself; the allowlists and forbidden lists the fixtures use are written
+# HERE, independently of the repo's own .publish-include / .publish-forbidden,
+# so the guard is never tested against its own copy of the rule. The repo's
+# real lists get their own section at the end, fed inputs this file writes.
+#
+# Three verdicts, and every test names the one it expects: 0 clean, 1 refused,
+# 2 could not tell. A mutation that reddens the guard must redden it for the
+# rule the test is named for, so refusals are matched on the guard's `[name]`
+# and the offending path/needle, not on the exit code alone.
+#
+# gitleaks is replaced by a PATH shim in every test but one, so the verdict
+# plumbing (clean / leak / crash / absent) is exercised hermetically; the last
+# gitleaks test runs the real binary when it is on PATH and skips visibly
+# otherwise.
+
+GUARD=""
+REPO=""
+SRC=""
+OUT=""
+SHIM=""
+
+setup() {
+  REPO="$(cd "${BATS_TEST_DIRNAME}/../.." && pwd)"
+  GUARD="$REPO/scripts/publish-guard.sh"
+  SRC="$BATS_TEST_TMPDIR/src"
+  OUT="$BATS_TEST_TMPDIR/out"
+  SHIM="$BATS_TEST_TMPDIR/shim"
+  mkdir -p "$SHIM"
+  # gitleaks shim: GL_MODE=clean|leak|crash. Prints its own marker so a test can
+  # tell the shim ran.
+  cat >"$SHIM/gitleaks" <<'EOF'
+#!/usr/bin/env bash
+case "${1:-}" in version) echo "shim-9.9.9"; exit 0 ;; esac
+echo "shim gitleaks ran: $*"
+case "${GL_MODE:-clean}" in
+  clean) exit 0 ;;
+  leak)  echo "Finding: REDACTED"; echo "File: tree/README.md"; echo "Line: 1"; exit 9 ;;
+  crash) echo "panic: shim crash"; exit 1 ;;
+esac
+EOF
+  chmod +x "$SHIM/gitleaks"
+  export PUBLISH_GUARD_GITLEAKS="$SHIM/gitleaks"
+  mk_repo
+}
+
+# A fixture repository with the shapes the guards must tell apart: deliverable
+# files, a test suite inside a deliverable directory, source, build files.
+mk_repo() {
+  mkdir -p "$SRC"
+  git -C "$SRC" init -q
+  git -C "$SRC" config user.email t@example.invalid
+  git -C "$SRC" config user.name t
+  add_file README.md 'Fixture chart. Deployment help: support@tracebloc.io'
+  add_file LICENSE 'Apache-2.0'
+  add_file client/Chart.yaml 'apiVersion: v2'
+  add_file client/templates/deploy.yaml 'kind: Deployment'
+  add_file client/tests/x_test.yaml 'suite: x'
+  add_file notes/tests 'a FILE named tests, not a directory'
+  add_file scripts/install.sh '#!/usr/bin/env bash'
+  add_file scripts/lib/common.sh 'log() { :; }'
+  add_file scripts/tests/a.bats '@test "x" { :; }'
+  add_file other/scripts/tests/b.bats '@test "y" { :; }'
+  add_file Makefile 'all:'
+  add_file CLAUDE.md 'guidance'
+  add_file .github/workflows/ci.yml 'on: push'
+  add_file internal/main.go 'package main'
+  commit
+  write_include \
+    'client/**' \
+    '!client/tests/**' \
+    'scripts/install.sh' \
+    'scripts/lib/**' \
+    'notes/**' \
+    'README.md' \
+    'LICENSE'
+  write_forbidden
+}
+
+add_file() { # path content
+  mkdir -p "$SRC/$(dirname "$1")"
+  printf '%s\n' "$2" >"$SRC/$1"
+  git -C "$SRC" add -f "$1"
+}
+commit() { git -C "$SRC" commit -q -m fixture --allow-empty; }
+write_include() { printf '# fixture allowlist\n' >"$SRC/.publish-include"; printf '%s\n' "$@" >>"$SRC/.publish-include"; }
+# The fixture's forbidden list — a DIFFERENT list from the repo's, written here.
+# Two string tiers: a mailbox and an ARN refuse; a tracker reference, an RFC
+# identifier and a non-production host are reported.
+write_forbidden() {
+  {
+    printf '[paths]\n'
+    printf '%s\n' 'tests/' 'scripts/tests/' 'Makefile' 'CLAUDE.md' '.github/' '*.go' 'kubeconfig*'
+    printf '\n[strings-refuse]\n'
+    printf '%s\n' '[A-Za-z0-9._%+-]+@tracebloc\.io' 'arn:aws:'
+    printf '\n[strings-report]\n'
+    printf '%s\n' 'backend#' 'RFC-0' 'dev-api\.tracebloc\.io'
+    printf '\n[allow]\n'
+    printf '%s\n' 'support@tracebloc\.io'
+  } >"$SRC/.publish-forbidden"
+}
+# A forbidden list with only a [paths] section and the refuse tier below, for
+# the tests that exercise path matching alone.
+paths_only_forbidden() { printf '[paths]\n%s\n[strings-refuse]\narn:aws:\n' "$1" >"$SRC/.publish-forbidden"; }
+# plant PATH LINE — append LINE to a fixture file, commit, and PROVE it landed
+# (an inert mutation and real coverage look identical in a log).
+plant() {
+  printf '%s\n' "$2" >>"$SRC/$1"
+  git -C "$SRC" add "$1" && commit
+  [ "$(grep -cF -- "$2" "$SRC/$1")" -eq 1 ] || return 1
+}
+guard() { run bash "$GUARD" --source "$SRC" --out "$OUT" "$@"; }
+staged() { ( cd "$OUT/tree" && find . -type f | sed 's|^\./||' | sort ); }
+
+# ── the clean case, and what "clean" is made of ───────────────────────────────
+
+@test "a clean fixture is staged, all four guards report, exit 0" {
+  guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[allowlist] staged 7 of 14 tracked file(s)"* ]] || { echo "$output"; return 1; }
+  [[ "$output" == *"[forbidden-paths] clean (7 pattern(s) against 7 staged path(s))"* ]] || return 1
+  [[ "$output" == *"[forbidden-strings] clean (2 refuse + 3 report needle(s), 1 allow token(s); 7 text file(s) scanned, 0 binary"* ]] || return 1
+  [[ "$output" == *"[gitleaks] clean"* ]] || return 1
+  [[ "$output" == *"publish-guard: OK — all 4 guards ran and passed"* ]] || return 1
+  # The staged tree is exactly the allowlisted set: nothing more, nothing less.
+  [ "$(staged | paste -sd' ' -)" = "LICENSE README.md client/Chart.yaml client/templates/deploy.yaml notes/tests scripts/install.sh scripts/lib/common.sh" ] || { staged; return 1; }
+  [ ! -e "$OUT/tree/Makefile" ] || return 1
+  [ ! -e "$OUT/tree/client/tests" ] || return 1
+  [ ! -e "$OUT/tree/internal" ] || return 1
+}
+
+@test "an untracked file matching the allowlist is not staged (tracked files only)" {
+  printf 'scratch\n' >"$SRC/client/untracked.yaml"
+  guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ ! -e "$OUT/tree/client/untracked.yaml" ] || return 1
+}
+
+@test "every guard still runs and reports after an earlier one has refused" {
+  write_include 'client/**' 'README.md'     # drops the !client/tests/** line
+  guard
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"[forbidden-paths] REFUSED"* ]] || return 1
+  [[ "$output" == *"[forbidden-strings] clean"* ]] || return 1
+  [[ "$output" == *"[gitleaks] clean"* ]] || return 1
+  [[ "$output" == *"publish-guard: REFUSED — do not publish"* ]] || return 1
+}
+
+# ── guard 2: forbidden paths ──────────────────────────────────────────────────
+
+@test "mutation: dropping the !client/tests/** exclusion is caught by the tests/ path rule" {
+  write_include 'client/**' 'README.md'
+  guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [ -f "$OUT/tree/client/tests/x_test.yaml" ] || return 1   # the mutation landed: the file WAS staged
+  [[ "$output" == *"[forbidden-paths] REFUSED — forbidden path pattern 'tests/' matched:"* ]] || return 1
+  [[ "$output" == *"tree:client/tests/x_test.yaml"* ]] || return 1
+}
+
+@test "mutation: allowlisting Go source is caught by *.go, by file name anywhere in the tree" {
+  write_include 'client/**' '!client/tests/**' 'internal/**' 'README.md'
+  guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"forbidden path pattern '*.go' matched:"* ]] || return 1
+  [[ "$output" == *"tree:internal/main.go"* ]] || return 1
+}
+
+@test "mutation: allowlisting the Makefile and CLAUDE.md is caught by name" {
+  write_include 'Makefile' 'CLAUDE.md' 'README.md'
+  guard
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"forbidden path pattern 'Makefile' matched:"*"tree:Makefile"* ]] || return 1
+  [[ "$output" == *"forbidden path pattern 'CLAUDE.md' matched:"*"tree:CLAUDE.md"* ]] || return 1
+}
+
+@test "mutation: a workflow directory is caught by .github/ as a directory anywhere" {
+  write_include '.github/**' 'README.md'
+  guard
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"forbidden path pattern '.github/' matched:"*"tree:.github/workflows/ci.yml"* ]] || return 1
+}
+
+@test "a pattern with a slash is anchored to the staged root; one without matches any component" {
+  write_include 'scripts/tests/**' 'other/**' 'README.md'
+  paths_only_forbidden 'scripts/tests/'
+  guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"forbidden path pattern 'scripts/tests/' matched:"* ]] || return 1
+  [[ "$output" == *"tree:scripts/tests/a.bats"* ]] || return 1
+  [[ "$output" != *"tree:other/scripts/tests/b.bats"* ]] || return 1   # anchored: not this one
+  # The unanchored form reaches it.
+  rm -rf "$OUT"
+  paths_only_forbidden 'tests/'
+  guard
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"tree:other/scripts/tests/b.bats"* ]] || return 1
+}
+
+@test "a trailing slash means 'as a directory': a FILE named tests is not a tests/ finding" {
+  guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ -f "$OUT/tree/notes/tests" ] || return 1
+}
+
+@test "a release asset named like a credential file is refused by the path rule" {
+  mkdir -p "$BATS_TEST_TMPDIR/assets"
+  printf 'apiVersion: v1\n' >"$BATS_TEST_TMPDIR/assets/kubeconfig.yaml"
+  guard --assets "$BATS_TEST_TMPDIR/assets"
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"forbidden path pattern 'kubeconfig*' matched:"*"assets:kubeconfig.yaml"* ]] || return 1
+}
+
+@test "no [paths] entries is could-not-tell, not clean" {
+  printf '[strings-refuse]\narn:aws:\n' >"$SRC/.publish-forbidden"
+  guard
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"[forbidden-paths] COULD NOT TELL — '"*"' has no [paths] entries"* ]] || return 1
+}
+
+# ── guard 3: forbidden strings — the refuse tier ──────────────────────────────
+
+@test "mutation: a refuse-tier needle in a staged file is refused, tier, file and line named" {
+  plant README.md 'role arn:aws:iam::000000000000:role/planted'
+  guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[forbidden-strings] REFUSED — [strings-refuse] needle 'arn:aws:' found in 1 staged line(s):"* ]] || return 1
+  [[ "$output" == *"    tree/README.md:2"* ]] || return 1
+  # The matched text itself is never echoed.
+  [[ "$output" != *"role/planted"* ]] || return 1
+  [[ "$output" == *"[forbidden-strings] 1 refuse-tier hit(s), 0 report-tier hit(s) counted"* ]] || return 1
+}
+
+@test "needles match case-insensitively" {
+  plant README.md 'ARN:AWS:s3:::planted'
+  guard
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"[strings-refuse] needle 'arn:aws:' found in 1 staged line(s)"* ]] || return 1
+}
+
+@test "an [allow] token spares a line only when it was the whole reason the needle hit" {
+  # The fixture README already carries support@tracebloc.io → clean.
+  guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  # A personal mailbox on the same line as the allowed one is still refused.
+  rm -rf "$OUT"
+  plant README.md 'or write to someone@tracebloc.io / support@tracebloc.io'
+  guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[strings-refuse] needle '[A-Za-z0-9._%+-]+@tracebloc\.io' found in 1 staged line(s):"*"tree/README.md:2"* ]] || return 1
+}
+
+@test "mutation: an [allow] token is stripped as a whole word only — a mailbox that merely ends in it is refused" {
+  # The unanchored strip this replaces left `dev` behind and the mailbox rule no
+  # longer matched, so an internal address ending in the public one shipped.
+  plant README.md 'escalate to devsupport@tracebloc.io'
+  guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[strings-refuse] needle '[A-Za-z0-9._%+-]+@tracebloc\.io' found in 1 staged line(s):"*"tree/README.md:2"* ]] || return 1
+}
+
+@test "an [allow] token matches case-insensitively, like the scan, and a sentence-ending dot is still a boundary" {
+  plant README.md 'Questions? Write to Support@Tracebloc.io.'
+  guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[forbidden-strings] clean ("* ]] || return 1
+}
+
+@test "mutation: a private needle supplied with --extra-forbidden joins the refuse tier, named by number only" {
+  printf '# private list\nplanted-tenant\n' >"$BATS_TEST_TMPDIR/tenants.txt"
+  plant client/templates/deploy.yaml '# for Planted-Tenant only'
+  guard --extra-forbidden "$BATS_TEST_TMPDIR/tenants.txt"
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[strings-refuse] private needle #1 found in 1 staged line(s):"*"tree/client/templates/deploy.yaml:2"* ]] || return 1
+  [[ "$output" == *"[forbidden-strings] 1 refuse-tier hit(s), 0 report-tier hit(s) counted (3 refuse + 3 report needle(s)"* ]] || return 1
+  # The private pattern is the identifier kept out of the public list; it must
+  # not surface in the log (teed into the public run summary) or in the report.
+  ! grep -qi 'planted-tenant' <<<"$output" || { echo "$output"; return 1; }
+  ! grep -qi 'planted-tenant' "$OUT/publish-guard-report.txt" || return 1
+}
+
+@test "an empty --extra-forbidden list is could-not-tell: the private needles were not supplied" {
+  printf '# nothing here\n\n' >"$BATS_TEST_TMPDIR/tenants.txt"
+  guard --extra-forbidden "$BATS_TEST_TMPDIR/tenants.txt"
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[forbidden-strings] COULD NOT TELL — extra forbidden list '"*"tenants.txt' is empty"* ]] || return 1
+}
+
+@test "a missing --extra-forbidden list is could-not-tell" {
+  guard --extra-forbidden "$BATS_TEST_TMPDIR/absent.txt"
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"[forbidden-strings] COULD NOT TELL — extra forbidden list '"*"absent.txt' is missing or unreadable"* ]] || return 1
+}
+
+@test "a refuse-tier needle inside a release asset is refused with the asset named" {
+  mkdir -p "$BATS_TEST_TMPDIR/assets"
+  printf '#!/bin/sh\n# bucket arn:aws:s3:::planted\n' >"$BATS_TEST_TMPDIR/assets/install.sh"
+  guard --assets "$BATS_TEST_TMPDIR/assets"
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[assets] staged 1 release asset(s):"*"    install.sh"* ]] || return 1
+  [[ "$output" == *"[strings-refuse] needle 'arn:aws:' found in 1 staged line(s):"*"assets/install.sh:2"* ]] || return 1
+}
+
+@test "a binary asset is opaque to the string scan and counted as such" {
+  mkdir -p "$BATS_TEST_TMPDIR/assets"
+  printf 'ELF\000\000arn:aws:x\000' >"$BATS_TEST_TMPDIR/assets/tracebloc-linux-amd64"
+  guard --assets "$BATS_TEST_TMPDIR/assets"
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"7 text file(s) scanned, 1 binary file(s) opaque to this scan"* ]] || return 1
+}
+
+# ── guard 3: forbidden strings — the report tier and --strict ─────────────────
+
+@test "a report-tier hit alone is counted, not refused: exit 0, per-needle total, most-hit files" {
+  plant README.md 'see backend#1234 for the rationale'
+  guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[forbidden-strings] [strings-report] needle 'backend#' found in 1 staged line(s) — counted, not refused (--strict refuses)"* ]] || return 1
+  [[ "$output" == *"[forbidden-strings] [strings-report] 1 hit(s) in 1 file(s); most-hit files:"* ]] || return 1
+  [[ "$output" == *"         1  tree/README.md"* ]] || return 1
+  [[ "$output" == *"[forbidden-strings] 0 refuse-tier hit(s), 1 report-tier hit(s) counted"* ]] || return 1
+  [[ "$output" != *"REFUSED"* ]] || return 1
+  [[ "$output" == *"publish-guard: OK — all 4 guards ran and passed"* ]] || return 1
+  # The matched text itself is never echoed; the full location list is in the report.
+  [[ "$output" != *"for the rationale"* ]] || return 1
+  grep -qF "[strings-report] needle 'backend#':" "$OUT/publish-guard-report.txt" || return 1
+  grep -qF "tree/README.md:2" "$OUT/publish-guard-report.txt" || return 1
+}
+
+@test "mutation: the same report-tier hit under --strict is refused, tier named" {
+  plant README.md 'see backend#1234 for the rationale'
+  guard --strict
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[forbidden-strings] REFUSED — [strings-report (strict)] needle 'backend#' found in 1 staged line(s):"* ]] || return 1
+  [[ "$output" == *"    tree/README.md:2"* ]] || return 1
+  [[ "$output" == *"[forbidden-strings] 0 refuse-tier hit(s), 1 report-tier hit(s) refused under --strict"* ]] || return 1
+  [[ "$output" == *"publish-guard: REFUSED — do not publish"* ]] || return 1
+}
+
+@test "--strict with no report-tier hit is still clean" {
+  guard --strict
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[forbidden-strings] clean (2 refuse + 3 report needle(s)"* ]] || return 1
+}
+
+@test "a non-production hostname is report-tier: counted, and refused under --strict" {
+  plant client/Chart.yaml '# points at dev-api.tracebloc.io'
+  guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[strings-report] needle 'dev-api\.tracebloc\.io' found in 1 staged line(s) — counted, not refused"* ]] || return 1
+  rm -rf "$OUT"
+  guard --strict
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"REFUSED — [strings-report (strict)] needle 'dev-api\.tracebloc\.io' found in 1 staged line(s):"*"tree/client/Chart.yaml:2"* ]] || return 1
+}
+
+@test "the most-hit table sums every report-tier needle per file, largest first, ten rows at most" {
+  plant README.md 'backend#1 and RFC-0001 on one line'
+  plant README.md 'backend#2 on another'
+  plant client/Chart.yaml '# backend#3'
+  guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[strings-report] needle 'backend#' found in 3 staged line(s)"* ]] || return 1
+  [[ "$output" == *"[strings-report] needle 'RFC-0' found in 1 staged line(s)"* ]] || return 1
+  # 3 + 1 hits, 2 files; README's two lines (three hits) outrank Chart.yaml's one.
+  [[ "$output" == *"[strings-report] 4 hit(s) in 2 file(s); most-hit files:"*"         3  tree/README.md"*"         1  tree/client/Chart.yaml"* ]] || { echo "$output"; return 1; }
+  # Eleven files, one hit each: the table stops at ten.
+  rm -rf "$OUT"
+  local i
+  for i in 01 02 03 04 05 06 07 08 09 10 11; do add_file "notes/n$i.md" "ref backend#$i"; done
+  commit
+  guard
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[strings-report] 15 hit(s) in 13 file(s); most-hit files:"* ]] || { echo "$output"; return 1; }
+  [ "$(printf '%s\n' "$output" | grep -cE '^ +[0-9]+  (tree|assets)/')" -eq 10 ] || { echo "$output"; return 1; }
+}
+
+@test "a refuse-tier hit and a report-tier hit in one run: refused, and the report tier still counted" {
+  plant README.md 'arn:aws:iam::000000000000:root — see backend#9'
+  guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"REFUSED — [strings-refuse] needle 'arn:aws:' found in 1 staged line(s):"* ]] || return 1
+  [[ "$output" == *"[strings-report] needle 'backend#' found in 1 staged line(s) — counted, not refused"* ]] || return 1
+  [[ "$output" == *"[forbidden-strings] 1 refuse-tier hit(s), 1 report-tier hit(s) counted"* ]] || return 1
+}
+
+# ── guard 3: the forbidden list itself ────────────────────────────────────────
+
+@test "no [strings-refuse] entries is could-not-tell, not clean (a guard with nothing to refuse)" {
+  printf '[paths]\ntests/\n[strings-report]\nbackend#\n' >"$SRC/.publish-forbidden"
+  guard
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[forbidden-strings] COULD NOT TELL — '"*"' has no [strings-refuse] entries — a guard with nothing to refuse is misconfigured"* ]] || return 1
+  # A present-but-empty section is the same finding.
+  rm -rf "$OUT"
+  printf '[paths]\ntests/\n[strings-refuse]\n# none yet\n[strings-report]\nbackend#\n' >"$SRC/.publish-forbidden"
+  guard
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"has no [strings-refuse] entries"* ]] || return 1
+}
+
+@test "an empty [strings-refuse] is judged before the private needles join it" {
+  printf '[paths]\ntests/\n[strings-report]\nbackend#\n' >"$SRC/.publish-forbidden"
+  printf 'planted-tenant\n' >"$BATS_TEST_TMPDIR/tenants.txt"
+  guard --extra-forbidden "$BATS_TEST_TMPDIR/tenants.txt"
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"has no [strings-refuse] entries"* ]] || return 1
+}
+
+@test "a needle listed in both string tiers is could-not-tell, the duplicate named" {
+  printf '[paths]\ntests/\n[strings-refuse]\narn:aws:\nbackend#\n[strings-report]\nbackend#\nRFC-0\n' >"$SRC/.publish-forbidden"
+  guard
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[forbidden-strings] COULD NOT TELL — '"*"' lists needle 'backend#' in both [strings-refuse] and [strings-report] — a needle has one tier"* ]] || return 1
+}
+
+@test "an unknown section header is could-not-tell for both scans that read the list" {
+  # The retired name is the likeliest misspelling; nothing under it may be read
+  # as a rule of the section before it.
+  printf '[paths]\ntests/\n[strings]\narn:aws:\n' >"$SRC/.publish-forbidden"
+  guard
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[forbidden-paths] COULD NOT TELL — '"*"' has an unknown section [strings] — the guard reads only [paths] [strings-refuse] [strings-report] [allow]"* ]] || return 1
+  [[ "$output" == *"[forbidden-strings] COULD NOT TELL — '"*"' has an unknown section [strings]"* ]] || return 1
+  # A header with a space or a case slip is a header too, refused by name rather
+  # than read as a needle of the section before it.
+  rm -rf "$OUT"
+  printf '[paths]\ntests/\n[strings-refuse]\narn:aws:\n[strings report]\nbackend#\n' >"$SRC/.publish-forbidden"
+  guard
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"has an unknown section [strings report]"* ]] || return 1
+  [[ "$output" != *"needle 'backend#'"* ]] || return 1
+}
+
+@test "a missing forbidden list is could-not-tell for both scans that read it" {
+  rm "$SRC/.publish-forbidden"
+  guard
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"[forbidden-paths] COULD NOT TELL — forbidden list '"*"' is missing or unreadable"* ]] || return 1
+  [[ "$output" == *"[forbidden-strings] COULD NOT TELL — forbidden list '"*"' is missing or unreadable"* ]] || return 1
+}
+
+# ── guard 1: allowlist fail-closed cases ──────────────────────────────────────
+
+@test "an allowlist with no include entries is could-not-tell" {
+  printf '# only comments\n\n' >"$SRC/.publish-include"
+  guard
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"[allowlist] COULD NOT TELL — allowlist '"*"' lists no include entries"* ]] || return 1
+}
+
+@test "a missing allowlist is could-not-tell" {
+  rm "$SRC/.publish-include"
+  guard
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"[allowlist] COULD NOT TELL — allowlist '"*"' is missing or unreadable"* ]] || return 1
+}
+
+@test "an allowlist that matches nothing is could-not-tell (a mirror with nothing in it)" {
+  write_include 'nothing-here/**'
+  guard
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"[allowlist] COULD NOT TELL — the allowlist matched none of the 14 tracked files"* ]] || return 1
+}
+
+@test "a symlink in the allowlisted set is could-not-tell" {
+  ln -s ../Makefile "$SRC/client/link"
+  git -C "$SRC" add client/link && commit
+  guard
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[allowlist] COULD NOT TELL — 'client/link' is a symlink"* ]] || return 1
+}
+
+@test "a non-empty --out is could-not-tell" {
+  mkdir -p "$OUT" && printf 'stale\n' >"$OUT/stale.txt"
+  guard
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"COULD NOT TELL — --out '"*"' is not empty"* ]] || return 1
+}
+
+@test "a source that is not a git work tree is could-not-tell" {
+  mkdir -p "$BATS_TEST_TMPDIR/plain"
+  cp "$SRC/.publish-include" "$SRC/.publish-forbidden" "$BATS_TEST_TMPDIR/plain/"
+  run bash "$GUARD" --source "$BATS_TEST_TMPDIR/plain" --out "$OUT"
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"[allowlist] COULD NOT TELL — git ls-files failed"* ]] || return 1
+}
+
+@test "an --assets directory with no files is could-not-tell" {
+  mkdir -p "$BATS_TEST_TMPDIR/assets"
+  guard --assets "$BATS_TEST_TMPDIR/assets"
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"COULD NOT TELL — --assets '"*"' holds no files"* ]] || return 1
+}
+
+# ── guard 4: gitleaks plumbing, then the real thing ───────────────────────────
+
+@test "a missing scanner is could-not-tell, never clean" {
+  PUBLISH_GUARD_GITLEAKS="$BATS_TEST_TMPDIR/no-such-gitleaks" guard
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"[gitleaks] COULD NOT TELL — scanner '"*"no-such-gitleaks' is not on PATH"* ]] || return 1
+  [[ "$output" == *"publish-guard: COULD NOT TELL — do not publish"* ]] || return 1
+}
+
+@test "a scanner that finds a secret refuses, and its redacted report is shown" {
+  GL_MODE=leak guard
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"[gitleaks] REFUSED — secrets detected in the staged tree:"* ]] || return 1
+  [[ "$output" == *"Finding: REDACTED"* ]] || return 1
+}
+
+@test "a scanner that crashes is could-not-tell (its exit code is not a verdict)" {
+  GL_MODE=crash guard
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"[gitleaks] COULD NOT TELL — scanner exited 1"* ]] || return 1
+}
+
+@test "the scanner is pointed at the staged tree, with --no-git and --redact" {
+  # The scanner's own output is shown on a refusal, which is when its argv is
+  # visible to this test. The guard resolves --out to an absolute path, so only
+  # the tail of the --source value is compared.
+  GL_MODE=leak guard
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"shim gitleaks ran: detect --no-git --redact --no-banner --exit-code 9 --source "*"/out"* ]] || { echo "$output"; return 1; }
+}
+
+@test "real gitleaks: a planted access-key-shaped string in a staged file is refused" {
+  command -v gitleaks >/dev/null 2>&1 || skip "gitleaks not on PATH (the publish workflow installs it; run locally with gitleaks installed)"
+  unset PUBLISH_GUARD_GITLEAKS
+  # Built at run time, so this file never carries a key-shaped literal.
+  local key
+  key="AKIA$(LC_ALL=C tr -dc 'A-Z2-7' "$STAGE/README.md"
+  printf 'license\n' >"$STAGE/LICENSE"
+  printf 'doc\n' >"$STAGE/docs/a.md"
+  git init -q --bare "$BARE"
+  # gh shim: records every argv line to GH_LOG; `release view` answers per
+  # GH_VIEW_RC / GH_VIEW_ERR; everything else succeeds.
+  cat >"$SHIM/gh" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' "$*" >>"${GH_LOG:?}"
+if [ "${1:-}" = release ] && [ "${2:-}" = view ]; then
+  printf '%s\n' "${GH_VIEW_ERR:-release not found}" >&2
+  exit "${GH_VIEW_RC:-1}"
+fi
+exit 0
+EOF
+  chmod +x "$SHIM/gh"
+  export GH_LOG="$BATS_TEST_TMPDIR/gh.log"
+  : >"$GH_LOG"
+}
+
+pub() { run bash "$PUB" "$@"; }
+tree() { pub tree --stage "$STAGE" --repo tracebloc/mirror --branch main --message "Publish v1.0.0" --remote "file://$BARE" "$@"; }
+mirror_files() { git -C "$BARE" ls-tree -r --name-only "$1" | sort | paste -sd' ' -; }
+
+# ── target ────────────────────────────────────────────────────────────────────
+
+@test "target: no mirror configured is refused — there is no default" {
+  pub target --mirror '' --source-repo tracebloc/client
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"REFUSED — no mirror repository is configured (MIRROR_REPO is unset)"* ]] || return 1
+}
+
+@test "target: the source repository itself is refused, case-insensitively" {
+  pub target --mirror client --source-repo tracebloc/client
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"REFUSED — mirror 'tracebloc/client' is this repository"* ]] || return 1
+  pub target --mirror Client --source-repo tracebloc/client
+  [ "$status" -eq 1 ] || return 1
+}
+
+@test "target: a name with characters a repository cannot have, or an OWNER/NAME, is refused" {
+  pub target --mirror 'cli mirror' --source-repo tracebloc/client
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"contains characters a repository name cannot"* ]] || return 1
+  pub target --mirror 'other/cli' --source-repo tracebloc/client
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"must be a bare repository name"* ]] || return 1
+}
+
+@test "target: a valid mirror prints OWNER/NAME in the source's organisation" {
+  pub target --mirror client-public --source-repo tracebloc/client
+  [ "$status" -eq 0 ] || return 1
+  [ "$output" = "tracebloc/client-public" ] || return 1
+}
+
+@test "target: a missing --source-repo is could-not-tell (the self-publish check needs it)" {
+  pub target --mirror client-public
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"COULD NOT TELL — target: --source-repo is required"* ]] || return 1
+}
+
+# --output: the workflow runs the publisher DIRECTLY and reads results from a
+# file, so a refusal's ::error:: line is on stdout where Actions annotates it —
+# captured through $(...) it would be swallowed by set -e (Bugbot on the PR).
+
+@test "target: --output writes repo= and name= for the workflow; stdout still names the mirror" {
+  local out="$BATS_TEST_TMPDIR/out"
+  pub target --mirror client-public --source-repo tracebloc/client --output "$out"
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$output" = "tracebloc/client-public" ] || return 1
+  [ "$(cat "$out")" = $'repo=tracebloc/client-public\nname=client-public' ] || { cat "$out"; return 1; }
+}
+
+@test "target: a refusal puts the ::error:: line on stdout and writes nothing to --output" {
+  local out="$BATS_TEST_TMPDIR/out"
+  pub target --mirror '' --source-repo tracebloc/client --output "$out"
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == "::error::publish-mirror: REFUSED — no mirror repository is configured"* ]] || { echo "$output"; return 1; }
+  [ ! -e "$out" ] || return 1
+  pub target --mirror client --source-repo tracebloc/client --output "$out"
+  [ "$status" -eq 1 ] || return 1
+  [ ! -e "$out" ] || return 1
+}
+
+# ── tree ──────────────────────────────────────────────────────────────────────
+
+@test "tree: the first publish starts the branch and the mirror holds exactly the stage" {
+  tree
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == pushed\ [0-9a-f]* ]] || return 1
+  [ "$(mirror_files main)" = "LICENSE README.md docs/a.md" ] || return 1
+  [ "$(git -C "$BARE" rev-list --count main)" -eq 1 ] || return 1
+}
+
+@test "tree: an identical stage is a no-op, reported as unchanged" {
+  tree; [ "$status" -eq 0 ] || return 1
+  tree
+  [ "$status" -eq 0 ] || return 1
+  [[ "$output" == unchanged\ [0-9a-f]* ]] || return 1
+  [ "$(git -C "$BARE" rev-list --count main)" -eq 1 ] || return 1
+}
+
+@test "tree: a later publish replaces the content — removed files vanish, history is appended" {
+  tree; [ "$status" -eq 0 ] || return 1
+  local first
+  first="$(git -C "$BARE" rev-parse main)"
+  rm "$STAGE/docs/a.md"; printf 'new\n' >"$STAGE/CHANGES.md"
+  tree
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$(mirror_files main)" = "CHANGES.md LICENSE README.md" ] || return 1
+  [ "$(git -C "$BARE" rev-list --count main)" -eq 2 ] || return 1
+  [ "$(git -C "$BARE" rev-parse main^)" = "$first" ] || return 1   # appended, not rewritten
+}
+
+@test "tree: a mirror branch with prior content not from this pipeline is replaced on top, not force-pushed over" {
+  local seed="$BATS_TEST_TMPDIR/seed"
+  git clone -q "file://$BARE" "$seed" 2>/dev/null
+  printf 'old\n' >"$seed/old.txt"
+  git -C "$seed" -c user.name=t -c user.email=t@example.invalid add old.txt
+  git -C "$seed" -c user.name=t -c user.email=t@example.invalid commit -q -m seed
+  git -C "$seed" push -q origin HEAD:refs/heads/main
+  local seeded
+  seeded="$(git -C "$BARE" rev-parse main)"
+  tree
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$(mirror_files main)" = "LICENSE README.md docs/a.md" ] || return 1
+  [ "$(git -C "$BARE" rev-parse main^)" = "$seeded" ] || return 1
+}
+
+@test "tree: a remote that does not answer is could-not-tell, not a fresh start" {
+  pub tree --stage "$STAGE" --repo tracebloc/mirror --branch main --message m --remote "file://$BATS_TEST_TMPDIR/no-such.git"
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"COULD NOT TELL — tree: the mirror remote did not answer"* ]] || return 1
+}
+
+@test "tree: an empty stage, or one that is a checkout, is could-not-tell" {
+  rm -r "$STAGE"; mkdir -p "$STAGE"
+  tree
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"holds no files"* ]] || return 1
+  mkdir -p "$STAGE/.git"; printf 'x\n' >"$STAGE/README.md"
+  tree
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"contains a .git entry"* ]] || return 1
+}
+
+@test "tree: --output writes result= and sha= (pushed, then unchanged); a refusal writes nothing and annotates stdout" {
+  local out="$BATS_TEST_TMPDIR/out"
+  tree --output "$out"
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [ "$(sed -n 1p "$out")" = "result=pushed" ] || { cat "$out"; return 1; }
+  [ "$(sed -n 2p "$out")" = "sha=$(git -C "$BARE" rev-parse main)" ] || { cat "$out"; return 1; }
+  rm "$out"
+  tree --output "$out"
+  [ "$status" -eq 0 ] || return 1
+  [ "$(sed -n 1p "$out")" = "result=unchanged" ] || { cat "$out"; return 1; }
+  [ "$(sed -n 2p "$out")" = "sha=$(git -C "$BARE" rev-parse main)" ] || return 1
+  rm "$out"
+  pub tree --stage "$STAGE" --repo tracebloc/mirror --branch main --message m --remote "file://$BATS_TEST_TMPDIR/no-such.git" --output "$out"
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == "::error::publish-mirror: COULD NOT TELL — tree: the mirror remote did not answer"* ]] || { echo "$output"; return 1; }
+  [ ! -e "$out" ] || return 1
+}
+
+@test "tree: an unwritable --output is could-not-tell — a result the caller never receives is not a publish" {
+  tree --output "$BATS_TEST_TMPDIR/no-such-dir/out"
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"COULD NOT TELL — could not write results to"* ]] || return 1
+}
+
+@test "tree: the script never forces a push" {
+  run grep -nE -- '--force|\+refs/|-f[[:space:]]' "$PUB"
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+}
+
+# ── release ───────────────────────────────────────────────────────────────────
+
+release() {
+  PATH="$SHIM:$PATH" pub release --tag v1.0.0 --repo tracebloc/mirror --target 0123456789abcdef0123456789abcdef01234567 --assets "$STAGE" --notes "$BATS_TEST_TMPDIR/notes.md" "$@"
+}
+
+@test "release: creates the tag at the target with every asset, no generated notes" {
+  printf 'Release notes\n' >"$BATS_TEST_TMPDIR/notes.md"
+  release
+  [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+  [[ "$output" == "released v1.0.0 on tracebloc/mirror at 0123456789abcdef0123456789abcdef01234567 with 2 asset(s)" ]] || return 1
+  grep -q '^release view v1.0.0 --repo tracebloc/mirror$' "$GH_LOG" || return 1
+  local create
+  create="$(grep '^release create' "$GH_LOG")"
+  [[ "$create" == "release create v1.0.0 --repo tracebloc/mirror --target 0123456789abcdef0123456789abcdef01234567 --title v1.0.0 --notes-file $BATS_TEST_TMPDIR/notes.md $STAGE/LICENSE $STAGE/README.md" ]] || { echo "$create"; return 1; }
+  [[ "$create" != *"--generate-notes"* ]] || return 1
+  [[ "$create" != *"--prerelease"* ]] || return 1
+}
+
+@test "release: --prerelease is passed through" {
+  printf 'notes\n' >"$BATS_TEST_TMPDIR/notes.md"
+  release --prerelease
+  [ "$status" -eq 0 ] || return 1
+  grep -q '^release create .* --prerelease ' "$GH_LOG" || return 1
+}
+
+@test "release: a tag that already exists on the mirror is refused, never overwritten" {
+  printf 'notes\n' >"$BATS_TEST_TMPDIR/notes.md"
+  GH_VIEW_RC=0 release
+  [ "$status" -eq 1 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"REFUSED — release: 'v1.0.0' already exists on 'tracebloc/mirror'"* ]] || return 1
+  run grep -c '^release create' "$GH_LOG"
+  [ "$output" = "0" ] || return 1
+}
+
+@test "release: a view failure that is not 'not found' is could-not-tell" {
+  printf 'notes\n' >"$BATS_TEST_TMPDIR/notes.md"
+  GH_VIEW_RC=1 GH_VIEW_ERR='HTTP 401: Bad credentials' release
+  [ "$status" -eq 2 ] || { echo "$output"; return 1; }
+  [[ "$output" == *"COULD NOT TELL — release: could not read releases of 'tracebloc/mirror'"* ]] || return 1
+  run grep -c '^release create' "$GH_LOG"
+  [ "$output" = "0" ] || return 1
+}
+
+@test "release: a malformed tag is refused; a short sha, empty notes or no assets are could-not-tell" {
+  printf 'notes\n' >"$BATS_TEST_TMPDIR/notes.md"
+  PATH="$SHIM:$PATH" pub release --tag main --repo tracebloc/mirror --target 0123456789abcdef0123456789abcdef01234567 --assets "$STAGE" --notes "$BATS_TEST_TMPDIR/notes.md"
+  [ "$status" -eq 1 ] || return 1
+  [[ "$output" == *"'main' is not a release tag"* ]] || return 1
+  PATH="$SHIM:$PATH" pub release --tag v1.0.0 --repo tracebloc/mirror --target abc123 --assets "$STAGE" --notes "$BATS_TEST_TMPDIR/notes.md"
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"is not a full commit sha"* ]] || return 1
+  : >"$BATS_TEST_TMPDIR/empty.md"
+  PATH="$SHIM:$PATH" pub release --tag v1.0.0 --repo tracebloc/mirror --target 0123456789abcdef0123456789abcdef01234567 --assets "$STAGE" --notes "$BATS_TEST_TMPDIR/empty.md"
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"notes file"*"is missing or empty"* ]] || return 1
+  mkdir -p "$BATS_TEST_TMPDIR/none"
+  PATH="$SHIM:$PATH" pub release --tag v1.0.0 --repo tracebloc/mirror --target 0123456789abcdef0123456789abcdef01234567 --assets "$BATS_TEST_TMPDIR/none" --notes "$BATS_TEST_TMPDIR/notes.md"
+  [ "$status" -eq 2 ] || return 1
+  [[ "$output" == *"holds no files"* ]] || return 1
+  run grep -c '^release create' "$GH_LOG"
+  [ "$output" = "0" ] || return 1
+}