From d4bc639094cddd59a21fba61d7b20ba367ed14b6 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Tue, 8 Sep 2026 20:32:33 -0700 Subject: [PATCH] refactor(toolbox): hooks over copies, one Actions pin, reproducible render Extension is a hook, never a copy: the engine's smoke ends in a smoke-mirror hook, and pipeline and plan-pipeline ship with the engine. The failure path is one verb, failed, which sync runs in the pipeline's container and the reusable workflow runs from outside for a run the runner cut off. The workflow passes the dispatch input as data, chains the caller's own file name, installs the 1Password CLI only for a mirror with an op.env, and checks this repository out at its own commit for the toolbox action, so the workflow pin is the one pin. render mounts an empty directory over .run, so a var read from it at parse time renders the same from any working tree. clean deletes .run, staging and the Taskfile cache and nothing else. reconcile spares directory keys only when INDEX is set; unset, the suffix degenerated to "/" and spared every key ending in one. pull and push are the state's two moves, MAX_BATCHES and RETRY_BASE are read once, GPGCHECK is the one signature check, and verify works from STAGING by cd, since a path-based include's dir: is joined onto the include's directory even when absolute. docker/lock.py is the one lock reader. The examples' .taskrc.yml key is cache-expiry; expiry is not a key task knows, and the cache was never used. offline now covers prepare and verify over fixtures/tree, a signed subtree whose throwaway key's fingerprint the example pins. --- .github/actions/toolbox/action.yml | 19 +- .github/workflows/check.yml | 9 +- .github/workflows/sync.yml | 31 +- CLAUDE.md | 43 +- README.md | 139 +- docker/lock.py | 15 + docker/proton.Dockerfile | 4 +- docker/rsync.Dockerfile | 6 +- .../plans/2026-09-08-toolbox-library.md | 1122 ----------------- engines/rsync.yml | 142 ++- examples/proton/.taskrc.yml | 2 +- examples/proton/Taskfile.yml | 9 +- examples/proton/render.txt | 2 +- examples/rsync/.taskrc.yml | 2 +- examples/rsync/Taskfile.yml | 54 +- examples/rsync/fixtures/run-tl/applied.txt | 0 examples/rsync/fixtures/run-tl/batch-0001.txt | 7 + examples/rsync/fixtures/run-tl/batch-bad.txt | 1 + examples/rsync/fixtures/run-tl/changed.txt | 7 + examples/rsync/fixtures/run-tl/deleted.txt | 0 .../rsync/fixtures/tree/tl/archive/bad.tar.xz | 1 + .../fixtures/tree/tl/archive/foo.doc.tar.xz | 1 + .../fixtures/tree/tl/archive/foo.r1.tar.xz | 1 + .../rsync/fixtures/tree/tl/archive/foo.tar.xz | 1 + .../fixtures/tree/tl/tlpkg/gpg/pubring.gpg | Bin 0 -> 275 bytes .../fixtures/tree/tl/tlpkg/texlive.tlpdb | 12 + .../tree/tl/tlpkg/texlive.tlpdb.sha512 | 1 + .../tree/tl/tlpkg/texlive.tlpdb.sha512.asc | 8 + .../fixtures/tree/tl/tlpkg/texlive.tlpdb.xz | Bin 0 -> 368 bytes examples/rsync/render.txt | 42 +- toolbox.yml | 48 +- 31 files changed, 403 insertions(+), 1326 deletions(-) create mode 100755 docker/lock.py delete mode 100644 docs/superpowers/plans/2026-09-08-toolbox-library.md create mode 100644 examples/rsync/fixtures/run-tl/applied.txt create mode 100644 examples/rsync/fixtures/run-tl/batch-0001.txt create mode 100644 examples/rsync/fixtures/run-tl/batch-bad.txt create mode 100644 examples/rsync/fixtures/run-tl/changed.txt create mode 100644 examples/rsync/fixtures/run-tl/deleted.txt create mode 100644 examples/rsync/fixtures/tree/tl/archive/bad.tar.xz create mode 100644 examples/rsync/fixtures/tree/tl/archive/foo.doc.tar.xz create mode 100644 examples/rsync/fixtures/tree/tl/archive/foo.r1.tar.xz create mode 100644 examples/rsync/fixtures/tree/tl/archive/foo.tar.xz create mode 100644 examples/rsync/fixtures/tree/tl/tlpkg/gpg/pubring.gpg create mode 100644 examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb create mode 100644 examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.sha512 create mode 100644 examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.sha512.asc create mode 100644 examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.xz diff --git a/.github/actions/toolbox/action.yml b/.github/actions/toolbox/action.yml index 817ec2e..2f448b7 100644 --- a/.github/actions/toolbox/action.yml +++ b/.github/actions/toolbox/action.yml @@ -1,21 +1,28 @@ name: toolbox -description: Install go-task and the 1Password CLI at the versions in katoptra/lib's toolchain.lock.toml +description: Install go-task, and the 1Password CLI unless op is false, at the versions in katoptra/lib's toolchain.lock.toml +inputs: + op: + description: Install the 1Password CLI; a mirror without an op.env has no use for it + default: 'true' runs: using: composite steps: - shell: bash env: - # GitHub checks the action's repository out beside the workflow, so the lock at - # the repository root is three levels up from this file. - LOCK: ${{ github.action_path }}/../../../toolchain.lock.toml + # GitHub checks the action's repository out beside the workflow, so the repository + # root, with the lock and the reader in docker/, is three levels up from this file. + LIB: ${{ github.action_path }}/../../.. + OP: ${{ inputs.op }} run: | set -eu - lock() { python3 -c 'import functools,sys,tomllib; x=tomllib.load(open(sys.argv[2],"rb")); print(functools.reduce(lambda d,k: d[k], sys.argv[1].split("."), x))' "$1" "$LOCK"; } + lock() { python3 "$LIB/docker/lock.py" "$1" "$LIB/toolchain.lock.toml"; } arch="$(dpkg --print-architecture)" curl -fsSL "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz echo "$(lock task.linux_${arch}.sha256) /tmp/task.tgz" | sha256sum -c - sudo tar -xzf /tmp/task.tgz -C /usr/local/bin task + task --version + test "$OP" = true || exit 0 curl -fsSL "$(lock op.base_url)/$(lock op.linux_${arch}.archive)" -o /tmp/op.zip echo "$(lock op.linux_${arch}.sha256) /tmp/op.zip" | sha256sum -c - sudo unzip -q -o /tmp/op.zip op -d /usr/local/bin - task --version && op --version + op --version diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b03c457..e98947f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -10,7 +10,10 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # The action is this repository's, pinned to a release commit: a mirror whose Actions - # policy requires a full SHA on every uses: rejects a tag here too. Dependabot bumps it. - - uses: katoptra/lib/.github/actions/toolbox@a80d19d76766d058e9e10190277525f9d452b21d # v1.0.1 + # This repository at the commit the caller pinned this workflow to, so the toolbox + # action and the lock it reads are the release the workflow is: one pin, not two. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: {repository: katoptra/lib, ref: '${{ github.job_workflow_sha }}', path: .lib} + - uses: ./.lib/.github/actions/toolbox + with: {op: 'false'} # a dry run needs no secrets - run: task check diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 8a71968..53556ed 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -1,7 +1,7 @@ -# Reusable: one mirror, one run. The caller is a ten-line workflow_dispatch that +# Reusable: one mirror, one run. The caller is a workflow_dispatch of a few lines that # jshvn/dispatch triggers; it passes `vars` through and inherits its repository secrets. # 1Password is read once per run: one `op run` resolves op.env around the whole pipeline, -# fail ping included. A mirror without an op.env runs on its repository secrets instead, +# failure path included. A mirror without an op.env runs on its repository secrets instead, # exported into the sync step's environment and crossing into the container by the names # its PASS var lists. name: sync @@ -30,26 +30,41 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # The action is this repository's, pinned to a release commit: a mirror whose Actions - # policy requires a full SHA on every uses: rejects a tag here too. Dependabot bumps it. - - uses: katoptra/lib/.github/actions/toolbox@a80d19d76766d058e9e10190277525f9d452b21d # v1.0.1 + # This repository at the commit the caller pinned this workflow to, so the toolbox + # action and the lock it reads are the release the workflow is: one pin, not two. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: {repository: katoptra/lib, ref: '${{ github.job_workflow_sha }}', path: .lib} + - uses: ./.lib/.github/actions/toolbox + with: {op: "${{ hashFiles('op.env') != '' }}"} # a mirror without an op.env never calls op # df is the host's: the image and the batches share the runner's disk. - run: df -h . && task image - name: task sync + id: sync env: SECRETS: ${{ toJSON(secrets) }} + VARS: ${{ inputs.vars }} # Every inherited secret becomes an environment variable of this step, by name, in # memory: no file, no command line, and GitHub masks the values in the log. With an - # op.env the pipeline reads 1Password and these are unused. + # op.env the pipeline reads 1Password and these are unused. VARS is data the shell + # splits into words, never an expression spliced into the script. run: | eval "$(printf '%s' "$SECRETS" | jq -r 'to_entries[] | "export \(.key)=\(.value | @sh)"')" - task sync -- ${{ inputs.vars }} + task sync -- $VARS + # The pipeline's own failure path runs inside its container, so a run the runner cut + # off (the timeout, a cancellation, an image that never pulled) reported and pinged + # nothing. This is that path from outside. An ordinary failure has already run it. + - name: The run was cut off + if: always() && steps.sync.outcome != 'success' && steps.sync.outcome != 'failure' + env: + HEALTHCHECK_URL: ${{ secrets.HEALTHCHECK_URL }} + run: task op -- task failed - name: Chain the next run if: success() env: GH_TOKEN: ${{ github.token }} + # The caller's own file name, whatever it is; workflow_ref is owner/repo/.github/workflows/@. run: | if [ -f .run/chain ]; then - gh workflow run sync.yml --ref "$GITHUB_REF_NAME" + f="${GITHUB_WORKFLOW_REF##*/}"; gh workflow run "${f%%@*}" --ref "$GITHUB_REF_NAME" echo "chained: next run queued" >> "$GITHUB_STEP_SUMMARY" fi diff --git a/CLAUDE.md b/CLAUDE.md index 1b320bd..86ebd92 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,14 +10,32 @@ The toolbox every katoptra mirror includes by URL. Read `README.md` for the cont mirror's root value. - Verb names are reserved across host and container. `plan` is the host-side read-only run; an engine's batch planner is `split`. The hooks, `prepare`, `verify`, `index`, - `smoke`, `report-engine` in the engine and `report-mirror` in the toolbox, are the - only verbs a mirror redefines, each excluded on the include that defines it. - `report-engine` exists in both files, so an engine consumer excludes it on the - toolbox include. + `smoke`, `smoke-mirror`, `report-engine` in the engine and `report-mirror` in the + toolbox, are the only verbs a mirror redefines, each excluded on the include that + defines it; `pipeline` and `plan-pipeline` come from the engine and a mirror excludes + them only for another order. `report-engine` exists in both files, so an engine + consumer excludes it on the toolbox include. Extension is a hook, never a copy: a + verb that needs more than the engine does gets a `-mirror` hook the engine calls last. - A mirror's root var shadows a command-line `KEY=value` inside an included verb, so an - engine tunable is an inline default and never a root var of the example. -- Every tool in an image comes from `toolchain.lock.toml` with a checksum. The AWS CLI - zip is the marked exception. + engine tunable is an inline default and never a root var of the example. A var of + another name that reads the tunable, `BATCHES_MAX` from `MAX_BATCHES`, still sees the + command line (verified), so each default is spelled once. +- A called task sees none of its caller's call vars. A hook a verb calls with an + overridable `RUN` or `STAGING` gets them passed on explicitly, as `smoke` does. +- A global var that reads another var is rendered once, from the root's and the command + line's values, never from a call var: `GPGCHECK` takes the fingerprint as an awk `-v` + at each use, where a call var `TL_KEY` is visible. +- No `dir:` on an engine verb. In a flattened include read by path, task joins even an + absolute `dir:` onto the include's directory (verified, 3.53.1), so `verify` starts + each command with `cd {{.STAGING}}` instead. +- `render` mounts an empty directory over `.run`, so a `sh:` var read from it at parse + time renders the same from any working tree. A verb that reads `.run` at parse time + must tolerate an empty one. +- `.taskrc.yml` keys are `trusted-hosts` and `cache-expiry`; task ignores a key it does + not know, silently, and refetches on every invocation. +- Every tool in an image comes from `toolchain.lock.toml` with a checksum, read through + `docker/lock.py`, the one reader the Dockerfiles and the toolbox action share. The AWS + CLI zip is the marked exception. - Images set `TASK_REMOTE_OFFLINE=1`. Inside a run the include resolves from the mirror's `.task/remote` cache, bind-mounted with the repo, never from the network. - `run` and `render` mount the repository's git top level at `/work` and set the working @@ -27,7 +45,11 @@ The toolbox every katoptra mirror includes by URL. Read `README.md` for the cont so the container engine's own progress lines never reach `render.txt`. - Inside a `sh:` var, `printf -- '-e %s'` prints dashes: task's built-in shell takes the `--` as the format. Use `printf '%s %s ' -e "$v"`. -- Actions pinned to a full SHA with the version in a trailing comment. +- Actions pinned to a full SHA with the version in a trailing comment. A mirror pins the + two reusable workflows that way; each checks this repository out at its own commit + (`github.job_workflow_sha`) for the toolbox action and the lock, so a workflow pin is + the one pin. The include and the image float at `v1` by design: moving that tag is + the rollout. ## Verifying a change @@ -37,4 +59,7 @@ cd examples/proton && task image-build && task run -- task tools && task check ``` A verb change updates the `render.txt` files via `task render-update`; `offline` is -the engine's own check over `examples/rsync/fixtures/`. +the engine's own check over `examples/rsync/fixtures/`: the list diff over `run-root` +and `run-empty`, `retry`'s exit codes, and `prepare` and `verify` over `tree/`, a signed +subtree whose tlpdb is signed by a throwaway key pinned in the example. Regenerate the +tree with a new key only to change its shape; the private half was never kept. diff --git a/README.md b/README.md index 9c7079b..cc5ac6b 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ flowchart TB tf["Taskfile.yml
root vars: SOURCE, BUCKET, HOST
pipeline, plan-pipeline
overrides, if any"] opv["op.env
op:// references"] rt["render.txt
the committed dry run"] - cw["sync.yml, check.yml
ten-line callers"] + cw["sync.yml, check.yml
callers of lib's"] end subgraph lib["katoptra/lib, pinned to v1"] tb["toolbox.yml
menu, image, run, op, sync, plan
render, check, clock, ping"] @@ -28,7 +28,7 @@ flowchart TB tf -- "includes, flattened" --> tb tf -- "includes, flattened" --> en tb -- "runs pipeline inside" --> im - cw -- "uses @v1" --> wf + cw -- "uses @release SHA" --> wf wf -- "task sync / task check" --> tb ``` @@ -37,7 +37,7 @@ flowchart TB | Toolbox | `toolbox.yml` | How a run starts, is contained, resolves secrets, is rendered, checked and reported | Never; it is included as is | | Engine | `engines/.yml` | How bytes move for one transport: rsync to R2, Proton Drive, HTTPS | It lists a verb under `excludes:` and defines its own | | Image | `docker/`, `toolchain.lock.toml` | Every tool a run needs, pinned by checksum | Never; it names one in `IMAGE` | -| Workflows | `.github/workflows`, `.github/actions/toolbox` | How Actions installs the tools and runs `task sync` and `task check` | Never; its callers are ten lines | +| Workflows | `.github/workflows`, `.github/actions/toolbox` | How Actions installs the tools and runs `task sync` and `task check` | Never; its callers are a few lines each | | Mirror | The mirror's own repository | Identity, pipeline order, the exceptions | Always; this is all it holds | ## What a run looks like @@ -61,9 +61,11 @@ sequenceDiagram C->>C: engine verbs: list, diff, split, batches, reconcile ... C->>C: report, then ping alt pipeline failed - C->>C: report STATUS=failed, then ping-fail + C->>C: failed: report STATUS=failed, then ping-fail + else the runner cut the run off + W->>C: task op -- task failed else .run/chain exists - W->>D: gh workflow run sync.yml + W->>D: gh workflow run, the caller's own file end ``` @@ -74,10 +76,13 @@ Three things the diagram hides: plus `HEALTHCHECK_URL`, `GITHUB_STEP_SUMMARY` and `GITHUB_RUN_ID` always, so no value ever appears on a command line or in a log. - **1Password is read once per run.** One `op run` wraps the whole pipeline, and the - fail ping runs inside the same container, so a failed run costs no second read. A - mirror without an `op.env` runs on its repository secrets: the reusable workflow - exports every inherited secret into the sync step's environment by name, and the ones - the mirror lists in `PASS` cross into the container. + failure path, `failed`, runs inside the same container, so a failed run costs no + second read. The one exception is a run the runner cut off, by the timeout or a + cancellation, which never reached its own failure path: the workflow runs `failed` + again from outside, one more read on a rare day. A mirror without an `op.env` runs + on its repository secrets: the reusable workflow exports every inherited secret into + the sync step's environment by name, the ones the mirror lists in `PASS` cross into + the container, and the 1Password CLI is not installed at all. - **Nothing inside the container reaches the network for task itself.** The image sets `TASK_REMOTE_OFFLINE=1` and the mirror's `.task/remote` cache rides in with the repo. - **`plan` is `sync` with the read-only half.** It runs `plan-pipeline` instead of @@ -89,10 +94,9 @@ Three things the diagram hides: | Name | Kind | Meaning | |---|---|---| -| `pipeline` | task | The full run, in order, inside the image. `sync` calls it. | -| `plan-pipeline` | task | The read-only half, inside the image. `plan` calls it. | | `SOURCE`, `BUCKET`, `HOST` | root vars | The mirror's identity. Engines read them; the library never defaults them. | | `NAME`, `DESC`, `IMAGE` | include vars | The menu's title and line, and the image the run happens inside. | +| `pipeline`, `plan-pipeline` | tasks | The full run and its read-only half, in order, inside the image; `sync` and `plan` call them. An engine supplies both; a mirror with no engine defines them. | ### What a mirror may define @@ -100,6 +104,7 @@ Three things the diagram hides: |---|---|---| | `PASS` | include var | Host environment names that cross into the container beside the ones in `op.env`, for a pipeline that reads its environment. Task vars are not environment: they go after `--`, as `task sync -- MAX_BATCHES=8`. | | `report-mirror` | task | The mirror's rows of the run summary, after the toolbox's and the engine's. Excluded on the toolbox include. | +| `smoke-mirror` | task | The mirror's own read-back checks, after the engine's sample. Excluded on the engine include. | | `MENU` | include var | Extra lines for the menu, one per mirror-specific verb. | | `LIB_DIR` | include var | Where `image-build` finds `docker/`; defaults to `../lib`. | | `op.env` | file | `op://` references, one per secret. Absent means the environment is already resolved: on a laptop, whatever is exported; in Actions, the repository's secrets, crossing by the names in `PASS`. | @@ -117,13 +122,13 @@ call and the menu documents. | `plan` | `op -- task plan-pipeline`: the read-only half | | `check` | `render`, then diff against `render.txt` | | `render-update` | `render`, then accept it as `render.txt` | -| `render` | `task --dry --force pipeline` inside the image, saved to `.run/render.txt` | +| `render` | `task --dry --force pipeline` inside the image, over an empty `.run`, saved to `.run/render.txt` | | `run -- ` | Anything inside the image, with the repo at `/work` | | `op -- ` | `run`, wrapped in `op run --env-file=op.env` when the file exists | | `image` | Pull `IMAGE`; a no-op while it exists locally | | `image-build` | Build `IMAGE` from `LIB_DIR/docker/.Dockerfile` | | `image-clean` | Remove `IMAGE` | -| `clean` | Delete every ignored file | +| `clean` | Delete `.run`, `staging` and the Taskfile cache; nothing else | Inside the image. A mirror or engine may override these when it keeps its own. @@ -134,9 +139,11 @@ Inside the image. A mirror or engine may override these when it keeps its own. | `report-engine`, `report-mirror` | Hooks: no-ops here, the engine's and the mirror's rows | | `ping` | GET `HEALTHCHECK_URL`; skipped when unset | | `ping-fail` | GET `HEALTHCHECK_URL/fail`; skipped when unset | +| `failed` | The failure path: `report STATUS=failed`, then `ping-fail` | -`sync` runs `pipeline`, and on a failure `report STATUS=failed` then `ping-fail`, all in -the one container. One table, three layers: the toolbox's rows (when the run started +`sync` runs `pipeline`, and on a failure `failed`, all in the one container; the +workflow runs `failed` from outside only for a run the runner cut off. One table, three +layers: the toolbox's rows (when the run started and how long it took, the image, whether the next run is queued), the engine's, the mirror's, each a `| Label | value |` line appended to the same file. Every row tolerates a missing file, because the report also runs after a failed pipeline. @@ -150,6 +157,7 @@ an rsync upstream into an S3 bucket, as a list diff and never a local tree. | Verb | Does | |---|---| +| `pipeline`, `plan-pipeline` | The verbs below in order, and the read-only half of them: `clock`, `list`, `state`, `diff`, `split` | | `list` | `rsync --list-only` of `SOURCE`, through `FILTER`, normalised to `.run/upstream.txt` as `path TAB size TAB mtime`, byte-sorted; a listing under `LIST_FLOOR` lines stops the run | | `state` | Fetch `.state/applied.txt.xz` from the bucket; a missing one asks `rebuild` | | `rebuild` | List the bucket and make the state exactly what it holds at upstream's sizes | @@ -161,14 +169,17 @@ an rsync upstream into an S3 bucket, as a list diff and never a local tree. | `delete` | Remove the keys upstream dropped, 1,000 per call, once every batch has landed, and drop them from the state | | `reconcile` | In the run that starts in hour 03 UTC, or with `RECONCILE=true`: rebuild the state, then delete what neither upstream, `OWN` nor the state's own directories own | | `index` | Hook. Nothing here; a mirror that draws directory pages or a landing page replaces it | -| `smoke` | Hook. A sample of the run's keys read back through `HOST`, sizes against the listing, and the tlpdb sha512 when `TL` is set | +| `smoke` | A sample of the run's keys read back through `HOST`, sizes against the listing, the tlpdb sha512 when `TL` is set, then `smoke-mirror` | +| `smoke-mirror` | Hook. Nothing here; a mirror with more to read back defines it | | `report-engine` | Hook. The engine's rows of the run summary | | `retry` | Run a command, retrying rsync's transport exit codes with backoff; 24 is a success | -`normalise`, `batch`, `fetch`, `publish`, `merge`, `checkpoint` and `remove` are the -verbs those call. A mirror replaces a hook by listing it under `excludes:` on the -engine include and defining its own; `report-engine` is defined in the toolbox too, as -a no-op, so an engine consumer's toolbox include excludes it. +`normalise`, `pull`, `push`, `batch`, `fetch`, `publish`, `merge`, `checkpoint` and +`remove` are the verbs those call; `pull` and `push` move an xz-compressed key between +the bucket and `.run`, and a mirror's own verb may call them. A mirror replaces a hook by +listing it under `excludes:` on the engine include and defining its own; `report-engine` +is defined in the toolbox too, as a no-op, so an engine consumer's toolbox include +excludes it. A mirror sets `SOURCE`, `BUCKET` and `HOST` in its root vars, always. Everything else has an inline default in the engine, and a mirror sets only what differs: @@ -184,11 +195,12 @@ has an inline default in the engine, and a mirror sets only what differs: | `TL`, `TL_KEY` | empty | A signed TeX Live subtree and the fingerprint that signs it; empty, no signature checks | | `FILTER` | empty | rsync filter arguments that narrow the listing, for a mirror of a subtree | | `OWN` | empty | Bucket-root keys the mirror owns, space separated; `reconcile` never deletes them | -| `INDEX` | empty | The key suffix of the directory pages a mirror's `index` draws, which `reconcile` spares | +| `INDEX` | empty | The key suffix of the directory pages a mirror's `index` draws; set, `reconcile` spares those pages and every bare directory of the state, their second key | A var the mirror puts in its root `vars:` is fixed for every run: inside an included verb a root value shadows a `KEY=value` from the command line. One the mirror leaves to -its default is the run's to set, `task sync -- MAX_BATCHES=8 RECONCILE=true`. +its default is the run's to set, `task sync -- MAX_BATCHES=8 RECONCILE=true`. So a root +var that restates a default is worse than none: it silences the command line. A mirror of one signed subtree, shaped like tlnet: @@ -217,28 +229,25 @@ includes: NAME: tlnet DESC: a daily mirror of TeX Live's tlnet at https://tlnet.ijosh.com/ IMAGE: ghcr.io/katoptra/toolbox:rsync-v1 - PASS: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_ENDPOINT_URL AWS_REGION + PASS: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_ENDPOINT_URL rsync: taskfile: https://raw.githubusercontent.com/katoptra/lib/v1/engines/rsync.yml flatten: true excludes: [index] tasks: - pipeline: - cmds: [{task: clock}, {task: list}, {task: state}, {task: rebuild}, {task: diff}, {task: split}, {task: prepare}, {task: batches}, {task: delete}, {task: reconcile}, {task: index}, {task: smoke}, {task: report}, {task: ping}] - plan-pipeline: - cmds: [{task: clock}, {task: list}, {task: state}, {task: diff}, {task: split}] index: desc: The landing page, dated today, at the bucket root cmds: - sed "s/DATE/$(date -u +%Y-%m-%d)/" site/index.html | aws s3 cp --content-type text/html --cache-control no-cache - s3://{{.BUCKET}}/index.html report-mirror: + internal: true cmds: - 'echo "| Landing page | index.html, dated $(date -u +%Y-%m-%d) |" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"' ``` `SOURCE` is the archive root and `FILTER` narrows the listing to the subtree, so every key keeps its `systems/texlive/tlnet/` prefix. `OWN` keeps `reconcile` off the landing -page, and `index` is the one hook the mirror fills. +page, `index` is the one hook the mirror fills, and the pipeline is the engine's. ### The bucket is the mirror; the state is a cache @@ -264,20 +273,23 @@ A duplicate without `excludes` is a parse error, on purpose. ```yaml includes: - toolbox: - taskfile: https://raw.githubusercontent.com/katoptra/lib/v1/toolbox.yml + rsync: + taskfile: https://raw.githubusercontent.com/katoptra/lib/v1/engines/rsync.yml flatten: true - excludes: [clock] # a Python engine keeps its own clock - vars: {NAME: photos, DESC: ..., IMAGE: ghcr.io/katoptra/toolbox:proton-v1} + excludes: [index] # the mirror draws its own landing page tasks: - clock: - cmds: [python -c 'import time; print(int(time.time()))'] + index: + cmds: [...] ``` The rules that keep this honest: - Keep the name and the meaning. A `verify` that does something other than verify is a new verb with its own name, listed in `MENU`. +- Extend through a hook, never a copy. A mirror that needs the engine's `smoke` and + more defines `smoke-mirror`, which `smoke` runs last, rather than excluding `smoke` + and pasting it: the copy stops following releases the day it is made. A mirror with a + step to add beside `clock` adds a step to its pipeline, not a `clock` of its own. - The replacement reads the mirror's root vars, the same as the original did. - Override engine verbs and the in-container toolbox verbs. Do not override the host side: `sync`, `plan`, `check` and `run` are what the workflows and the menu promise. @@ -309,15 +321,16 @@ and a `list` that reads an index instead of `rsync --list-only`. ## Using it in a mirror -A mirror repository holds six things: +A mirror repository holds seven things: ``` -Taskfile.yml root vars, the two includes, pipeline, plan-pipeline, overrides +Taskfile.yml root vars, the two includes, overrides .taskrc.yml trusts raw.githubusercontent.com, refetches at most hourly op.env op:// references only (optional) render.txt the committed dry run -.github/workflows/sync.yml calls katoptra/lib/.github/workflows/sync.yml@v1 -.github/workflows/check.yml calls katoptra/lib/.github/workflows/check.yml@v1 +.github/workflows/sync.yml calls katoptra/lib/.github/workflows/sync.yml at a release commit +.github/workflows/check.yml calls katoptra/lib/.github/workflows/check.yml at a release commit +.github/dependabot.yml bumps the two calls on a lib release ``` ```yaml @@ -336,26 +349,24 @@ includes: NAME: ctan DESC: an hourly mirror of CTAN at https://ctan.ijosh.com/ IMAGE: ghcr.io/katoptra/toolbox:rsync-v1 - PASS: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_ENDPOINT_URL AWS_REGION + PASS: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_ENDPOINT_URL rsync: taskfile: https://raw.githubusercontent.com/katoptra/lib/v1/engines/rsync.yml flatten: true -tasks: - pipeline: - cmds: [{task: clock}, {task: list}, {task: state}, {task: rebuild}, {task: diff}, {task: split}, {task: prepare}, {task: batches}, {task: delete}, {task: reconcile}, {task: index}, {task: smoke}, {task: report}, {task: ping}] - plan-pipeline: - cmds: [{task: clock}, {task: list}, {task: state}, {task: diff}, {task: split}] ``` -A mirror on another transport includes that transport's engine instead, and a mirror -with a Python pipeline of its own includes the toolbox alone and defines the verbs its -`pipeline` names. +That is a whole mirror: the engine supplies `pipeline` and `plan-pipeline`. The three +`AWS_*` names are the secrets, on a laptop exported and in Actions the repository's; +`AWS_REGION` is `auto` in the image. A mirror on another transport includes that +transport's engine instead, and a mirror with a Python pipeline of its own includes the +toolbox alone and defines the two pipeline verbs from the toolbox's `clock`, `report` +and `ping` and its own steps. ```yaml # .taskrc.yml remote: trusted-hosts: [raw.githubusercontent.com] - expiry: 1h + cache-expiry: 1h ``` ```yaml @@ -368,7 +379,7 @@ permissions: {contents: read, actions: write} # the chain step; a called workf concurrency: {group: sync, cancel-in-progress: false} jobs: sync: - uses: katoptra/lib/.github/workflows/sync.yml@v1 + uses: katoptra/lib/.github/workflows/sync.yml@04f7901cf1aa7551bc43db5ff801a225a189c42a # v1.0.2 with: {vars: '${{ inputs.vars }}'} secrets: inherit ``` @@ -379,12 +390,26 @@ name: check on: {pull_request: {}} jobs: check: - uses: katoptra/lib/.github/workflows/check.yml@v1 + uses: katoptra/lib/.github/workflows/check.yml@04f7901cf1aa7551bc43db5ff801a225a189c42a # v1.0.2 +``` + +```yaml +# .github/dependabot.yml +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: {interval: weekly} + groups: {actions: {patterns: ['*']}} ``` -A repository whose Actions policy requires a full commit SHA on every `uses:` pins the -two calls to a release commit instead, `@04f7901cf1aa7551bc43db5ff801a225a189c42a # v1.0.2`, -and Dependabot bumps them; the include and the image still float at `v1`. +Two pins, one policy. The workflow calls are pinned to a release commit with the +version in a trailing comment, as an Actions policy that requires a full SHA on every +`uses:` demands, and Dependabot bumps them; each reusable workflow checks this +repository out at that same commit for the toolbox action and the lock, so the workflow +pin is the only Actions-side pin. The include and the image float at `v1`, on purpose: +moving that tag is how a verb or a tool reaches every mirror on its next run. `timeout- +minutes` goes in `with:` only when it differs from the workflow's default of 355. Then `task render-update` once, commit `render.txt`, and `task check` from then on. Every pull request that changes what the mirror executes shows up as a diff in @@ -400,9 +425,11 @@ $ cd ../proton && task image-build && task check `task image-build` builds the image the example names from `docker/`. `task check` renders the example's pipeline inside it and diffs against `render.txt`; `task run -- -task offline` runs the engine's verbs that need no bucket over `fixtures/`. Change a -verb, run `task render-update` in each example, and the diff in the pull request is the -review. `CONTRIBUTING.md` has the three rules this repository adds to the org's. +task offline` runs the engine's verbs that need no bucket over `fixtures/`: the list +diff, `retry`, and `prepare` and `verify` over a signed subtree whose key is the +example's own. Change a verb, run `task render-update` in each example, and the diff in +the pull request is the review. `CONTRIBUTING.md` has the three rules this repository +adds to the org's. ## Releasing diff --git a/docker/lock.py b/docker/lock.py new file mode 100755 index 0000000..d690b22 --- /dev/null +++ b/docker/lock.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +"""lock KEY [FILE]: one value of toolchain.lock.toml, KEY dotted (task.linux_arm64.sha256). + +The one reader the Dockerfiles and the toolbox action share; FILE defaults to where the +images keep the lock. +""" + +import functools +import sys +import tomllib + +path = sys.argv[2] if len(sys.argv) > 2 else "/etc/toolchain.lock.toml" +with open(path, "rb") as f: + lock = tomllib.load(f) +print(functools.reduce(lambda d, k: d[k], sys.argv[1].split("."), lock)) diff --git a/docker/proton.Dockerfile b/docker/proton.Dockerfile index 625b838..b2e74ec 100644 --- a/docker/proton.Dockerfile +++ b/docker/proton.Dockerfile @@ -6,14 +6,14 @@ FROM python:3.13.15-slim-bookworm@sha256:ed86c82274b3c69b52fb5820f358f0bd7df0b60 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \ && rm -rf /var/lib/apt/lists/* -COPY toolchain.lock.toml /tmp/lock.toml +COPY toolchain.lock.toml /etc/toolchain.lock.toml +COPY docker/lock.py /usr/local/bin/lock # Architecture from the image itself: BuildKit sets TARGETARCH, Apple container does # not, and a defaulted arg would install amd64 binaries into an arm64 image. RUN set -eu; \ arch="$(dpkg --print-architecture)"; \ case "$arch" in amd64|arm64) ;; *) echo "unsupported architecture: $arch" >&2; exit 1 ;; esac; \ - lock() { python -c "import functools,sys,tomllib; x=tomllib.load(open('/tmp/lock.toml','rb')); print(functools.reduce(lambda d,k: d[k], sys.argv[1].split('.'), x))" "$1"; }; \ curl -fsSL "$(lock proton_drive_cli.linux_${arch}.url)" -o /tmp/proton-drive; \ echo "$(lock proton_drive_cli.linux_${arch}.sha512) /tmp/proton-drive" | sha512sum -c -; \ install -m 0755 /tmp/proton-drive /usr/local/bin/proton-drive; \ diff --git a/docker/rsync.Dockerfile b/docker/rsync.Dockerfile index c93be7f..7193574 100644 --- a/docker/rsync.Dockerfile +++ b/docker/rsync.Dockerfile @@ -6,14 +6,14 @@ FROM ubuntu:24.04@sha256:33ceb71981b602c1a7443a53469e4dba065f7503eab3078a2d7a57a RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl unzip python3 \ && rm -rf /var/lib/apt/lists/* -COPY toolchain.lock.toml /tmp/lock.toml +COPY toolchain.lock.toml /etc/toolchain.lock.toml +COPY docker/lock.py /usr/local/bin/lock # Architecture from the image itself: BuildKit sets TARGETARCH, Apple container does # not, and a defaulted arg would install amd64 binaries into an arm64 image. RUN set -eu; \ arch="$(dpkg --print-architecture)"; \ case "$arch" in amd64) m=x86_64 ;; arm64) m=aarch64 ;; *) echo "unsupported architecture: $arch" >&2; exit 1 ;; esac; \ - lock() { python3 -c "import functools,sys,tomllib; x=tomllib.load(open('/tmp/lock.toml','rb')); print(functools.reduce(lambda d,k: d[k], sys.argv[1].split('.'), x))" "$1"; }; \ curl -fsSL "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz; \ echo "$(lock task.linux_${arch}.sha256) /tmp/task.tgz" | sha256sum -c -; \ tar -xzf /tmp/task.tgz -C /tmp task; install -m 0755 /tmp/task /usr/local/bin/task; \ @@ -35,12 +35,12 @@ COPY --from=fetch /usr/local/bin/task /usr/local/bin/task COPY --from=fetch /usr/local/aws-cli /usr/local/aws-cli RUN ln -s /usr/local/aws-cli/v2/current/bin/aws /usr/local/bin/aws COPY toolchain.lock.toml /etc/toolchain.lock.toml +COPY docker/lock.py /usr/local/bin/lock # ponytail: botocore ships 432 service models and the pipelines call one, so all but s3, # sts and the data-root *.json stay behind. Ceiling: an `aws` call to any other service # dies on a model lookup; add it to the keep list in the fetch stage. RUN set -eu; \ - lock() { python3 -c "import functools,sys,tomllib; x=tomllib.load(open('/etc/toolchain.lock.toml','rb')); print(functools.reduce(lambda d,k: d[k], sys.argv[1].split('.'), x))" "$1"; }; \ task --version | grep -q "$(lock task.version)"; \ aws --version | grep -q "aws-cli/$(lock awscli.version)"; \ aws s3 ls s3://x --no-sign-request --endpoint-url http://127.0.0.1:1 2>&1 | grep -qiE 'connect|endpoint|refused'; \ diff --git a/docs/superpowers/plans/2026-09-08-toolbox-library.md b/docs/superpowers/plans/2026-09-08-toolbox-library.md deleted file mode 100644 index 76161c8..0000000 --- a/docs/superpowers/plans/2026-09-08-toolbox-library.md +++ /dev/null @@ -1,1122 +0,0 @@ -# Toolbox Library Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Seed `katoptra/lib` so that a mirror repository can include `toolbox.yml` by URL and get the menu, the container, the secrets model, the dry-run check and the workflows without copying anything. - -**Architecture:** One go-task include (`toolbox.yml`) carries every host-side and in-container primitive. Two container images built from one lock file carry the tools. Two reusable workflows and one composite action carry CI. Two example consumers inside the repo prove the include resolves and the images run, and their committed `render.txt` is the check. - -**Tech Stack:** go-task 3.53.1 (remote includes, flatten, excludes, `.taskrc.yml`), Apple `container` or Docker, GitHub Actions, GHCR, 1Password CLI, AWS CLI v2, proton-drive CLI, age. - -**Spec:** `docs/superpowers/specs/2026-09-08-toolbox-library-design.md` - -## Global Constraints - -- Default branch `main`. Commit format `(): `, imperative, under 75 characters. No attribution trailers. -- No emojis outside markdown. No em-dashes in prose. -- Every action pinned to a full commit SHA with the version in a trailing comment. -- No library `vars:` default for anything a mirror owns; defaults go inline as `{{.X | default N}}`. -- Host-side verbs and in-container verbs share one namespace; the names in the spec's two tables are reserved. -- Every tool in an image comes from `toolchain.lock.toml` and is checksum-verified, except the AWS CLI installer zip, which is HTTPS and version-pinned only, marked with a `ponytail:` comment. -- The one check is `task check` in each example: render inside the image, diff against `render.txt`. -- Local clone lives at `~/Git/katoptra/lib`. Nothing in this plan pushes to GitHub; creating the remote repository, pushing and tagging `v1.0.0` are the hand-off. - ---- - -### Task 1: Repository skeleton - -**Files:** -- Create: `LICENSE`, `.gitignore`, `.github/dependabot.yml`, `README.md` (stub, finished in Task 8) - -**Interfaces:** -- Produces: the repo layout every later task writes into. - -- [x] **Step 1: Copy the MIT license from the site repo and write the ignore file** - -```sh -cd ~/Git/katoptra/lib -cp ~/Git/katoptra/site/LICENSE LICENSE -cat > .gitignore <<'EOF' -.run/ -.task/ -EOF -``` - -- [x] **Step 2: Write dependabot for actions** - -`.github/dependabot.yml`: - -```yaml -version: 2 -updates: - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly - groups: - actions: - patterns: ['*'] -``` - -- [x] **Step 3: Write the README stub** - -```markdown -# lib - -The toolbox every katoptra mirror includes. Finished in Task 8. -``` - -- [x] **Step 4: Verify and commit** - -Run: `git add -A && git status --short` -Expected: four files staged. - -```sh -git commit -m "chore: seed the repository" -``` - ---- - -### Task 2: The toolchain lock - -**Files:** -- Create: `toolchain.lock.toml` - -**Interfaces:** -- Produces: tables `task`, `op`, `awscli`, `ubuntu`, `python`, `proton_drive_cli`, `age`, each with `version`, and per-arch tables `linux_amd64` and `linux_arm64` carrying `sha256` or `sha512`. Read by both Dockerfiles (Task 3) and the composite action (Task 5) with the same `lock` helper. - -- [x] **Step 1: Write the lock** - -```toml -# One lock for the family. Each Dockerfile and the toolbox action read the tables they -# need. Bumping a tool means bumping its version and every checksum beneath it. - -[ubuntu] -image = "ubuntu:24.04" -# pinned 2026-08-26: `container image pull ubuntu:24.04`, then the first digest -# `container image inspect ubuntu:24.04` prints. -digest = "sha256:33ceb71981b602c1a7443a53469e4dba065f7503eab3078a2d7a57a2ab987517" - -[python] -version = "3.13.15" -image = "python:3.13.15-slim-bookworm" -digest = "sha256:ed86c82274b3c69b52fb5820f358f0bd7df0b603332063cb5c6e32bd220c3e6e" - -[python.packages] -requests = "2.34.2" -certifi = "2026.7.22" -charset-normalizer = "3.5.1" -idna = "3.19" -urllib3 = "2.7.0" -boto3 = "1.43.88" -botocore = "1.43.88" -s3transfer = "0.19.2" -jmespath = "1.1.0" -python-dateutil = "2.9.0.post0" -six = "1.17.0" - -[python.test_packages] -pytest = "8.4.2" -ruff = "0.16.4" -iniconfig = "2.3.0" -packaging = "26.3" -pluggy = "1.6.0" -pygments = "2.21.0" - -[task] -version = "3.53.1" -base_url = "https://github.com/go-task/task/releases/download/v3.53.1" - -[task.linux_amd64] -archive = "task_linux_amd64.tar.gz" -sha256 = "a54a408f6861ff921f6e87774180db31bacd8c1e7c944ca696db9fea49a82fc7" - -[task.linux_arm64] -archive = "task_linux_arm64.tar.gz" -sha256 = "e3ad19101493a0112e1f22ae8ccc54bf03e533b1076a0ca1e6c782a09ad2e588" - -[op] -version = "2.39.0" -base_url = "https://cache.agilebits.com/dist/1P/op2/pkg/v2.39.0" - -[op.linux_amd64] -archive = "op_linux_amd64_v2.39.0.zip" -sha256 = "6fba7f376b6c6dec49f41b06408930a43ad064cce103c6a2ce5b3d0413a86434" - -[op.linux_arm64] -archive = "op_linux_arm64_v2.39.0.zip" -sha256 = "829baeff1c07e055cfa132031b1d9f2282ccdf5076258e482caf2fda70aea5d0" - -[awscli] -# ponytail: the installer zip carries a PGP signature this lock does not record; HTTPS -# and the version pin are the guarantee. Ceiling: a poisoned CDN. Upgrade path: import -# AWS's public key in the fetch stage and gpg --verify the zip. -version = "2.36.24" -url = "https://awscli.amazonaws.com/awscli-exe-linux-{arch}-{version}.zip" - -[proton_drive_cli] -version = "0.8.0" - -[proton_drive_cli.linux_amd64] -url = "https://proton.me/download/drive/cli/0.8.0/linux-x64/proton-drive" -sha512 = "cf61c2688c45e1055d8add6221d9471a5a5b64bf3bcdb86460f5cb18414596cc4df3cdb6627c9097c94bec32a3c9915ada3211ef2ae5be33c46ebbc996ccaa28" - -[proton_drive_cli.linux_arm64] -url = "https://proton.me/download/drive/cli/0.8.0/linux-arm64/proton-drive" -sha512 = "27a1aec1d2095fd4a1a81e1d47cd1f9fd4901bd579ffe50342d15e2e52078d6e8b2dddcf58a4a386438dc7562017778be26c1ba62399f901ae82c7430e2140a3" - -[age] -version = "1.2.1" -base_url = "https://github.com/FiloSottile/age/releases/download/v1.2.1" - -[age.linux_amd64] -archive = "age-v1.2.1-linux-amd64.tar.gz" -sha256 = "7df45a6cc87d4da11cc03a539a7470c15b1041ab2b396af088fe9990f7c79d50" - -[age.linux_arm64] -archive = "age-v1.2.1-linux-arm64.tar.gz" -sha256 = "57fd79a7ece5fe501f351b9dd51a82fbee1ea8db65a8839db17f5c080245e99f" -``` - -- [x] **Step 2: Verify it parses** - -Run: `python3 -c 'import tomllib; d = tomllib.load(open("toolchain.lock.toml","rb")); print(sorted(d))'` -Expected: `['age', 'awscli', 'op', 'proton_drive_cli', 'python', 'task', 'ubuntu']` - -- [x] **Step 3: Commit** - -```sh -git add toolchain.lock.toml -git commit -m "feat(lock): pin every tool the images and the action install" -``` - ---- - -### Task 3: The two images - -**Files:** -- Create: `docker/rsync.Dockerfile`, `docker/proton.Dockerfile` - -**Interfaces:** -- Consumes: `toolchain.lock.toml` (Task 2), copied to `/etc/toolchain.lock.toml` in each image. -- Produces: images that `toolbox.yml` (Task 4) runs with `-v :/work -w /work`. Both export `TASK_REMOTE_OFFLINE=1`, `AWS_REGION=auto`, `WORKDIR /work`, and have `task`, `curl` on PATH. - -- [x] **Step 1: Write `docker/rsync.Dockerfile`** - -```dockerfile -# syntax=docker/dockerfile:1.7 -# The rsync toolbox: what every rsync-to-R2 mirror runs inside, on a laptop and in -# Actions alike. The repo is bind-mounted at /work. Base pinned by digest; every tool -# comes from toolchain.lock.toml at the repo root, which is the build context. -FROM ubuntu:24.04@sha256:33ceb71981b602c1a7443a53469e4dba065f7503eab3078a2d7a57a2ab987517 AS fetch - -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl unzip python3 \ - && rm -rf /var/lib/apt/lists/* -COPY toolchain.lock.toml /tmp/lock.toml - -# Architecture from the image itself: BuildKit sets TARGETARCH, Apple container does -# not, and a defaulted arg would install amd64 binaries into an arm64 image. -RUN set -eu; \ - arch="$(dpkg --print-architecture)"; \ - case "$arch" in amd64|arm64) ;; *) echo "unsupported architecture: $arch" >&2; exit 1 ;; esac; \ - lock() { python3 -c "import functools,sys,tomllib; x=tomllib.load(open('/tmp/lock.toml','rb')); print(functools.reduce(lambda d,k: d[k], sys.argv[1].split('.'), x))" "$1"; }; \ - curl -fsSL "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz; \ - echo "$(lock task.linux_${arch}.sha256) /tmp/task.tgz" | sha256sum -c -; \ - tar -xzf /tmp/task.tgz -C /tmp task; install -m 0755 /tmp/task /usr/local/bin/task; \ - case "$arch" in amd64) m=x86_64 ;; arm64) m=aarch64 ;; esac; \ - url="$(lock awscli.url)"; url="${url/\{arch\}/$m}"; url="${url/\{version\}/$(lock awscli.version)}"; \ - curl -fsSL "$url" -o /tmp/awscli.zip; \ - cd /tmp && unzip -q awscli.zip && ./aws/install; \ - d=/usr/local/aws-cli/v2/current/dist/awscli; \ - mkdir /tmp/keep; for k in s3 sts; do mv "$d/botocore/data/$k" /tmp/keep/; done; \ - mv "$d/botocore/data"/*.json /tmp/keep/; rm -rf "$d/botocore/data"; mv /tmp/keep "$d/botocore/data"; \ - rm -rf "$d/examples" "$d/topics" "$d/data/ac.index" - -FROM ubuntu:24.04@sha256:33ceb71981b602c1a7443a53469e4dba065f7503eab3078a2d7a57a2ab987517 AS toolbox - -# perl carries shasum. Each RUN deletes its own scratch: a layer keeps what it leaves. -RUN apt-get update && apt-get install -y --no-install-recommends \ - rsync gnupg xz-utils curl ca-certificates perl python3 \ - && rm -rf /var/lib/apt/lists/* -COPY --from=fetch /usr/local/bin/task /usr/local/bin/task -COPY --from=fetch /usr/local/aws-cli /usr/local/aws-cli -RUN ln -s /usr/local/aws-cli/v2/current/bin/aws /usr/local/bin/aws -COPY toolchain.lock.toml /etc/toolchain.lock.toml - -# ponytail: botocore ships 432 service models and the pipelines call one, so all but s3, -# sts and the data-root *.json stay behind. Ceiling: an `aws` call to any other service -# dies on a model lookup; add it to the keep list in the fetch stage. -RUN set -eu; \ - lock() { python3 -c "import functools,sys,tomllib; x=tomllib.load(open('/etc/toolchain.lock.toml','rb')); print(functools.reduce(lambda d,k: d[k], sys.argv[1].split('.'), x))" "$1"; }; \ - task --version | grep -q "$(lock task.version)"; \ - aws --version | grep -q "aws-cli/$(lock awscli.version)"; \ - aws s3 ls s3://x --no-sign-request --endpoint-url http://127.0.0.1:1 2>&1 | grep -qiE 'connect|endpoint|refused'; \ - rsync --version | head -1; gpgv --version | head -1; xz --version | head -1; shasum --version - -# Inside a run there is no network for Taskfiles: the mirror's .task/remote cache rides -# in with the bind mount. R2 has one region; the value is a literal, not a secret. -ENV TASK_REMOTE_OFFLINE=1 \ - AWS_REGION=auto -WORKDIR /work -``` - -- [x] **Step 2: Write `docker/proton.Dockerfile`** - -```dockerfile -# syntax=docker/dockerfile:1.7 -# The proton toolbox: Python plus the Proton Drive CLI and age, for the mirrors whose -# engine is a Python package. The repo is bind-mounted at /work; PYTHONPATH finds its -# src/. Base pinned by digest; every tool from toolchain.lock.toml at the build context. -FROM python:3.13.15-slim-bookworm@sha256:ed86c82274b3c69b52fb5820f358f0bd7df0b603332063cb5c6e32bd220c3e6e AS fetch - -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \ - && rm -rf /var/lib/apt/lists/* -COPY toolchain.lock.toml /tmp/lock.toml - -RUN set -eu; \ - arch="$(dpkg --print-architecture)"; \ - case "$arch" in amd64|arm64) ;; *) echo "unsupported architecture: $arch" >&2; exit 1 ;; esac; \ - lock() { python -c "import functools,sys,tomllib; x=tomllib.load(open('/tmp/lock.toml','rb')); print(functools.reduce(lambda d,k: d[k], sys.argv[1].split('.'), x))" "$1"; }; \ - curl -fsSL "$(lock proton_drive_cli.linux_${arch}.url)" -o /tmp/proton-drive; \ - echo "$(lock proton_drive_cli.linux_${arch}.sha512) /tmp/proton-drive" | sha512sum -c -; \ - install -m 0755 /tmp/proton-drive /usr/local/bin/proton-drive; \ - curl -fsSL "$(lock age.base_url)/$(lock age.linux_${arch}.archive)" -o /tmp/age.tgz; \ - echo "$(lock age.linux_${arch}.sha256) /tmp/age.tgz" | sha256sum -c -; \ - tar -xzf /tmp/age.tgz -C /tmp; install -m 0755 /tmp/age/age /tmp/age/age-keygen /usr/local/bin/; \ - curl -fsSL "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz; \ - echo "$(lock task.linux_${arch}.sha256) /tmp/task.tgz" | sha256sum -c -; \ - tar -xzf /tmp/task.tgz -C /tmp task; install -m 0755 /tmp/task /usr/local/bin/task - -FROM python:3.13.15-slim-bookworm@sha256:ed86c82274b3c69b52fb5820f358f0bd7df0b603332063cb5c6e32bd220c3e6e AS toolbox - -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \ - && rm -rf /var/lib/apt/lists/* -COPY --from=fetch /usr/local/bin/proton-drive /usr/local/bin/age /usr/local/bin/age-keygen /usr/local/bin/task /usr/local/bin/ -COPY toolchain.lock.toml /etc/toolchain.lock.toml - -RUN python - <<'PY' -import subprocess, tomllib -lock = tomllib.load(open("/etc/toolchain.lock.toml", "rb")) -pins = [f"{k}=={v}" for t in ("packages", "test_packages") for k, v in lock["python"][t].items()] -subprocess.check_call(["pip", "install", "--no-cache-dir", "--no-deps", *pins]) -PY - -RUN python - <<'PY' -import importlib.metadata, platform, subprocess, tomllib -lock = tomllib.load(open("/etc/toolchain.lock.toml", "rb")) -assert platform.python_version() == lock["python"]["version"] -for table in ("packages", "test_packages"): - for name, version in lock["python"][table].items(): - assert importlib.metadata.version(name) == version, name -first = lambda argv: subprocess.check_output(argv, text=True).splitlines()[0] -assert "@" + lock["proton_drive_cli"]["version"] in first(["proton-drive", "version"]) -assert lock["age"]["version"] in first(["age", "--version"]) -assert lock["task"]["version"] in subprocess.check_output(["task", "--version"], text=True) -PY - -ENV PYTHONUNBUFFERED=1 \ - PYTHONPATH=/work/src \ - TASK_REMOTE_OFFLINE=1 \ - AWS_REGION=auto -WORKDIR /work -``` - -- [x] **Step 3: Build both images locally on the arm64 laptop** - -Run: - -```sh -cd ~/Git/katoptra/lib -container build -t ghcr.io/katoptra/toolbox:rsync-dev -f docker/rsync.Dockerfile . -container build -t ghcr.io/katoptra/toolbox:proton-dev -f docker/proton.Dockerfile . -``` - -Expected: both builds finish; the self-test RUN stages print the tool versions and exit 0. Substitute `docker build` when Docker is the engine. - -- [x] **Step 4: Prove the offline env var is set** - -Run: `container run --rm ghcr.io/katoptra/toolbox:rsync-dev sh -c 'echo $TASK_REMOTE_OFFLINE $AWS_REGION; task --version'` -Expected: `1 auto` then `Task version: v3.53.1`. - -- [x] **Step 5: Commit** - -```sh -git add docker/ -git commit -m "feat(docker): rsync and proton toolbox images from the lock" -``` - ---- - -### Task 4: toolbox.yml and the two examples - -**Files:** -- Create: `toolbox.yml` -- Create: `examples/rsync/Taskfile.yml`, `examples/rsync/.taskrc.yml`, `examples/rsync/op.env`, `examples/rsync/render.txt` -- Create: `examples/proton/Taskfile.yml`, `examples/proton/.taskrc.yml`, `examples/proton/op.env`, `examples/proton/render.txt` - -**Interfaces:** -- Consumes: images tagged as `IMAGE` (Task 3). -- Produces: every verb in the spec's two tables, by exact name. Consumers must define `pipeline` and `plan-pipeline`. Include vars: `NAME`, `DESC`, `IMAGE`, optional `PASS` (space-separated env names), optional `MENU` (extra menu lines), optional `LIB_DIR` (for `image-build`). - -- [x] **Step 1: Write `toolbox.yml`** - -```yaml -# The toolbox: every verb a katoptra mirror has whatever moves its bytes. A mirror -# includes this file flattened, passes NAME, DESC and IMAGE on the include, keeps its -# identity in its own root vars, and defines two verbs: pipeline and plan-pipeline, -# both of which run inside IMAGE. A verb listed under the include's excludes: and -# defined in the mirror replaces the one here. -# -# Rules this file keeps: no vars: default for anything a mirror owns (it would shadow -# the mirror's value; defaults go inline), and host-side and in-container verbs share -# one namespace, so the names below are reserved. -version: '3' - -vars: - RUN: '{{.ROOT_DIR}}/.run' - # The container runtime, not the sync engine: Apple container when its daemon is up, - # else Docker. ENGINE=docker forces Docker. - ENGINE: - sh: | - if [ -n "$ENGINE" ]; then echo "$ENGINE" - elif command -v container >/dev/null 2>&1 && container system status >/dev/null 2>&1; then echo container - else echo docker; fi - # Names cross into the container; values never appear on a command line. op.env - # names, the mirror's PASS list, and what the Actions job summary needs. - PASS_ENV: - sh: | - { test -f {{.ROOT_DIR}}/op.env && grep -v '^#' {{.ROOT_DIR}}/op.env | sed -n 's/=.*//p'; true; } | sed 's/^/-e /' | tr '\n' ' ' - for v in {{.PASS}} GITHUB_STEP_SUMMARY GITHUB_RUN_ID HEALTHCHECK_URL; do printf '%s %s ' -e "$v"; done - # The image variant is the tag up to its version: ghcr.io/katoptra/toolbox:rsync-v1 -> rsync. - VARIANT: - sh: echo "{{.IMAGE}}" | sed 's/.*://; s/-v[0-9].*//; s/-dev$//' - # The repository's top level rides in at /work and the Taskfile's directory is the - # working directory. For a mirror the two are the same; for the examples in - # katoptra/lib the include one level up must be reachable. - TOP: - sh: git -C {{.ROOT_DIR}} rev-parse --show-toplevel 2>/dev/null || echo {{.ROOT_DIR}} - PREFIX: - sh: git -C {{.ROOT_DIR}} rev-parse --show-prefix 2>/dev/null || true - -tasks: - default: - desc: Print the menu - silent: true - # ponytail: hand-maintained listing -- go-task can neither group nor annotate, so a - # new verb means a line below. `task --list` stays the generated view. - cmds: - - | - b=$(printf '\033[1m'); c=$(printf '\033[1;36m'); d=$(printf '\033[2m'); r=$(printf '\033[0m') - printf '%s\n' \ - "" \ - "${b}{{.NAME}}${r} ${d}{{.DESC}}${r}" \ - "" \ - "${c}pipeline -- needs the vault (op.env) or the secrets in your environment${r}" \ - " task sync One run: task pipeline inside the toolbox" \ - " task plan The read-only half: task plan-pipeline inside the toolbox" \ - "" \ - "${c}checks -- offline, no credentials${r}" \ - " task check Render every command of the pipeline and diff it against render.txt" \ - " task render-update Accept the current render as render.txt" \ - " task --list The pipeline's own verbs" \ - "" \ - "${c}toolbox -- {{.ENGINE}}, image {{.IMAGE}}${r}" \ - " task image Pull it (a no-op while it exists locally)" \ - " task image-build Build it from a local checkout of katoptra/lib (LIB_DIR=../lib)" \ - " task image-clean Remove it, so the next task pulls or builds again" \ - " task run -- Run anything inside it, with this repo at /work" \ - " task op -- Same, with secrets resolved by name from op.env" \ - " task clean Delete .run, the Taskfile cache and every other ignored file" \ - {{.MENU}} \ - "" - - # ---- toolbox ---- - image: - desc: Pull the toolbox image - run: once - status: ['{{.ENGINE}} image inspect {{.IMAGE}} >/dev/null 2>&1'] - cmds: - - '{{.ENGINE}} pull {{.IMAGE}}' - - image-build: - desc: 'Build the toolbox image from a local katoptra/lib checkout: task image-build LIB_DIR=../lib' - cmds: - - '{{.ENGINE}} build -t {{.IMAGE}} -f {{.LIB_DIR | default "../lib"}}/docker/{{.VARIANT}}.Dockerfile {{.LIB_DIR | default "../lib"}}' - - image-clean: - desc: Remove the toolbox image - cmds: - - '{{.ENGINE}} image rm {{.IMAGE}}' - - run: - desc: 'Run a command inside the toolbox with the repo at /work: task run -- task --list' - deps: [image] - # Neither -i nor -t: Apple container leaves the host terminal non-blocking after a - # run that attaches stdin or a tty. Output still streams and Ctrl-C still stops it. - # GITHUB_STEP_SUMMARY is a host path, mounted at the very path its value names. - # Each argument re-quoted, so task run -- sh -c 'a; b' arrives intact. - cmds: - - >- - {{.ENGINE}} run --rm - --user $(id -u):$(id -g) - -v "{{.TOP}}":/work -w /work/{{.PREFIX}} - ${GITHUB_STEP_SUMMARY:+-v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY"} - -e HOME=/tmp {{.PASS_ENV}} - {{.IMAGE}} {{range .CLI_ARGS_LIST}}{{shellQuote .}} {{end}} - - op: - desc: 'Run a command inside the toolbox with secrets from 1Password: task op -- task pipeline' - cmds: - # No op.env means the secrets are already in the environment (GitHub secrets, say). - - | - if test -f op.env; then op run --env-file=op.env -- task run -- {{range .CLI_ARGS_LIST}}{{shellQuote .}} {{end}} - else task run -- {{range .CLI_ARGS_LIST}}{{shellQuote .}} {{end}}; fi - - clean: - desc: Delete every file git ignores - cmds: - - git clean -fdX - - # ---- pipeline ---- - sync: - desc: One run, inside the toolbox - cmds: - - task: op - vars: {CLI_ARGS: task pipeline} - - plan: - desc: The read-only half, inside the toolbox - cmds: - - task: op - vars: {CLI_ARGS: task plan-pipeline} - - # ---- checks ---- - render: - desc: Render every command of the pipeline inside the toolbox, run none, save to .run/render.txt - deps: [image] - # Not via `run`: the output is the artifact, so it is captured, not streamed, and a - # dry run needs no secrets, so no names cross into the container. task writes the - # rendered commands to stderr; the redirect inside the container folds them into - # stdout, so the engine's own progress lines on the host's stderr stay out of the file. - cmds: - - mkdir -p {{.RUN}} - - >- - {{.ENGINE}} run --rm --user $(id -u):$(id -g) -v "{{.TOP}}":/work -w /work/{{.PREFIX}} - -e HOME=/tmp {{.IMAGE}} sh -c 'task --dry --force pipeline 2>&1' > {{.RUN}}/render.txt - - cat {{.RUN}}/render.txt - - check: - desc: render, then diff against the committed render.txt - cmds: - - task: render - - 'diff -u render.txt {{.RUN}}/render.txt && echo "check: render matches render.txt"' - - render-update: - desc: render, then accept it as render.txt - cmds: - - task: render - - cp {{.RUN}}/render.txt render.txt - - # ---- inside the toolbox ---- - clock: - desc: Record the run's start as "epoch UTC-hour weekday" in .run/start.txt - cmds: - - mkdir -p {{.RUN}} && date -u '+%s %H %u' > {{.RUN}}/start.txt - - ping: - desc: Dead man's switch; healthchecks.io emails when its grace passes without this (skipped if HEALTHCHECK_URL is unset) - cmds: - - test -z "$HEALTHCHECK_URL" || curl -fsS -m 10 --retry 3 -o /dev/null "$HEALTHCHECK_URL" - - ping-fail: - desc: Tell healthchecks.io the run failed (skipped if HEALTHCHECK_URL is unset) - cmds: - - test -z "$HEALTHCHECK_URL" || curl -fsS -m 10 --retry 3 -o /dev/null "$HEALTHCHECK_URL/fail" -``` - -- [x] **Step 2: Write the rsync example** - -`examples/rsync/Taskfile.yml`: - -```yaml -# A consumer of the toolbox with the rsync image. Inside this repo the include is a -# path; a mirror includes https://raw.githubusercontent.com/katoptra/lib/v1/toolbox.yml. -version: '3' -vars: - SOURCE: rsync://rsync.example.org/pub/ - BUCKET: example - HOST: example.ijosh.com -includes: - toolbox: - taskfile: ../../toolbox.yml - flatten: true - vars: - NAME: example-rsync - DESC: the rsync toolbox, exercised - IMAGE: ghcr.io/katoptra/toolbox:rsync-dev - LIB_DIR: ../.. - PASS: SEED MAX_BATCHES -tasks: - pipeline: - cmds: - - {task: clock} - - {task: tools} - - {task: list} - - {task: ping} - plan-pipeline: - cmds: - - {task: clock} - - {task: tools} - - {task: list} - tools: - desc: Every tool the rsync image promises, by version - cmds: - - rsync --version | head -1 - - aws --version - - gpgv --version | head -1 - - xz --version | head -1 - - shasum --version - list: - cmds: - - rsync --list-only {{.SOURCE}} > {{.RUN}}/listing.txt -``` - -`examples/rsync/.taskrc.yml`: - -```yaml -remote: - trusted-hosts: [raw.githubusercontent.com] - expiry: 1h -``` - -`examples/rsync/op.env`: - -``` -# op:// references only; `op run --env-file=op.env` resolves them at run time. -AWS_ACCESS_KEY_ID=op://VAULT/r2/access_key_id -AWS_SECRET_ACCESS_KEY=op://VAULT/r2/secret_access_key -AWS_ENDPOINT_URL=op://VAULT/r2/endpoint -HEALTHCHECK_URL=op://VAULT/healthcheck/url -``` - -- [x] **Step 3: Write the proton example** - -`examples/proton/Taskfile.yml`: - -```yaml -# A consumer of the toolbox with the proton image. -version: '3' -includes: - toolbox: - taskfile: ../../toolbox.yml - flatten: true - vars: - NAME: example-proton - DESC: the proton toolbox, exercised - IMAGE: ghcr.io/katoptra/toolbox:proton-dev - LIB_DIR: ../.. - excludes: [clock] # a Python engine keeps its own clock -tasks: - pipeline: - cmds: - - {task: clock} - - {task: tools} - - {task: ping} - plan-pipeline: - cmds: - - {task: clock} - - {task: tools} - clock: - cmds: - - python -c 'import time; print(int(time.time()))' - tools: - desc: Every tool the proton image promises, by version - cmds: - - python --version - - proton-drive version | head -1 - - age --version - - python -c 'import boto3, requests; print(boto3.__version__, requests.__version__)' - - ruff --version - - pytest --version -``` - -`examples/proton/.taskrc.yml` and `examples/proton/op.env`: the same two files as the rsync example. - -- [x] **Step 4: Render and accept both examples** - -Run: - -```sh -cd ~/Git/katoptra/lib/examples/rsync && task render-update && cat render.txt -cd ~/Git/katoptra/lib/examples/proton && task render-update && cat render.txt -``` - -Expected: each `render.txt` lists every command of `pipeline` as `task: [verb] command` lines with `/work/.run` paths, and no line from `image` or `run` themselves. The rsync one names `rsync --list-only rsync://rsync.example.org/pub/`. - -- [x] **Step 5: Run the tools verb for real in each image** - -Run: - -```sh -cd ~/Git/katoptra/lib/examples/rsync && task run -- task tools -cd ~/Git/katoptra/lib/examples/proton && task run -- task tools -``` - -Expected: version lines from every tool; exit 0. This is the image's runtime check and the `run` verb's plumbing check in one. - -- [x] **Step 6: Prove check catches a change** - -Run: `cd ~/Git/katoptra/lib/examples/rsync && sed -i '' 's|rsync.example.org|changed.example.org|' Taskfile.yml && task check; git checkout Taskfile.yml` -Expected: `task check` exits nonzero with a unified diff whose changed line is the `list` command. - -- [x] **Step 7: Prove the trust rule and the offline rule** - -Run: `cd ~/Git/katoptra/lib/examples/rsync && task run -- sh -c 'echo $TASK_REMOTE_OFFLINE'` -Expected: `1`. - -- [x] **Step 8: Commit** - -```sh -git add toolbox.yml examples/ -git commit -m "feat(toolbox): the shared verbs, with an example consumer per image" -``` - ---- - -### Task 5: The composite action - -**Files:** -- Create: `.github/actions/toolbox/action.yml` - -**Interfaces:** -- Consumes: `toolchain.lock.toml` at `${{ github.action_path }}/../../../toolchain.lock.toml`, which is where GitHub checks the action's repository out. -- Produces: `task` and `op` on the runner PATH at the lock's versions. Used by Tasks 6 and 7. - -- [x] **Step 1: Write the action** - -```yaml -name: toolbox -description: Install go-task and the 1Password CLI at the versions in katoptra/lib's toolchain.lock.toml -runs: - using: composite - steps: - - shell: bash - env: - LOCK: ${{ github.action_path }}/../../../toolchain.lock.toml - run: | - set -eu - lock() { python3 -c 'import functools,sys,tomllib; x=tomllib.load(open(sys.argv[2],"rb")); print(functools.reduce(lambda d,k: d[k], sys.argv[1].split("."), x))' "$1" "$LOCK"; } - arch="$(dpkg --print-architecture)" - curl -fsSL "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz - echo "$(lock task.linux_${arch}.sha256) /tmp/task.tgz" | sha256sum -c - - sudo tar -xzf /tmp/task.tgz -C /usr/local/bin task - curl -fsSL "$(lock op.base_url)/$(lock op.linux_${arch}.archive)" -o /tmp/op.zip - echo "$(lock op.linux_${arch}.sha256) /tmp/op.zip" | sha256sum -c - - sudo unzip -q -o /tmp/op.zip op -d /usr/local/bin - task --version && op --version -``` - -- [x] **Step 2: Verify the lock path logic locally** - -Run: `cd ~/Git/katoptra/lib && LOCK=.github/actions/toolbox/../../../toolchain.lock.toml python3 -c 'import os,tomllib; print(tomllib.load(open(os.environ["LOCK"],"rb"))["task"]["version"])'` -Expected: `3.53.1`. - -- [x] **Step 3: Commit** - -```sh -git add .github/actions/toolbox/action.yml -git commit -m "feat(action): install task and op at the lock's versions" -``` - ---- - -### Task 6: The reusable sync and check workflows - -**Files:** -- Create: `.github/workflows/sync.yml`, `.github/workflows/check.yml` - -**Interfaces:** -- Consumes: the toolbox action (Task 5) at the same ref as the workflow, via `uses: katoptra/lib/.github/actions/toolbox@v1`. Inside this repository's own CI the path form `./.github/actions/toolbox` is used instead. -- Produces: `workflow_call` workflows a mirror calls with `uses: katoptra/lib/.github/workflows/sync.yml@v1`. - -- [x] **Step 1: Write `sync.yml`** - -```yaml -# Reusable: one mirror, one run. The caller is a ten-line workflow_dispatch that -# jshvn/dispatch triggers; it passes `vars` through and inherits its repository secret. -name: sync -on: - workflow_call: - inputs: - vars: - description: 'KEY=value pairs appended to task sync, e.g. "SEED=true MAX_BATCHES=8"' - type: string - default: '' - timeout-minutes: - type: number - default: 355 - secrets: - OP_SERVICE_ACCOUNT_TOKEN: - required: false -permissions: - contents: read - actions: write # chain: gh workflow run -jobs: - sync: - runs-on: ubuntu-latest - timeout-minutes: ${{ inputs.timeout-minutes }} - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: katoptra/lib/.github/actions/toolbox@v1 - - run: df -h . && task image - - run: task sync ${{ inputs.vars }} - - name: Ping /fail unless the job succeeded - if: failure() - run: task op -- task ping-fail - - name: Chain the next run - if: success() - env: - GH_TOKEN: ${{ github.token }} - run: | - if [ -f .run/chain ]; then - gh workflow run sync.yml --ref "$GITHUB_REF_NAME" - echo "chained: next run queued" >> "$GITHUB_STEP_SUMMARY" - fi -``` - -- [x] **Step 2: Write `check.yml`** - -```yaml -# Reusable: render the mirror's pipeline inside its image and diff it against render.txt. -name: check -on: - workflow_call: {} -permissions: - contents: read -jobs: - check: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: katoptra/lib/.github/actions/toolbox@v1 - - run: task check -``` - -- [x] **Step 3: Verify both parse** - -Run: `for f in .github/workflows/sync.yml .github/workflows/check.yml; do ruby -ryaml -e 'YAML.load_file(ARGV[0]); puts "ok #{ARGV[0]}"' "$f"; done` -Expected: `ok` for both. macOS ships ruby; GitHub validates the schema on push. - -- [x] **Step 4: Commit** - -```sh -git add .github/workflows/sync.yml .github/workflows/check.yml -git commit -m "feat(workflows): reusable sync and check for every mirror" -``` - ---- - -### Task 7: This repository's CI and release - -**Files:** -- Create: `.github/workflows/ci.yml`, `.github/workflows/release.yml` - -**Interfaces:** -- Consumes: the examples (Task 4), the Dockerfiles (Task 3), the action (Task 5) by path. -- Produces: images at `ghcr.io/katoptra/toolbox:-` and `-v`, and a moved `v` git tag, on every `v*.*.*` tag push. - -- [x] **Step 1: Write `ci.yml`** - -```yaml -name: ci -on: - pull_request: - push: - branches: [main] -permissions: - contents: read -jobs: - images: - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - variant: [rsync, proton] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/toolbox - - name: Build the image the example names - run: docker build -t ghcr.io/katoptra/toolbox:${{ matrix.variant }}-dev -f docker/${{ matrix.variant }}.Dockerfile . - - name: The example's tools resolve inside the image - working-directory: examples/${{ matrix.variant }} - run: task run -- task tools - - name: The example's render matches its render.txt - working-directory: examples/${{ matrix.variant }} - run: task check -``` - -- [x] **Step 2: Write `release.yml`** - -```yaml -# On a semver tag: build each image for amd64 and arm64, push both tags, move the -# floating major tag. Mirrors pin v in the include URL and the image name, so -# moving it is the rollout. -name: release -on: - push: - tags: ['v[0-9]+.[0-9]+.[0-9]+'] -permissions: - contents: write - packages: write -jobs: - images: - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - variant: [rsync, proton] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0 - - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Tags - id: tags - run: | - tag="${GITHUB_REF_NAME}"; major="${tag%%.*}" - echo "semver=ghcr.io/katoptra/toolbox:${{ matrix.variant }}-${tag}" >> "$GITHUB_OUTPUT" - echo "major=ghcr.io/katoptra/toolbox:${{ matrix.variant }}-${major}" >> "$GITHUB_OUTPUT" - - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: . - file: docker/${{ matrix.variant }}.Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ${{ steps.tags.outputs.semver }} - ${{ steps.tags.outputs.major }} - tag: - needs: images - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Move the floating major tag - run: | - major="${GITHUB_REF_NAME%%.*}" - git tag -f "$major" "$GITHUB_SHA" - git push -f origin "refs/tags/$major" -``` - -The docker action SHAs above were resolved from each action's latest release tag on 2026-09-08 with `git ls-remote`. Dependabot keeps them current afterwards. - -- [x] **Step 3: Commit** - -```sh -git add .github/workflows/ci.yml .github/workflows/release.yml -git commit -m "ci: build and check the examples; release images and the major tag" -``` - ---- - -### Task 8: README, CONTRIBUTING, CLAUDE.md - -**Files:** -- Modify: `README.md` -- Create: `CONTRIBUTING.md`, `CLAUDE.md` - -**Interfaces:** -- Consumes: everything above. -- Produces: the documentation a mirror author reads to consume the library, in the org's README format (badges, How it works, Working on it, Want your own?, Pull requests are welcome, MIT line). - -- [x] **Step 1: Write `README.md`** - -```markdown -# lib - -[![ci](https://github.com/katoptra/lib/actions/workflows/ci.yml/badge.svg)](https://github.com/katoptra/lib/actions/workflows/ci.yml) -[![license](https://img.shields.io/github/license/katoptra/lib)](LICENSE) - -The toolbox every katoptra mirror includes: the menu, the container, the secrets model, -the dry-run check and the workflows. A mirror keeps its identity and its pipeline; -everything about how a run is started, contained and reported comes from here. - -## How it works - -1. **`toolbox.yml`** is a go-task include. A mirror includes it by URL, flattened, and - gets `sync`, `plan`, `check`, `run`, `op`, `image`, `clock`, `ping` and the rest. A - verb the mirror lists under `excludes:` and defines itself replaces the library's. -2. **`toolchain.lock.toml`** pins every tool by version and checksum. The two images - under `docker/` and the `toolbox` action install from it and nothing else. -3. **Images** are published to `ghcr.io/katoptra/toolbox` as `rsync-v1` and `proton-v1` - by the release workflow. A run pulls one and never builds. -4. **Workflows** under `.github/workflows` are reusable: a mirror's `sync.yml` and - `check.yml` are ten lines each and call these by tag. -5. **`examples/`** hold one consumer per image. Their committed `render.txt` is the - library's own check: every verb resolves and every command renders. - -## Using it in a mirror - -```yaml -version: '3' -vars: - SOURCE: rsync://rsync.dante.ctan.org/CTAN/ - BUCKET: ctan - HOST: ctan.ijosh.com -includes: - toolbox: - taskfile: https://raw.githubusercontent.com/katoptra/lib/v1/toolbox.yml - flatten: true - vars: - NAME: ctan - DESC: an hourly mirror of CTAN at https://ctan.ijosh.com/ - IMAGE: ghcr.io/katoptra/toolbox:rsync-v1 - PASS: SEED RECONCILE MAX_BATCHES -tasks: - pipeline: {cmds: [{task: clock}, {task: list}, {task: ping}]} - plan-pipeline: {cmds: [{task: clock}, {task: list}]} -``` - -Beside it, a `.taskrc.yml` so nothing prompts and the include is fetched at most hourly: - -```yaml -remote: - trusted-hosts: [raw.githubusercontent.com] - expiry: 1h -``` - -An `op.env` of `op://` references, if the secrets live in 1Password. And the two -caller workflows: - -```yaml -# .github/workflows/sync.yml -name: sync -on: - workflow_dispatch: - inputs: {vars: {type: string, default: ''}} -concurrency: {group: sync, cancel-in-progress: false} -jobs: - sync: - uses: katoptra/lib/.github/workflows/sync.yml@v1 - with: {vars: '${{ inputs.vars }}'} - secrets: inherit -``` - -```yaml -# .github/workflows/check.yml -name: check -on: {pull_request: {}} -jobs: - check: - uses: katoptra/lib/.github/workflows/check.yml@v1 -``` - -Then `task render-update` once, commit `render.txt`, and `task check` from then on. - -## Working on it - -```sh -$ git clone https://github.com/katoptra/lib -$ cd lib/examples/rsync && task image-build && task check -$ cd ../proton && task image-build && task check -``` - -`task image-build` builds the image the example names from `docker/`. `task check` -renders the example's pipeline inside it and diffs against `render.txt`. Change a verb, -run `task render-update` in each example, and the diff in the pull request is the -review. - -## Releasing - -Tag a commit `vX.Y.Z` and push the tag. The release workflow builds both images for -amd64 and arm64, pushes `-vX.Y.Z` and `-vX`, and moves the `vX` git -tag. Every mirror pinned to `vX` picks the change up on its next run. A breaking change -to a verb's name or contract is a new major. - -Pull requests are welcome. - -MIT licensed. Built by [Josh Vaughen](https://ijosh.com). -``` - -- [x] **Step 2: Write `CONTRIBUTING.md`** - -```markdown -# Contributing - -The [organization's rules](https://github.com/katoptra/.github/blob/main/CONTRIBUTING.md) -apply. This repository adds three. - -- **No `vars:` default for anything a mirror owns.** A default declared in this file's - `vars:` block shadows the mirror's root value. Defaults go inline in the command as - `{{.X | default N}}`. -- **The verb names in `toolbox.yml` are reserved.** Host-side and in-container verbs - share one namespace when flattened into a mirror. Renaming one is a major version. -- **A change to a verb is a change to both `render.txt` files.** Run `task render-update` - in each example and commit the result with the change. The pull request diff is the - review. - -## Checking a change - -```sh -cd examples/rsync && task image-build && task run -- task tools && task check -cd examples/proton && task image-build && task run -- task tools && task check -``` - -CI runs the same three commands per image on every pull request. -``` - -- [x] **Step 3: Write `CLAUDE.md`** - -```markdown -# lib - -The toolbox every katoptra mirror includes by URL. Read `README.md` for the contract and -`docs/superpowers/specs/2026-09-08-toolbox-library-design.md` for the decisions. - -## Constraints - -- `toolbox.yml` declares no `vars:` default for anything a mirror owns. Defaults go - inline. This is a go-task fact, verified: a library `vars:` default shadows the - mirror's root value. -- Verb names are reserved across host and container. `plan` is the host-side read-only - run; an engine's batch planner is `split`. -- Every tool in an image comes from `toolchain.lock.toml` with a checksum. The AWS CLI - zip is the marked exception. -- Images set `TASK_REMOTE_OFFLINE=1`. Inside a run the include resolves from the - mirror's `.task/remote` cache, bind-mounted with the repo, never from the network. -- Actions pinned to a full SHA with the version in a trailing comment. - -## Verifying a change - -```sh -cd examples/rsync && task image-build && task run -- task tools && task check -cd examples/proton && task image-build && task run -- task tools && task check -``` - -A verb change updates both `render.txt` files via `task render-update`. -``` - -- [x] **Step 4: Commit** - -```sh -git add README.md CONTRIBUTING.md CLAUDE.md -git commit -m "docs: the contract, how to work on it, how to release" -``` - ---- - -### Task 9: Hand-off - -Not automated. After the plan is executed locally: - -1. `gh repo create katoptra/lib --public --source ~/Git/katoptra/lib --push` with the description "The toolbox every katoptra mirror includes: verbs, images, workflows". -2. Confirm the org allows workflows to write packages (Settings, Packages) so `release.yml` can push to GHCR. -3. Push, watch `ci` pass, then `git tag v1.0.0 && git push origin v1.0.0` and watch `release` publish `ghcr.io/katoptra/toolbox:rsync-v1` and `proton-v1`, and move `v1`. -4. Make the two packages public in the org's package settings, or runs cannot pull them. -5. Then one migration plan per mirror, tlnet first because it is smallest. diff --git a/engines/rsync.yml b/engines/rsync.yml index 4b8a6b4..5c9cbfb 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -13,16 +13,19 @@ # the mirror owns and reconcile never deletes, space separated; empty: none), INDEX (the # directory-page key suffix an index hook writes; empty: none). # -# Hooks: prepare, verify, index, smoke and report-engine are the verbs a mirror may -# replace, by listing each under excludes: on this include and defining its own. prepare -# and verify do nothing until TL_KEY is set; index does nothing here. +# Hooks: prepare, verify, index, smoke, smoke-mirror and report-engine are the verbs a +# mirror may replace, by listing each under excludes: on this include and defining its +# own. prepare and verify do nothing until TL_KEY is set; index and smoke-mirror do +# nothing here. pipeline and plan-pipeline are the vocabulary in order; a mirror that +# needs another order excludes and redefines them. # # Rules this file keeps, as the toolbox does: no vars: default for anything a mirror # owns, because a value here shadows the mirror's root value, and every tunable is an # inline {{.X | default N}}, because a mirror's root value in turn shadows a KEY=value # given on the command line inside the image (task sync -- MAX_BATCHES=8 reaches these -# verbs only while no vars: block names MAX_BATCHES). The vars below derive from the -# mirror's or are the engine's own helpers. +# verbs only while no vars: block names MAX_BATCHES). A var of another name that reads +# the tunable still sees the command line, so each default is spelled once, below. The +# rest of the vars derive from the mirror's or are the engine's own helpers. version: '3' vars: @@ -30,9 +33,16 @@ vars: URL: https://{{.HOST}} STATE: .state/applied.txt.xz # the listing of what the bucket holds; never in upstream STAGING: '{{.ROOT_DIR}}/staging' + BATCHES_MAX: '{{.MAX_BATCHES | default 4}}' # batches per run; the rest chain the next run + RETRY_S: '{{.RETRY_BASE | default 15}}' # seconds; a retry sleeps RETRY_S * 2^i plus jitter RSYNC: rsync --timeout=300 --contimeout=60 --no-h CURL: curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 AWS_FLAGS: --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 + # The keyring is read as a plain file from the same mirror as the signature, so the pinned + # fingerprint is the only real check; GOODSIG too, because gpgv reports an expired or + # revoked key as VALIDSIG with exit 0. An awk program over gpgv's --status-fd 1 lines, + # given the fingerprint as k where it is used, so a call var TL_KEY reaches it. + GPGCHECK: '/^\[GNUPG:\] GOODSIG /{g=1} /^\[GNUPG:\] VALIDSIG / && $NF == k {v=1} END{exit !(g&&v)}' # An object's size from the headers of a one-byte ranged read, which costs no bandwidth on # a 6.87 GB file and answers where a HEAD does not: Cloudflare streams text/html and sends # it no content-length. content-range carries the total, an empty object answers 416, and @@ -59,6 +69,33 @@ vars: { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' tasks: + pipeline: + desc: clock -> list -> state -> rebuild? -> diff -> split -> prepare -> batches -> delete -> reconcile? -> index -> smoke -> report -> ping + cmds: + - {task: clock} + - {task: list} + - {task: state} + - {task: rebuild} + - {task: diff} + - {task: split} + - {task: prepare} + - {task: batches} + - {task: delete} + - {task: reconcile} + - {task: index} + - {task: smoke} + - {task: report} + - {task: ping} + + plan-pipeline: + desc: 'The read-only half: clock -> list -> state -> diff -> split' + cmds: + - {task: clock} + - {task: list} + - {task: state} + - {task: diff} + - {task: split} + # ---- the listing and the state ---- list: desc: List upstream, narrowed by FILTER when set, and normalise it to RUN/upstream.txt as "path TAB size TAB mtime", byte-sorted @@ -97,18 +134,33 @@ tasks: desc: Fetch the state file to RUN/applied.txt; a missing one is rebuilt from a bucket listing (an empty bucket gives an empty state, and the first run fills it) cmds: - mkdir -p {{.RUN}} + # The bucket is the mirror and the state is a cache of it: a missing one costs one + # listing, never an upload. + - {task: pull, vars: {KEY: '{{.STATE}}', OUT: '{{.RUN}}/applied.txt', MISSING: 'echo "no state at {{.STATE}}: rebuilding it from the bucket" >&2; touch {{.RUN}}/rebuild-now'}} + + pull: + desc: An xz-compressed KEY from the bucket, decompressed to OUT; MISSING runs when the bucket has no such key + internal: true + requires: {vars: [KEY, OUT, MISSING]} + cmds: # aws s3 ls exits 1 when nothing matches and 0 when the key exists; any other code is - # a real error and must not be mistaken for a missing state. The bucket is the mirror - # and the state is a cache of it: a missing one costs one listing, never an upload. + # a real error and must not be mistaken for a missing key. - | - rc=0; aws s3 ls {{.S3}}/{{.STATE}} > /dev/null || rc=$? + rc=0; aws s3 ls {{.S3}}/{{.KEY}} > /dev/null || rc=$? case $rc in - 0) aws s3 cp {{.AWS_FLAGS}} {{.S3}}/{{.STATE}} {{.RUN}}/applied.txt.xz - xz -dc {{.RUN}}/applied.txt.xz > {{.RUN}}/applied.txt ;; - 1) echo "no state at {{.STATE}}: rebuilding it from the bucket" >&2; touch {{.RUN}}/rebuild-now ;; + 0) aws s3 cp {{.AWS_FLAGS}} {{.S3}}/{{.KEY}} {{.OUT}}.xz && xz -dc {{.OUT}}.xz > {{.OUT}} ;; + 1) {{.MISSING}} ;; *) exit $rc ;; esac + push: + desc: FILE, xz-compressed, to the bucket's KEY as one PutObject; the old object or the new one is in the bucket, never half + internal: true + requires: {vars: [FILE, KEY]} + cmds: + - xz -T0 -c {{.FILE}} > {{.FILE}}.xz + - aws s3 cp {{.AWS_FLAGS}} {{.FILE}}.xz {{.S3}}/{{.KEY}} + rebuild: desc: List the bucket and make the state exactly what is there (a lost state file, or the daily reconcile) status: ['test ! -f {{.RUN}}/rebuild-now'] @@ -125,8 +177,7 @@ tasks: - >- LC_ALL=C join -t "$(printf '\t')" -o 1.1,1.2,1.3,2.2 {{.RUN}}/upstream.txt {{.RUN}}/bucket.txt | awk -F'\t' -v OFS='\t' '$2 == $4 { print $1, $2, $3 }' > {{.RUN}}/applied.new - - xz -T0 -c {{.RUN}}/applied.new > {{.RUN}}/applied.new.xz - - aws s3 cp {{.AWS_FLAGS}} {{.RUN}}/applied.new.xz {{.S3}}/{{.STATE}} + - {task: push, vars: {FILE: '{{.RUN}}/applied.new', KEY: '{{.STATE}}'}} - mv {{.RUN}}/applied.new {{.RUN}}/applied.txt - rm -f {{.RUN}}/rebuild-now @@ -182,26 +233,22 @@ tasks: - task: retry vars: {CMD: '{{.RSYNC}} -Lt --files-from={{.RUN}}/tl/files.txt {{.SOURCE}}{{.TL}}/ {{.RUN}}/tl/'} - xz -dkf {{.RUN}}/tl/tlpkg/texlive.tlpdb.xz - # The keyring is read as a plain file from the same mirror as the signature, so the - # pinned fingerprint is the only real check; GOODSIG too, because gpgv reports an - # expired or revoked key as VALIDSIG with exit 0. - >- cd {{.RUN}}/tl/tlpkg && shasum -a 512 -c texlive.tlpdb.sha512 && - gpgv --status-fd 1 --keyring gpg/pubring.gpg texlive.tlpdb.sha512.asc texlive.tlpdb.sha512 - | awk '/^\[GNUPG:\] GOODSIG /{g=1} /^\[GNUPG:\] VALIDSIG .* {{.TL_KEY}}$/{v=1} END{exit !(g&&v)}' + gpgv --status-fd 1 --keyring gpg/pubring.gpg texlive.tlpdb.sha512.asc texlive.tlpdb.sha512 | awk -v k={{.TL_KEY}} '{{.GPGCHECK}}' batches: desc: Work the first MAX_BATCHES of RUN/batch-NNNN.txt in name order; each commits before the next starts vars: BATCHES: - sh: ls {{.RUN}}/batch-*.txt 2>/dev/null | head -n {{.MAX_BATCHES | default 4}} + sh: ls {{.RUN}}/batch-*.txt 2>/dev/null | head -n {{.BATCHES_MAX}} cmds: - for: {var: BATCHES, split: "\n", as: B} task: batch vars: {B: '{{.B}}'} # Batches past MAX_BATCHES wait for the next run; the chain file asks the workflow to # queue it now rather than at the next hour, which is how an empty bucket fills. - - test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -le {{.MAX_BATCHES | default 4}} || touch {{.RUN}}/chain + - test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -le {{.BATCHES_MAX}} || touch {{.RUN}}/chain batch: internal: true @@ -234,15 +281,15 @@ tasks: verify: requires: {vars: [B]} # public, so a mirror can run it by hand against a canned batch status: ['test -z "{{.TL_KEY}}"'] # the no-op hook until a mirror pins a key - dir: '{{.STAGING}}' + # Each command works from STAGING by its own cd, not dir:, which a path-based include + # resolves against the include's directory and no call var can move. cmds: # Every signed sha512 in this batch (root installers and updaters, the tlpdb) checks # against the keyring already pinned by prepare. The while loop's status is the pipeline's. - >- - cut -f1 {{.B}} | grep -E '^{{.TL}}/(tlpkg/texlive\.tlpdb|[^/]+)\.sha512$' | while IFS= read -r f; do + cd {{.STAGING}} && cut -f1 {{.B}} | grep -E '^{{.TL}}/(tlpkg/texlive\.tlpdb|[^/]+)\.sha512$' | while IFS= read -r f; do (cd "$(dirname "$f")" && f=$(basename "$f") && shasum -a 512 -c "$f" && - gpgv --status-fd 1 --keyring {{.RUN}}/tl/tlpkg/gpg/pubring.gpg "$f.asc" "$f" - | awk '/^\[GNUPG:\] GOODSIG /{g=1} /^\[GNUPG:\] VALIDSIG .* {{.TL_KEY}}$/{v=1} END{exit !(g&&v)}') + gpgv --status-fd 1 --keyring {{.RUN}}/tl/tlpkg/gpg/pubring.gpg "$f.asc" "$f" | awk -v k={{.TL_KEY}} '{{.GPGCHECK}}') || exit 1; done # Every TL container in this batch must match the checksum the verified tlpdb gives # it, under both the names upstream carries: the stable one the tlpdb names and the @@ -258,7 +305,7 @@ tasks: # checked inside the same command that writes them, so no batch is ever checked against # the sums left by the batch before it -- fetch empties staging between the two. - >- - test "$(grep -c '^{{.TL}}/archive/' {{.B}})" = 0 || { + cd {{.STAGING}} && { test "$(grep -c '^{{.TL}}/archive/' {{.B}})" = 0 || { awk -F'\t' -v TL="{{.TL}}" 'NR == FNR { if (index($1, TL "/archive/") == 1) want[$1] = 1; next } /^name /{split($0, a, " "); n = a[2]} /^revision /{split($0, a, " "); r = a[2]} /^containerchecksum /{split($0, a, " "); emit(a[2], n)} @@ -269,13 +316,13 @@ tasks: if (q in want) { print sum " " q; delete want[q] } } END { for (p in want) { print "no checksum in the tlpdb for " p > "/dev/stderr"; bad = 1 } exit bad }' \ {{.B}} {{.RUN}}/tl/tlpkg/texlive.tlpdb > {{.RUN}}/tl/sums.txt && - shasum -a 512 -c --quiet {{.RUN}}/tl/sums.txt; } + shasum -a 512 -c --quiet {{.RUN}}/tl/sums.txt; }; } # The decision batch. The tlpdb it carries must be the one verified at the start of # the run, the .xz tlmgr downloads must decompress to it byte for byte, and every # container it names must be in the bucket after this run: the state, plus this # batch, minus this run's deletions. - >- - test ! -f {{.TL}}/tlpkg/texlive.tlpdb || { + cd {{.STAGING}} && { test ! -f {{.TL}}/tlpkg/texlive.tlpdb || { cmp {{.TL}}/tlpkg/texlive.tlpdb {{.RUN}}/tl/tlpkg/texlive.tlpdb && xz -dc {{.TL}}/tlpkg/texlive.tlpdb.xz | cmp - {{.TL}}/tlpkg/texlive.tlpdb && { cut -f1 {{.RUN}}/applied.txt; cut -f1 {{.B}}; } | LC_ALL=C sort -u @@ -285,7 +332,7 @@ tasks: /^doccontainerchecksum /{print TL "/archive/" n ".doc.tar.xz"} /^srccontainerchecksum /{print TL "/archive/" n ".source.tar.xz"}' {{.TL}}/tlpkg/texlive.tlpdb \ | LC_ALL=C sort | LC_ALL=C comm -23 - {{.RUN}}/tl/after.txt > {{.RUN}}/tl/missing.txt && - test ! -s {{.RUN}}/tl/missing.txt; } + test ! -s {{.RUN}}/tl/missing.txt; }; } publish: internal: true @@ -318,9 +365,7 @@ tasks: set: [pipefail] cmds: - {task: merge, vars: {B: '{{.B}}'}} - - xz -T0 -c {{.RUN}}/applied.new > {{.RUN}}/applied.new.xz - # One PutObject; the old file or the new one is in the bucket, never half. - - aws s3 cp {{.AWS_FLAGS}} {{.RUN}}/applied.new.xz {{.S3}}/{{.STATE}} + - {task: push, vars: {FILE: '{{.RUN}}/applied.new', KEY: '{{.STATE}}'}} - mv {{.RUN}}/applied.new {{.RUN}}/applied.txt - find {{.STAGING}} -mindepth 1 -delete @@ -348,7 +393,7 @@ tasks: D: '{{if .D}}{{.D}}{{else}}{{.RUN}}/deleted.txt{{end}}' # Nothing to delete, or a batch is still waiting for a later hour: the live tlpdb may # still name these containers, so they go only in the hour every batch has landed. - status: ['test ! -s {{.D}} || test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -gt {{.MAX_BATCHES | default 4}}'] + status: ['test ! -s {{.D}} || test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -gt {{.BATCHES_MAX}}'] cmds: - {task: remove, vars: {K: '{{.D}}'}} - {task: checkpoint, vars: {B: '{{.D}}'}} @@ -363,17 +408,19 @@ tasks: cmds: - touch {{.RUN}}/rebuild-now - {task: rebuild} - # A mirror's own keys are no orphans: the bucket-root keys in OWN, and the directory - # pages an index hook draws under two keys when INDEX is set. One key names itself - # (the INDEX suffix); the other is a bare directory of the state, indistinguishable - # from an upstream path, so the state's own directory set is the list. A path that - # stops being a directory upstream leaves the set and is judged as a file. + # A mirror's own keys are no orphans: the bucket-root keys in OWN, and, only when + # INDEX is set, the directory pages an index hook draws under two keys. One key names + # itself (the INDEX suffix); the other is a bare directory of the state, + # indistinguishable from an upstream path, so the state's own directory set is the + # list. A path that stops being a directory upstream leaves the set and is judged as + # a file. Without INDEX both lists are empty: the suffix would otherwise degenerate to + # "/" and spare every key that ends in one. - >- - {{.ANCESTORS}} {{.RUN}}/applied.txt - | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > {{.RUN}}/index-keys.txt + {{if .INDEX}}{{.ANCESTORS}} {{.RUN}}/applied.txt + | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > {{.RUN}}/index-keys.txt{{else}}: > {{.RUN}}/index-keys.txt{{end}} - >- cut -f1 {{.RUN}}/bucket.txt | grep -vE '^\.state/'{{if .OWN}} | grep -vxF -f <(printf '%s\n' {{.OWN}}){{end}} - | awk -v i="/{{.INDEX}}" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i' + {{if .INDEX}}| awk -v i="/{{.INDEX}}" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i'{{end}} | LC_ALL=C comm -23 - {{.RUN}}/index-keys.txt | LC_ALL=C comm -23 - {{.RUN}}/paths.txt > {{.RUN}}/orphans.txt - {task: delete, vars: {D: '{{.RUN}}/orphans.txt'}} @@ -384,7 +431,7 @@ tasks: cmds: [] smoke: - desc: Read a sample of this run's keys back through the domain, sizes against the listing, and with TL set the tlpdb sha512 against the verified copy; a mirror with more to check replaces it + desc: Read a sample of this run's keys back through the domain, sizes against the listing, and with TL set the tlpdb sha512 against the verified copy; then the mirror's own checks, smoke-mirror vars: # The check below that names /timestamp asks for it only once the state says it landed. # It rides the decision batch, the last one, so a first fill capped at MAX_BATCHES has @@ -407,8 +454,15 @@ tasks: # MAX_BATCHES has not published this hour's copy yet and has nothing to compare. - >- {{if .TL}}test ! -f {{.RUN}}/tl/tlpkg/texlive.tlpdb.sha512 || - test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -gt {{.MAX_BATCHES | default 4}} || + test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -gt {{.BATCHES_MAX}} || {{.CURL}} {{.URL}}/{{.TL}}/tlpkg/texlive.tlpdb.sha512 | cmp - {{.RUN}}/tl/tlpkg/texlive.tlpdb.sha512{{else}}echo "no TL subtree: no tlpdb to read back"{{end}} + # A called task sees none of its caller's call vars, so the three a check reads are + # handed on; in a run they are the values above, and an offline check passes its own. + - {task: smoke-mirror, vars: {RUN: '{{.RUN}}', STAGING: '{{.STAGING}}', URL: '{{.URL}}'}} + + smoke-mirror: + desc: A hook, empty here; a mirror with more to read back than the sample replaces it, and smoke runs it last + cmds: [] report-engine: desc: The engine's rows of the run summary, appended after the toolbox's @@ -419,7 +473,7 @@ tasks: gb() { test -f "$1" && awk -F'\t' '{ s += $2 } END { printf "%.2f", s / 1e9 }' "$1" || echo 0.00; } cat >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" </dev/null | wc -l | awk -v m={{.MAX_BATCHES | default 4}} '{ print $1 " batches" ($1 > m ? ", " $1 - m " left for the next run" : "") }'); $(n {{.RUN}}/deleted.txt) deleted, $(n {{.RUN}}/orphans.txt) orphaned | + | Delta | $(n {{.RUN}}/changed.txt) new or changed in $(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l | awk -v m={{.BATCHES_MAX}} '{ print $1 " batches" ($1 > m ? ", " $1 - m " left for the next run" : "") }'); $(n {{.RUN}}/deleted.txt) deleted, $(n {{.RUN}}/orphans.txt) orphaned | | Published to R2 | $(awk '/^upload:/ { n++ } END { print n + 0 }' {{.RUN}}/publish.txt 2>/dev/null || echo 0) uploaded | | State | $(n {{.RUN}}/applied.txt) lines in {{.STATE}} | | Storage | $(test -f {{.RUN}}/bucket.txt && echo "$(gb {{.RUN}}/bucket.txt) GB in $(n {{.RUN}}/bucket.txt) objects (reconciled)" || echo "not measured this run"){{if .CEILING_GB}} of the {{.CEILING_GB}} GB ceiling{{end}} | @@ -435,5 +489,5 @@ tasks: rc=0; ( {{.CMD}} ) || rc=$? # subshell so CMD's exit cannot end the loop; || so Task's errexit cannot case $rc in 0|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break - s=$(({{.RETRY_BASE | default 15}} * 2 ** i + RANDOM % (2 * {{.RETRY_BASE | default 15}} + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s + s=$(({{.RETRY_S}} * 2 ** i + RANDOM % (2 * {{.RETRY_S}} + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s done; exit $rc diff --git a/examples/proton/.taskrc.yml b/examples/proton/.taskrc.yml index 37c45f5..a8d9f1f 100644 --- a/examples/proton/.taskrc.yml +++ b/examples/proton/.taskrc.yml @@ -1,3 +1,3 @@ remote: trusted-hosts: [raw.githubusercontent.com] - expiry: 1h + cache-expiry: 1h diff --git a/examples/proton/Taskfile.yml b/examples/proton/Taskfile.yml index 80b9711..ed0c08e 100644 --- a/examples/proton/Taskfile.yml +++ b/examples/proton/Taskfile.yml @@ -1,4 +1,5 @@ -# A consumer of the toolbox with the proton image. +# A consumer of the toolbox with the proton image. No engine, so the mirror defines the +# two pipeline verbs itself, from the toolbox's clock, report and ping and its own steps. version: '3' includes: toolbox: @@ -9,21 +10,19 @@ includes: DESC: the proton toolbox, exercised IMAGE: ghcr.io/katoptra/toolbox:proton-dev LIB_DIR: ../.. - excludes: [clock] # a Python engine keeps its own clock tasks: pipeline: + desc: clock -> tools -> report -> ping cmds: - {task: clock} - {task: tools} - {task: report} - {task: ping} plan-pipeline: + desc: 'The read-only half: clock -> tools' cmds: - {task: clock} - {task: tools} - clock: - cmds: - - python -c 'import time; print(int(time.time()))' tools: desc: Every tool the proton image promises, by version cmds: diff --git a/examples/proton/render.txt b/examples/proton/render.txt index 0f890c8..6499d89 100644 --- a/examples/proton/render.txt +++ b/examples/proton/render.txt @@ -1,4 +1,4 @@ -task: [clock] python -c 'import time; print(int(time.time()))' +task: [clock] mkdir -p /work/examples/proton/.run && rm -f /work/examples/proton/.run/chain && date -u '+%s %H %u' > /work/examples/proton/.run/start.txt task: [tools] python --version task: [tools] proton-drive version | head -1 task: [tools] age --version diff --git a/examples/rsync/.taskrc.yml b/examples/rsync/.taskrc.yml index 37c45f5..a8d9f1f 100644 --- a/examples/rsync/.taskrc.yml +++ b/examples/rsync/.taskrc.yml @@ -1,3 +1,3 @@ remote: trusted-hosts: [raw.githubusercontent.com] - expiry: 1h + cache-expiry: 1h diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml index 40a6829..20d713a 100644 --- a/examples/rsync/Taskfile.yml +++ b/examples/rsync/Taskfile.yml @@ -2,9 +2,8 @@ # the includes are paths; a mirror includes # https://raw.githubusercontent.com/katoptra/lib/v1/toolbox.yml and # https://raw.githubusercontent.com/katoptra/lib/v1/engines/rsync.yml. The pipeline is -# the engine's vocabulary in order; render.txt, every command of it, is the library's -# own check of the engine, and `task run -- task offline` runs the verbs that need no -# bucket over fixtures/. +# the engine's; render.txt, every command of it, is the library's own check of the engine, +# and `task run -- task offline` runs the verbs that need no bucket over fixtures/. version: '3' vars: SOURCE: rsync://rsync.example.org/pub/ @@ -26,29 +25,6 @@ includes: taskfile: ../../engines/rsync.yml flatten: true tasks: - pipeline: - cmds: - - {task: clock} - - {task: list} - - {task: state} - - {task: rebuild} - - {task: diff} - - {task: split} - - {task: prepare} - - {task: batches} - - {task: delete} - - {task: reconcile} - - {task: index} - - {task: smoke} - - {task: report} - - {task: ping} - plan-pipeline: - cmds: - - {task: clock} - - {task: list} - - {task: state} - - {task: diff} - - {task: split} tools: desc: Every tool the rsync image promises, by version cmds: @@ -61,6 +37,12 @@ tasks: desc: Every engine verb that needs no bucket, over fixtures/ copied under .run, inside the image vars: F: '{{.RUN}}/fixtures' + # fixtures/tree is a signed subtree, tl/, the shape of TeX Live's: a tlpdb naming one + # package's containers, its sha512 signed by a throwaway key whose public half is the + # tree's own keyring, as upstream's is. FP is what a mirror pins as TL_KEY. + TREE: '{{.ROOT_DIR}}/fixtures/tree/' + LOCAL: rsync --no-h # a local tree; the engine's daemon-only flags are a usage error here + FP: 3505F9E22FF030FE5D9C655A825C376AB4571163 cmds: # list and normalise over a local tree: its two regular files, byte-sorted, and no directory. - {task: list, vars: {SOURCE: '{{.ROOT_DIR}}/fixtures/run-root/staging/', RSYNC: 'rsync --no-h'}} @@ -84,4 +66,24 @@ tasks: - task retry CMD='exit 24' RETRY_BASE=0 - '! task retry CMD=''exit 5'' RETRY_BASE=0 2>/dev/null' - '! task retry CMD=''exit 3'' RETRY_BASE=0 2>/dev/null' + # run-tl: the signed subtree. prepare fetches the control files from the tree and checks + # the signed sha512 against the pinned key; fetch stages the batch; verify checks the + # signed file it carries, every container against the tlpdb under both of its names + # (three sums: the stable and the revision-stamped container, and the doc container), + # and, the batch carrying the tlpdb, the decision batch whole. Staged in the engine's + # own STAGING: verify's dir is fixed when the include is read, so no call var moves it. + - {task: prepare, vars: {RUN: '{{.F}}/run-tl', SOURCE: '{{.TREE}}', RSYNC: '{{.LOCAL}}', TL: tl, TL_KEY: '{{.FP}}'}} + - test -f {{.F}}/run-tl/tl/tlpkg/texlive.tlpdb + - {task: fetch, vars: {RUN: '{{.F}}/run-tl', SOURCE: '{{.TREE}}', RSYNC: '{{.LOCAL}}', B: '{{.F}}/run-tl/batch-0001.txt'}} + - {task: verify, vars: {RUN: '{{.F}}/run-tl', B: '{{.F}}/run-tl/batch-0001.txt', TL: tl, TL_KEY: '{{.FP}}'}} + - test "$(wc -l < {{.F}}/run-tl/tl/sums.txt)" -eq 3 && test ! -s {{.F}}/run-tl/tl/missing.txt + # A container the tlpdb does not describe cannot pass, whatever its bytes. + - '! task verify-refuses 2>/dev/null' - 'echo "offline: every check passed"' + verify-refuses: + desc: verify over a batch naming a container the tlpdb does not describe, after offline staged run-tl; offline expects it to fail + vars: + R: '{{.RUN}}/fixtures/run-tl' + cmds: + - {task: fetch, vars: {RUN: '{{.R}}', SOURCE: '{{.ROOT_DIR}}/fixtures/tree/', RSYNC: 'rsync --no-h', B: '{{.R}}/batch-bad.txt'}} + - {task: verify, vars: {RUN: '{{.R}}', B: '{{.R}}/batch-bad.txt', TL: tl, TL_KEY: 3505F9E22FF030FE5D9C655A825C376AB4571163}} diff --git a/examples/rsync/fixtures/run-tl/applied.txt b/examples/rsync/fixtures/run-tl/applied.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/rsync/fixtures/run-tl/batch-0001.txt b/examples/rsync/fixtures/run-tl/batch-0001.txt new file mode 100644 index 0000000..aa6fe48 --- /dev/null +++ b/examples/rsync/fixtures/run-tl/batch-0001.txt @@ -0,0 +1,7 @@ +tl/archive/foo.doc.tar.xz 14 2026/09/08 00:00:00 +tl/archive/foo.r1.tar.xz 20 2026/09/08 00:00:00 +tl/archive/foo.tar.xz 20 2026/09/08 00:00:00 +tl/tlpkg/texlive.tlpdb 422 2026/09/08 00:00:00 +tl/tlpkg/texlive.tlpdb.sha512 144 2026/09/08 00:00:00 +tl/tlpkg/texlive.tlpdb.sha512.asc 265 2026/09/08 00:00:00 +tl/tlpkg/texlive.tlpdb.xz 368 2026/09/08 00:00:00 diff --git a/examples/rsync/fixtures/run-tl/batch-bad.txt b/examples/rsync/fixtures/run-tl/batch-bad.txt new file mode 100644 index 0000000..80e2910 --- /dev/null +++ b/examples/rsync/fixtures/run-tl/batch-bad.txt @@ -0,0 +1 @@ +tl/archive/bad.tar.xz 40 2026/09/08 00:00:00 diff --git a/examples/rsync/fixtures/run-tl/changed.txt b/examples/rsync/fixtures/run-tl/changed.txt new file mode 100644 index 0000000..aa6fe48 --- /dev/null +++ b/examples/rsync/fixtures/run-tl/changed.txt @@ -0,0 +1,7 @@ +tl/archive/foo.doc.tar.xz 14 2026/09/08 00:00:00 +tl/archive/foo.r1.tar.xz 20 2026/09/08 00:00:00 +tl/archive/foo.tar.xz 20 2026/09/08 00:00:00 +tl/tlpkg/texlive.tlpdb 422 2026/09/08 00:00:00 +tl/tlpkg/texlive.tlpdb.sha512 144 2026/09/08 00:00:00 +tl/tlpkg/texlive.tlpdb.sha512.asc 265 2026/09/08 00:00:00 +tl/tlpkg/texlive.tlpdb.xz 368 2026/09/08 00:00:00 diff --git a/examples/rsync/fixtures/run-tl/deleted.txt b/examples/rsync/fixtures/run-tl/deleted.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/rsync/fixtures/tree/tl/archive/bad.tar.xz b/examples/rsync/fixtures/tree/tl/archive/bad.tar.xz new file mode 100644 index 0000000..9a7f069 --- /dev/null +++ b/examples/rsync/fixtures/tree/tl/archive/bad.tar.xz @@ -0,0 +1 @@ +a container the tlpdb does not describe diff --git a/examples/rsync/fixtures/tree/tl/archive/foo.doc.tar.xz b/examples/rsync/fixtures/tree/tl/archive/foo.doc.tar.xz new file mode 100644 index 0000000..3bc905a --- /dev/null +++ b/examples/rsync/fixtures/tree/tl/archive/foo.doc.tar.xz @@ -0,0 +1 @@ +foo doc bytes diff --git a/examples/rsync/fixtures/tree/tl/archive/foo.r1.tar.xz b/examples/rsync/fixtures/tree/tl/archive/foo.r1.tar.xz new file mode 100644 index 0000000..c67277d --- /dev/null +++ b/examples/rsync/fixtures/tree/tl/archive/foo.r1.tar.xz @@ -0,0 +1 @@ +foo container bytes diff --git a/examples/rsync/fixtures/tree/tl/archive/foo.tar.xz b/examples/rsync/fixtures/tree/tl/archive/foo.tar.xz new file mode 100644 index 0000000..c67277d --- /dev/null +++ b/examples/rsync/fixtures/tree/tl/archive/foo.tar.xz @@ -0,0 +1 @@ +foo container bytes diff --git a/examples/rsync/fixtures/tree/tl/tlpkg/gpg/pubring.gpg b/examples/rsync/fixtures/tree/tl/tlpkg/gpg/pubring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..d9345167d269989082428adce2c61a5b85e56e93 GIT binary patch literal 275 zcmbPX%#yX>LZld{HX9=g<1Kf7Mn-mrQ=I!cN^gjL)ZcXP(57HzK}k`fLRw};Noi54f(?}60F%+n%qvUG$xN~9SkEFX#>EgWrpRK- z`tyQ*A>%Ln9pn9b+bG zW>#)a4t6FcB_=jeE=~?^CKfR!W=1ACb|&!#1};tkm|5;S8U9MYo?@xp=-Be0D>B8} zqgFeg763Nr$N_7%aO4&l*-m?0fZr=L+Q%3C# HyEq;Ix2In7 literal 0 HcmV?d00001 diff --git a/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb b/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb new file mode 100644 index 0000000..ac25a65 --- /dev/null +++ b/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb @@ -0,0 +1,12 @@ +name 00texlive.config +category TLCore +revision 1 + +name foo +category Package +revision 1 +containersize 20 +containerchecksum 5563e9aaf252fa83eedb79c71736eaafd4888c18c42b3f3cfa231bc81e09a46bfa66915ff39ffde3ecbee96db49d79ea1296eee6b32ed14fc689658383b84276 +doccontainersize 14 +doccontainerchecksum d1c9b5e94e64ba89bbace9ce38ee1ae134f44183f9e997e1821c5d8acdc63f0c4d59d8e660a60c597e4042e48f577ef174f0caa8e5cc60ee09bd2d3d75fa00f3 + diff --git a/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.sha512 b/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.sha512 new file mode 100644 index 0000000..8fb31f0 --- /dev/null +++ b/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.sha512 @@ -0,0 +1 @@ +689e41401df5c017f1208e739ff257d7839ea772f0198b1e9837a94acc79ca962cf65051fefef318780160d3a8d66139955fba35bd5ed95619fbcf6ccc7bb6a7 texlive.tlpdb diff --git a/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.sha512.asc b/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.sha512.asc new file mode 100644 index 0000000..747552e --- /dev/null +++ b/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.sha512.asc @@ -0,0 +1,8 @@ +-----BEGIN PGP SIGNATURE----- + +iJEEABYKADkWIQQ1BfniL/Aw/l2cZVqCXDdqtFcRYwUCaqDQWhsUgAAAAAAEAA5t +YW51MiwyLjUrMS4xMiwwLDMACgkQglw3arRXEWMRTQD5AaAfHYAZmrvqmJDkshNb +dpk963+3sPBBoT7suyirYMABAIbw77YnvmZ+ArWqEDv5Bx6p8HvwIpgPK/QhqZIk +pswC +=uAlD +-----END PGP SIGNATURE----- diff --git a/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.xz b/examples/rsync/fixtures/tree/tl/tlpkg/texlive.tlpdb.xz new file mode 100644 index 0000000000000000000000000000000000000000..87c9f6d134c53f3dba63354062249fe269bdcdab GIT binary patch literal 368 zcmV-$0gwLuH+ooF000E$*0e?hz^($O10ewx000000000T$Uj=(0i^*ZT>v*2N&WfV zMCt`3YhPJh{H|;31>ak8I59?k%vNgx$}26Yz81P_qrW8nj1l9_?pb=^>P}z?T-KUwF=iYfop|YaD@$s0=pnAIciDT2J90000ew?p|Bi=tit0m=fV0{{T;LZD-@ O#Ao{g000001X)@ug0h7G literal 0 HcmV?d00001 diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt index 5c2ae00..aea6a9e 100644 --- a/examples/rsync/render.txt +++ b/examples/rsync/render.txt @@ -16,18 +16,17 @@ task: [normalise] awk ' | LC_ALL=C sort > /work/examples/rsync/.run/upstream.txt task: [normalise] test "$(wc -l < /work/examples/rsync/.run/upstream.txt)" -gt 1 task: [state] mkdir -p /work/examples/rsync/.run -task: [state] rc=0; aws s3 ls s3://example/.state/applied.txt.xz > /dev/null || rc=$? +task: [pull] rc=0; aws s3 ls s3://example/.state/applied.txt.xz > /dev/null || rc=$? case $rc in - 0) aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 s3://example/.state/applied.txt.xz /work/examples/rsync/.run/applied.txt.xz - xz -dc /work/examples/rsync/.run/applied.txt.xz > /work/examples/rsync/.run/applied.txt ;; + 0) aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 s3://example/.state/applied.txt.xz /work/examples/rsync/.run/applied.txt.xz && xz -dc /work/examples/rsync/.run/applied.txt.xz > /work/examples/rsync/.run/applied.txt ;; 1) echo "no state at .state/applied.txt.xz: rebuilding it from the bucket" >&2; touch /work/examples/rsync/.run/rebuild-now ;; *) exit $rc ;; esac task: [rebuild] { aws s3 ls --recursive s3://example/ || test $? -eq 1; } | awk '{ if (match($0, /^[^ ]+ +[^ ]+ +[0-9]+ /)) print substr($0, RLENGTH + 1) "\t" $3 }' | LC_ALL=C sort > /work/examples/rsync/.run/bucket.txt task: [rebuild] LC_ALL=C join -t "$(printf '\t')" -o 1.1,1.2,1.3,2.2 /work/examples/rsync/.run/upstream.txt /work/examples/rsync/.run/bucket.txt | awk -F'\t' -v OFS='\t' '$2 == $4 { print $1, $2, $3 }' > /work/examples/rsync/.run/applied.new -task: [rebuild] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rsync/.run/applied.new.xz -task: [rebuild] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz +task: [push] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rsync/.run/applied.new.xz +task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz task: [rebuild] mv /work/examples/rsync/.run/applied.new /work/examples/rsync/.run/applied.txt task: [rebuild] rm -f /work/examples/rsync/.run/rebuild-now task: [diff] LC_ALL=C comm -13 /work/examples/rsync/.run/applied.txt /work/examples/rsync/.run/upstream.txt > /work/examples/rsync/.run/changed.txt @@ -60,7 +59,7 @@ task: [retry] for i in 1 2 3 4 5; do done; exit $rc task: [prepare] xz -dkf /work/examples/rsync/.run/tl/tlpkg/texlive.tlpdb.xz -task: [prepare] cd /work/examples/rsync/.run/tl/tlpkg && shasum -a 512 -c texlive.tlpdb.sha512 && gpgv --status-fd 1 --keyring gpg/pubring.gpg texlive.tlpdb.sha512.asc texlive.tlpdb.sha512 | awk '/^\[GNUPG:\] GOODSIG /{g=1} /^\[GNUPG:\] VALIDSIG .* $/{v=1} END{exit !(g&&v)}' +task: [prepare] cd /work/examples/rsync/.run/tl/tlpkg && shasum -a 512 -c texlive.tlpdb.sha512 && gpgv --status-fd 1 --keyring gpg/pubring.gpg texlive.tlpdb.sha512.asc texlive.tlpdb.sha512 | awk -v k= '/^\[GNUPG:\] GOODSIG /{g=1} /^\[GNUPG:\] VALIDSIG / && $NF == k {v=1} END{exit !(g&&v)}' task: [fetch] find /work/examples/rsync/staging -mindepth 1 -delete task: [fetch] cut -f1 > /work/examples/rsync/.run/files.txt task: [retry] for i in 1 2 3 4 5; do @@ -70,8 +69,8 @@ task: [retry] for i in 1 2 3 4 5; do s=$((15 * 2 ** i + RANDOM % (2 * 15 + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s done; exit $rc -task: [verify] cut -f1 | grep -E '^/(tlpkg/texlive\.tlpdb|[^/]+)\.sha512$' | while IFS= read -r f; do (cd "$(dirname "$f")" && f=$(basename "$f") && shasum -a 512 -c "$f" && gpgv --status-fd 1 --keyring /work/examples/rsync/.run/tl/tlpkg/gpg/pubring.gpg "$f.asc" "$f" | awk '/^\[GNUPG:\] GOODSIG /{g=1} /^\[GNUPG:\] VALIDSIG .* $/{v=1} END{exit !(g&&v)}') || exit 1; done -task: [verify] test "$(grep -c '^/archive/' )" = 0 || { awk -F'\t' -v TL="" 'NR == FNR { if (index($1, TL "/archive/") == 1) want[$1] = 1; next } +task: [verify] cd /work/examples/rsync/staging && cut -f1 | grep -E '^/(tlpkg/texlive\.tlpdb|[^/]+)\.sha512$' | while IFS= read -r f; do (cd "$(dirname "$f")" && f=$(basename "$f") && shasum -a 512 -c "$f" && gpgv --status-fd 1 --keyring /work/examples/rsync/.run/tl/tlpkg/gpg/pubring.gpg "$f.asc" "$f" | awk -v k= '/^\[GNUPG:\] GOODSIG /{g=1} /^\[GNUPG:\] VALIDSIG / && $NF == k {v=1} END{exit !(g&&v)}') || exit 1; done +task: [verify] cd /work/examples/rsync/staging && { test "$(grep -c '^/archive/' )" = 0 || { awk -F'\t' -v TL="" 'NR == FNR { if (index($1, TL "/archive/") == 1) want[$1] = 1; next } /^name /{split($0, a, " "); n = a[2]} /^revision /{split($0, a, " "); r = a[2]} /^containerchecksum /{split($0, a, " "); emit(a[2], n)} /^doccontainerchecksum /{split($0, a, " "); emit(a[2], n ".doc")} @@ -80,18 +79,18 @@ task: [verify] test "$(grep -c '^/archive/' )" = 0 || { awk -F'\t' -v TL="" 'NR if (p in want) { print sum " " p; delete want[p] } if (q in want) { print sum " " q; delete want[q] } } END { for (p in want) { print "no checksum in the tlpdb for " p > "/dev/stderr"; bad = 1 } exit bad }' \ - /work/examples/rsync/.run/tl/tlpkg/texlive.tlpdb > /work/examples/rsync/.run/tl/sums.txt && shasum -a 512 -c --quiet /work/examples/rsync/.run/tl/sums.txt; } -task: [verify] test ! -f /tlpkg/texlive.tlpdb || { cmp /tlpkg/texlive.tlpdb /work/examples/rsync/.run/tl/tlpkg/texlive.tlpdb && xz -dc /tlpkg/texlive.tlpdb.xz | cmp - /tlpkg/texlive.tlpdb && { cut -f1 /work/examples/rsync/.run/applied.txt; cut -f1 ; } | LC_ALL=C sort -u | LC_ALL=C comm -23 - /work/examples/rsync/.run/deleted.txt > /work/examples/rsync/.run/tl/after.txt && awk -v TL="" '/^name /{n=$2} + /work/examples/rsync/.run/tl/tlpkg/texlive.tlpdb > /work/examples/rsync/.run/tl/sums.txt && shasum -a 512 -c --quiet /work/examples/rsync/.run/tl/sums.txt; }; } +task: [verify] cd /work/examples/rsync/staging && { test ! -f /tlpkg/texlive.tlpdb || { cmp /tlpkg/texlive.tlpdb /work/examples/rsync/.run/tl/tlpkg/texlive.tlpdb && xz -dc /tlpkg/texlive.tlpdb.xz | cmp - /tlpkg/texlive.tlpdb && { cut -f1 /work/examples/rsync/.run/applied.txt; cut -f1 ; } | LC_ALL=C sort -u | LC_ALL=C comm -23 - /work/examples/rsync/.run/deleted.txt > /work/examples/rsync/.run/tl/after.txt && awk -v TL="" '/^name /{n=$2} /^containerchecksum /{print TL "/archive/" n ".tar.xz"} /^doccontainerchecksum /{print TL "/archive/" n ".doc.tar.xz"} /^srccontainerchecksum /{print TL "/archive/" n ".source.tar.xz"}' /tlpkg/texlive.tlpdb \ -| LC_ALL=C sort | LC_ALL=C comm -23 - /work/examples/rsync/.run/tl/after.txt > /work/examples/rsync/.run/tl/missing.txt && test ! -s /work/examples/rsync/.run/tl/missing.txt; } +| LC_ALL=C sort | LC_ALL=C comm -23 - /work/examples/rsync/.run/tl/after.txt > /work/examples/rsync/.run/tl/missing.txt && test ! -s /work/examples/rsync/.run/tl/missing.txt; }; } task: [publish] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --exclude timestamp /work/examples/rsync/staging/ s3://example/ | tee -a /work/examples/rsync/.run/publish.txt task: [publish] test ! -f /work/examples/rsync/staging/timestamp || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/staging/timestamp s3://example/timestamp | tee -a /work/examples/rsync/.run/publish.txt task: [merge] (cd /work/examples/rsync/staging && find . -type f | sed 's|^\./||' | LC_ALL=C sort) > /work/examples/rsync/.run/landed.txt task: [merge] awk -F'\t' 'NR == FNR { drop[$1] = 1; next } !($1 in drop)' /work/examples/rsync/.run/applied.txt | cat - <(LC_ALL=C join -t "$(printf '\t')" /work/examples/rsync/.run/landed.txt ) | LC_ALL=C sort > /work/examples/rsync/.run/applied.new -task: [checkpoint] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rsync/.run/applied.new.xz -task: [checkpoint] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz +task: [push] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rsync/.run/applied.new.xz +task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz task: [checkpoint] mv /work/examples/rsync/.run/applied.new /work/examples/rsync/.run/applied.txt task: [checkpoint] find /work/examples/rsync/staging -mindepth 1 -delete task: [batches] test "$(ls /work/examples/rsync/.run/batch-*.txt 2>/dev/null | wc -l)" -le 4 || touch /work/examples/rsync/.run/chain @@ -101,28 +100,27 @@ task: [remove] for f in /work/examples/rsync/.run/del-*; do awk 'BEGIN { printf > "$f.json" && aws s3api delete-objects --bucket example --delete "file://$f.json" | tee -a /work/examples/rsync/.run/publish.txt | { test "$(grep -c Errors)" = 0; }; done task: [merge] (cd /work/examples/rsync/staging && find . -type f | sed 's|^\./||' | LC_ALL=C sort) > /work/examples/rsync/.run/landed.txt task: [merge] awk -F'\t' 'NR == FNR { drop[$1] = 1; next } !($1 in drop)' /work/examples/rsync/.run/deleted.txt /work/examples/rsync/.run/applied.txt | cat - <(LC_ALL=C join -t "$(printf '\t')" /work/examples/rsync/.run/landed.txt /work/examples/rsync/.run/deleted.txt) | LC_ALL=C sort > /work/examples/rsync/.run/applied.new -task: [checkpoint] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rsync/.run/applied.new.xz -task: [checkpoint] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz +task: [push] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rsync/.run/applied.new.xz +task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz task: [checkpoint] mv /work/examples/rsync/.run/applied.new /work/examples/rsync/.run/applied.txt task: [checkpoint] find /work/examples/rsync/staging -mindepth 1 -delete task: [reconcile] touch /work/examples/rsync/.run/rebuild-now task: [rebuild] { aws s3 ls --recursive s3://example/ || test $? -eq 1; } | awk '{ if (match($0, /^[^ ]+ +[^ ]+ +[0-9]+ /)) print substr($0, RLENGTH + 1) "\t" $3 }' | LC_ALL=C sort > /work/examples/rsync/.run/bucket.txt task: [rebuild] LC_ALL=C join -t "$(printf '\t')" -o 1.1,1.2,1.3,2.2 /work/examples/rsync/.run/upstream.txt /work/examples/rsync/.run/bucket.txt | awk -F'\t' -v OFS='\t' '$2 == $4 { print $1, $2, $3 }' > /work/examples/rsync/.run/applied.new -task: [rebuild] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rsync/.run/applied.new.xz -task: [rebuild] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz +task: [push] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rsync/.run/applied.new.xz +task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz task: [rebuild] mv /work/examples/rsync/.run/applied.new /work/examples/rsync/.run/applied.txt task: [rebuild] rm -f /work/examples/rsync/.run/rebuild-now -task: [reconcile] awk -F'\t' '!("" in s) { s[""] = 1; print "" } - { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/examples/rsync/.run/applied.txt | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > /work/examples/rsync/.run/index-keys.txt -task: [reconcile] cut -f1 /work/examples/rsync/.run/bucket.txt | grep -vE '^\.state/' | awk -v i="/" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i' | LC_ALL=C comm -23 - /work/examples/rsync/.run/index-keys.txt | LC_ALL=C comm -23 - /work/examples/rsync/.run/paths.txt > /work/examples/rsync/.run/orphans.txt +task: [reconcile] : > /work/examples/rsync/.run/index-keys.txt +task: [reconcile] cut -f1 /work/examples/rsync/.run/bucket.txt | grep -vE '^\.state/' | LC_ALL=C comm -23 - /work/examples/rsync/.run/index-keys.txt | LC_ALL=C comm -23 - /work/examples/rsync/.run/paths.txt > /work/examples/rsync/.run/orphans.txt task: [remove] rm -f /work/examples/rsync/.run/del-*; split -l 1000 -a 4 /work/examples/rsync/.run/orphans.txt /work/examples/rsync/.run/del- task: [remove] for f in /work/examples/rsync/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \ > "$f.json" && aws s3api delete-objects --bucket example --delete "file://$f.json" | tee -a /work/examples/rsync/.run/publish.txt | { test "$(grep -c Errors)" = 0; }; done task: [merge] (cd /work/examples/rsync/staging && find . -type f | sed 's|^\./||' | LC_ALL=C sort) > /work/examples/rsync/.run/landed.txt task: [merge] awk -F'\t' 'NR == FNR { drop[$1] = 1; next } !($1 in drop)' /work/examples/rsync/.run/orphans.txt /work/examples/rsync/.run/applied.txt | cat - <(LC_ALL=C join -t "$(printf '\t')" /work/examples/rsync/.run/landed.txt /work/examples/rsync/.run/orphans.txt) | LC_ALL=C sort > /work/examples/rsync/.run/applied.new -task: [checkpoint] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rsync/.run/applied.new.xz -task: [checkpoint] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz +task: [push] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rsync/.run/applied.new.xz +task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz task: [checkpoint] mv /work/examples/rsync/.run/applied.new /work/examples/rsync/.run/applied.txt task: [checkpoint] find /work/examples/rsync/staging -mindepth 1 -delete task: [smoke] { grep -h '^upload:' /work/examples/rsync/.run/publish.txt 2>/dev/null | sed 's|^upload: .* to s3://example/||' | head -3; } | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } diff --git a/toolbox.yml b/toolbox.yml index ca28018..6a63b3b 100644 --- a/toolbox.yml +++ b/toolbox.yml @@ -1,17 +1,18 @@ # The toolbox: every verb a katoptra mirror has whatever moves its bytes. A mirror # includes this file flattened, passes NAME, DESC and IMAGE on the include, keeps its -# identity in its own root vars, and defines two verbs: pipeline and plan-pipeline, -# both of which run inside IMAGE. A verb listed under the include's excludes: and -# defined in the mirror replaces the one here. +# identity in its own root vars, and has two verbs, pipeline and plan-pipeline, both of +# which run inside IMAGE: an engine include supplies them, and a mirror with no engine +# defines them. A verb listed under the include's excludes: and defined in the mirror +# replaces the one here. # # Rules this file keeps: no vars: default for anything a mirror owns (it would shadow # the mirror's value; defaults go inline), and host-side and in-container verbs share # one namespace, so the names below are reserved, and so is the engine vocabulary a -# second include adds beside them: list, normalise, state, rebuild, diff, split, -# batches, batch, fetch, publish, merge, checkpoint, remove, delete, reconcile, retry, -# and the hooks prepare, verify, index, smoke and report-engine. report-engine lives -# here as a no-op and in every engine, so an engine consumer's toolbox include lists it -# under excludes:. +# second include adds beside them: pipeline, plan-pipeline, list, normalise, state, +# pull, push, rebuild, diff, split, batches, batch, fetch, publish, merge, checkpoint, +# remove, delete, reconcile, retry, and the hooks prepare, verify, index, smoke, +# smoke-mirror and report-engine. report-engine lives here as a no-op and in every +# engine, so an engine consumer's toolbox include lists it under excludes:. version: '3' vars: @@ -68,7 +69,7 @@ tasks: " task image-clean Remove it, so the next task pulls or builds again" \ " task run -- Run anything inside it, with this repo at /work" \ " task op -- Same, with secrets resolved by name from op.env" \ - " task clean Delete .run, the Taskfile cache and every other ignored file" \ + " task clean Delete .run, staging and the Taskfile cache" \ {{.MENU}} \ "" @@ -120,20 +121,20 @@ tasks: else task run -- {{.CMD}}; fi clean: - desc: Delete every file git ignores + desc: Delete .run, the engine's staging tree and the Taskfile cache; nothing else, so a fixture kept in .git/info/exclude survives cmds: - - git clean -fdX + - rm -rf {{.RUN}} {{.ROOT_DIR}}/staging {{.ROOT_DIR}}/.task # ---- pipeline ---- # The words after -- are task vars for the pipeline inside the image: task sync -- # MAX_BATCHES=8 RECONCILE=true. A KEY=value before -- would set a host-side var only. - # The report and the fail ping happen inside the same container, so 1Password is read - # once per run whatever the outcome; a second op run for the ping would be a second read. + # The failure path runs inside the same container, so 1Password is read once per run + # whatever the outcome; a second op run for the ping would be a second read. sync: desc: 'One run, inside the toolbox: task sync -- MAX_BATCHES=8' cmds: - task: op - vars: {ARGS: "sh -c 'task pipeline \"$@\" || { task report STATUS=failed || true; task ping-fail; exit 1; }' sh {{range .CLI_ARGS_LIST}}{{shellQuote .}} {{end}}"} + vars: {ARGS: "sh -c 'task pipeline \"$@\" || { task failed; exit 1; }' sh {{range .CLI_ARGS_LIST}}{{shellQuote .}} {{end}}"} plan: desc: 'The read-only half, inside the toolbox: task plan -- RECONCILE=true' @@ -149,11 +150,16 @@ tasks: # dry run needs no secrets, so no names cross into the container. task writes the # rendered commands to stderr; the redirect inside the container folds them into # stdout, so the engine's own progress lines on the host's stderr stay out of the file. + # An empty directory is mounted over .run, so a var read from it at parse time (an + # engine's batch list, say) renders the same from any working tree, a plan just run + # included. cmds: - mkdir -p {{.RUN}} - >- - {{.ENGINE}} run --rm --user $(id -u):$(id -g) -v "{{.TOP}}":/work -w /work/{{.PREFIX}} - -e HOME=/tmp {{.IMAGE}} sh -c 'task --dry --force pipeline 2>&1' > {{.RUN}}/render.txt + empty=$(mktemp -d) && + {{.ENGINE}} run --rm --user $(id -u):$(id -g) -v "{{.TOP}}":/work -v "$empty":/work/{{.PREFIX}}.run -w /work/{{.PREFIX}} + -e HOME=/tmp {{.IMAGE}} sh -c 'task --dry --force pipeline 2>&1' > {{.RUN}}/render.txt; + rc=$?; rmdir "$empty" 2>/dev/null; exit $rc - cat {{.RUN}}/render.txt check: @@ -186,6 +192,14 @@ tasks: cmds: - test -z "$HEALTHCHECK_URL" || curl -fsS -m 10 --retry 3 -o /dev/null "$HEALTHCHECK_URL/fail" + # One verb, so the workflow can run it too, for a run the runner cut off: a timeout, a + # cancellation, an image that never pulled. sync runs it in the pipeline's own container. + failed: + desc: 'The failure path: the run summary marked failed, then the fail ping' + cmds: + - task report STATUS=failed || true + - {task: ping-fail} + # One table, three layers: the rows below, then the engine's, then the mirror's, each # a "| Label | value |" line appended to the same file. Every row tolerates a missing # file, because the report also runs after a failed pipeline. @@ -200,7 +214,7 @@ tasks: | | | |---|---| - | Started | $(date -u -d "@$s" '+%H:%M UTC' 2>/dev/null || date -u -r "$s" '+%H:%M UTC'), took $(( (now - s) / 60 )) min | + | Started | $(date -u -d "@$s" '+%H:%M UTC'), took $(( (now - s) / 60 )) min | | Image | {{.IMAGE}} | | Next run | $(test -f {{.RUN}}/chain && echo "queued now, work remains" || echo "at the schedule") | EOF