From 0a28f8e1b0b885fdff3eb3eadb48382b63d50942 Mon Sep 17 00:00:00 2001 From: Bryce Thorpe Date: Mon, 10 Aug 2026 10:32:25 -0500 Subject: [PATCH 1/7] ci: harden and self-verify cache publishing --- .github/dependabot.yml | 23 +++++++++++++++++++++++ .github/workflows/ci.yml | 29 +++++++++++++++++++++++++++++ .github/workflows/niks3-push.yml | 31 ++++++++++++++++++++++++------- README.md | 24 +++++++++++++++++++++--- 4 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ci.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a5109c2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,23 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: monday + time: "09:00" + timezone: America/Chicago + groups: + actions: + patterns: ["*"] + open-pull-requests-limit: 5 + - package-ecosystem: nix + directory: / + schedule: + interval: monthly + time: "09:00" + timezone: America/Chicago + groups: + flake-inputs: + patterns: ["*"] + open-pull-requests-limit: 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..30fe088 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: ci +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3.21.9 + with: + extra-conf: | + extra-substituters = https://cache.secbear.dev + extra-trusted-public-keys = cache.secbear.dev-1:Pbeqskasb4M7FrHn+/kfnv1PCSvF0cJhl1snZ13Jn20= + - name: Evaluate every declared system + run: nix flake check --all-systems --no-build + - name: Format and statically validate repository files + run: nix build .#checks.x86_64-linux.treefmt --no-link --print-build-logs diff --git a/.github/workflows/niks3-push.yml b/.github/workflows/niks3-push.yml index 7865d77..0e6ce9c 100644 --- a/.github/workflows/niks3-push.yml +++ b/.github/workflows/niks3-push.yml @@ -10,6 +10,14 @@ on: description: niks3 write-plane URL. required: true type: string + substituter-url: + description: Public Nix binary-cache URL used before building. + required: true + type: string + substituter-public-key: + description: Public signing key for the Nix binary cache. + required: true + type: string max-concurrent-uploads: description: niks3 client upload concurrency. required: false @@ -19,10 +27,11 @@ on: description: Flake reference that provides the niks3 CLI package. required: false type: string - default: github:Mic92/niks3/v1.4.0#niks3 + default: github:Mic92/niks3/bb87dcb1b46a1f0c9426b733f4fe325245e386fa#niks3 jobs: push: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 60 permissions: contents: read id-token: write @@ -33,9 +42,15 @@ jobs: NIKS3_MAX_CONCURRENT: ${{ inputs.max-concurrent-uploads }} steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Install Nix - uses: cachix/install-nix-action@v31 + uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3.21.9 + with: + extra-conf: | + extra-substituters = ${{ inputs.substituter-url }} + extra-trusted-public-keys = ${{ inputs.substituter-public-key }} - name: Build requested installables shell: bash run: | @@ -48,16 +63,18 @@ jobs: exit 1 fi - nix build --no-link "${installables[@]}" + nix build --no-link --print-build-logs "${installables[@]}" - name: Fetch OIDC token id: oidc shell: bash run: | set -euo pipefail + + audience="$(jq -rn --arg value "$NIKS3_SERVER_URL" '$value | @uri')" token="$(curl -fsSL \ -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${NIKS3_SERVER_URL}" \ - | jq -r '.value')" + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${audience}" \ + | jq -er '.value | select(type == "string" and length > 0)')" echo "::add-mask::${token}" echo "token=${token}" >> "$GITHUB_OUTPUT" - name: Push closures to niks3 diff --git a/README.md b/README.md index 86f9c37..11e6e71 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,10 @@ Useful follow-up commands: - `just status` - `just down` -This repo ships a repo-managed pre-commit hook under `.githooks/pre-commit` for `fmt`, `lint`, and `nix flake check --no-build`. Clones must opt in with `git config core.hooksPath .githooks`. +This repo ships a repo-managed pre-commit hook under `.githooks/pre-commit` for `fmt`, `lint`, and +`nix flake check --no-build`. Clones must opt in with `git config core.hooksPath .githooks`. One +small GitHub Actions job independently evaluates every flake system and builds the Linux treefmt +check; it does not build or deploy the infrastructure. ## Secret Model @@ -208,7 +211,18 @@ The reusable workflow is: It uses GitHub Actions OIDC for authentication — no static secret is needed in calling workflows. The workflow requests an OIDC token with the niks3 write-plane URL as the audience. The server validates the token against the subject patterns configured in `oidc_github_subject_patterns`. -The workflow intentionally makes both the write-plane URL and the `niks3` CLI flake reference explicit inputs, so callers do not accidentally target this repo's live infrastructure by default. The default CLI ref is pinned to the same upstream `niks3` version this repo currently tracks. +The workflow intentionally makes the write-plane URL, public read URL, and signing key explicit +inputs, so callers do not accidentally target this repo's live infrastructure by default. It reads +from that cache before building, then uploads only the missing closure delta. The default CLI ref is +an immutable upstream commit matching the `niks3` version this repo currently tracks. + +`niks3 push` recursively discovers each requested installable's full Nix closure, so callers should +list only their expensive roots (for example the dev shell, dependency-only derivations, and final +container). Store paths, NARs, narinfos, build logs, and realisations in those closures are uploaded +transactionally; listing every transitive dependency is unnecessary. + +Run this job only after all release gates pass on a trusted branch. Pull requests should consume the +public cache read-only and must never receive cache-write authority. > **Note:** `id-token: write` permission is required, which means fork pull requests cannot push to the cache. This is intentional. @@ -220,6 +234,8 @@ jobs: uses: ./.github/workflows/niks3-push.yml with: server-url: https://secbear-cache-niks3.fly.dev + substituter-url: https://cache.secbear.dev + substituter-public-key: cache.secbear.dev-1:Pbeqskasb4M7FrHn+/kfnv1PCSvF0cJhl1snZ13Jn20= installables: | .#yourPackage .#yourOtherPackage @@ -230,9 +246,11 @@ Example from another repository: ```yaml jobs: cache: - uses: SecBear/nix-cache/.github/workflows/niks3-push.yml@main + uses: SecBear/nix-cache/.github/workflows/niks3-push.yml@ with: server-url: https://secbear-cache-niks3.fly.dev + substituter-url: https://cache.secbear.dev + substituter-public-key: cache.secbear.dev-1:Pbeqskasb4M7FrHn+/kfnv1PCSvF0cJhl1snZ13Jn20= installables: | .#yourPackage ``` From e75356efe7046755e2ef235717d624c71c196276 Mon Sep 17 00:00:00 2001 From: Bryce Thorpe Date: Mon, 10 Aug 2026 10:34:29 -0500 Subject: [PATCH 2/7] feat(ci): publish cache from the verified runner --- .github/actions/niks3-push/action.yml | 53 +++++++++++++++++++++++++++ README.md | 33 +++++++++++++++-- 2 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 .github/actions/niks3-push/action.yml diff --git a/.github/actions/niks3-push/action.yml b/.github/actions/niks3-push/action.yml new file mode 100644 index 0000000..2367ac8 --- /dev/null +++ b/.github/actions/niks3-push/action.yml @@ -0,0 +1,53 @@ +name: Push realized Nix closures to niks3 +description: Push already-built Nix installables with a short-lived GitHub Actions OIDC token. +inputs: + installables: + description: Newline-delimited, already-realized Nix installables to push. + required: true + server-url: + description: niks3 write-plane URL. + required: true + max-concurrent-uploads: + description: niks3 client upload concurrency. + required: false + default: "30" + niks3-cli-flake-ref: + description: Immutable flake reference that provides the niks3 CLI package. + required: false + default: github:Mic92/niks3/bb87dcb1b46a1f0c9426b733f4fe325245e386fa#niks3 +runs: + using: composite + steps: + - name: Push already-realized closures + shell: bash + env: + NIKS3_SERVER_URL: ${{ inputs.server-url }} + NIKS3_INSTALLABLES: ${{ inputs.installables }} + NIKS3_CLI_FLAKE_REF: ${{ inputs.niks3-cli-flake-ref }} + NIKS3_MAX_CONCURRENT: ${{ inputs.max-concurrent-uploads }} + run: | + set -euo pipefail + + mapfile -t installables < <(printf '%s\n' "$NIKS3_INSTALLABLES" | sed '/^[[:space:]]*$/d') + if [ "${#installables[@]}" -eq 0 ]; then + echo "installables input is empty" >&2 + exit 1 + fi + + # Deliberately do not build here. The caller must publish the exact closures that already + # passed its release gates on this runner. + mapfile -t paths < <(nix path-info "${installables[@]}") + + audience="$(jq -rn --arg value "$NIKS3_SERVER_URL" '$value | @uri')" + token="$(curl -fsSL \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${audience}" \ + | jq -er '.value | select(type == "string" and length > 0)')" + echo "::add-mask::${token}" + + nix shell "$NIKS3_CLI_FLAKE_REF" -c \ + niks3 push \ + --server-url "$NIKS3_SERVER_URL" \ + --auth-token "$token" \ + --max-concurrent-uploads "$NIKS3_MAX_CONCURRENT" \ + "${paths[@]}" diff --git a/README.md b/README.md index 11e6e71..7a92cef 100644 --- a/README.md +++ b/README.md @@ -205,11 +205,18 @@ This repo currently exposes GC as an on-demand command. It is not scheduled yet. ## CI Uploads -The reusable workflow is: +There are two upload surfaces: -- `.github/workflows/niks3-push.yml` +- `.github/actions/niks3-push/action.yml` pushes already-realized installables from the caller's + current runner. Prefer this as the final step of a trusted main CI job: it publishes the exact + store that passed the gates and performs no duplicate build. +- `.github/workflows/niks3-push.yml` is a convenient reusable workflow for callers that do not + already have a Nix build runner. It restores from the public cache, builds the requested roots on + its own runner, then pushes the closure delta. -It uses GitHub Actions OIDC for authentication — no static secret is needed in calling workflows. The workflow requests an OIDC token with the niks3 write-plane URL as the audience. The server validates the token against the subject patterns configured in `oidc_github_subject_patterns`. +Both use GitHub Actions OIDC for authentication — no static secret is needed in calling workflows. +They request an OIDC token with the niks3 write-plane URL as the audience. The server validates the +token against the subject patterns configured in `oidc_github_subject_patterns`. The workflow intentionally makes the write-plane URL, public read URL, and signing key explicit inputs, so callers do not accidentally target this repo's live infrastructure by default. It reads @@ -226,6 +233,26 @@ public cache read-only and must never receive cache-write authority. > **Note:** `id-token: write` permission is required, which means fork pull requests cannot push to the cache. This is intentional. +Same-run publisher example: + +```yaml +permissions: + contents: read + id-token: write + +steps: + - uses: SecBear/nix-cache/.github/actions/niks3-push@ + with: + server-url: https://secbear-cache-niks3.fly.dev + installables: | + .#yourAlreadyBuiltPackage +``` + +The publishing job must not run pull-request code. Use a distinct main-only job or reusable-workflow +caller rather than granting `id-token: write` to a PR job and relying on a conditional upload step. + +Reusable-workflow example: + Minimal caller example from this repo: ```yaml From d051663382a95ac287ee07fc3850fc5d7bd65b2b Mon Sep 17 00:00:00 2001 From: Bryce Thorpe Date: Mon, 10 Aug 2026 10:37:30 -0500 Subject: [PATCH 3/7] fix(ci): fail closed on incomplete cache roots --- .github/actions/niks3-push/action.yml | 13 ++++++++++++- .github/workflows/niks3-push.yml | 13 ++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/actions/niks3-push/action.yml b/.github/actions/niks3-push/action.yml index 2367ac8..bc12105 100644 --- a/.github/actions/niks3-push/action.yml +++ b/.github/actions/niks3-push/action.yml @@ -36,7 +36,18 @@ runs: # Deliberately do not build here. The caller must publish the exact closures that already # passed its release gates on this runner. - mapfile -t paths < <(nix path-info "${installables[@]}") + paths_output="$(nix path-info "${installables[@]}")" + mapfile -t paths <<<"$paths_output" + if [ "${#paths[@]}" -ne "${#installables[@]}" ]; then + echo "resolved ${#paths[@]} store paths for ${#installables[@]} installables" >&2 + exit 1 + fi + for path in "${paths[@]}"; do + if [[ "$path" != /nix/store/* ]]; then + echo "nix path-info returned an invalid store path: $path" >&2 + exit 1 + fi + done audience="$(jq -rn --arg value "$NIKS3_SERVER_URL" '$value | @uri')" token="$(curl -fsSL \ diff --git a/.github/workflows/niks3-push.yml b/.github/workflows/niks3-push.yml index 0e6ce9c..4dbdb7b 100644 --- a/.github/workflows/niks3-push.yml +++ b/.github/workflows/niks3-push.yml @@ -85,7 +85,18 @@ jobs: set -euo pipefail mapfile -t installables < <(printf '%s\n' "$NIKS3_INSTALLABLES" | sed '/^[[:space:]]*$/d') - mapfile -t paths < <(nix path-info "${installables[@]}") + paths_output="$(nix path-info "${installables[@]}")" + mapfile -t paths <<<"$paths_output" + if [ "${#paths[@]}" -ne "${#installables[@]}" ]; then + echo "resolved ${#paths[@]} store paths for ${#installables[@]} installables" >&2 + exit 1 + fi + for path in "${paths[@]}"; do + if [[ "$path" != /nix/store/* ]]; then + echo "nix path-info returned an invalid store path: $path" >&2 + exit 1 + fi + done nix shell "$NIKS3_CLI_FLAKE_REF" -c \ niks3 push \ From 85fe9a7791512bff50247db69d6a9a8cd9652f50 Mon Sep 17 00:00:00 2001 From: Bryce Thorpe Date: Mon, 10 Aug 2026 10:46:00 -0500 Subject: [PATCH 4/7] chore(infra): fail closed around cache authority --- .github/actions/niks3-push/action.yml | 9 ++- .github/workflows/ci.yml | 4 ++ .github/workflows/niks3-push.yml | 14 ++++- README.md | 18 ++++-- infra/opentofu/cloudflare.tf | 57 ++++++++++++++++--- infra/opentofu/locals.tf | 5 +- infra/opentofu/stack.auto.tfvars.example.json | 2 +- infra/opentofu/variables.tf | 8 ++- nix/flake/packages.nix | 14 ++++- 9 files changed, 106 insertions(+), 25 deletions(-) diff --git a/.github/actions/niks3-push/action.yml b/.github/actions/niks3-push/action.yml index bc12105..a6cf8cf 100644 --- a/.github/actions/niks3-push/action.yml +++ b/.github/actions/niks3-push/action.yml @@ -56,9 +56,14 @@ runs: | jq -er '.value | select(type == "string" and length > 0)')" echo "::add-mask::${token}" - nix shell "$NIKS3_CLI_FLAKE_REF" -c \ + umask 077 + token_file="$(mktemp "${RUNNER_TEMP:-/tmp}/niks3-auth-token.XXXXXX")" + trap 'rm -f -- "$token_file"' EXIT + printf '%s' "$token" >"$token_file" + unset token + + NIKS3_AUTH_TOKEN_FILE="$token_file" nix shell "$NIKS3_CLI_FLAKE_REF" -c \ niks3 push \ --server-url "$NIKS3_SERVER_URL" \ - --auth-token "$token" \ --max-concurrent-uploads "$NIKS3_MAX_CONCURRENT" \ "${paths[@]}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30fe088..9b21bf2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,10 @@ jobs: extra-conf: | extra-substituters = https://cache.secbear.dev extra-trusted-public-keys = cache.secbear.dev-1:Pbeqskasb4M7FrHn+/kfnv1PCSvF0cJhl1snZ13Jn20= + fallback = true + connect-timeout = 5 + stalled-download-timeout = 15 + require-sigs = true - name: Evaluate every declared system run: nix flake check --all-systems --no-build - name: Format and statically validate repository files diff --git a/.github/workflows/niks3-push.yml b/.github/workflows/niks3-push.yml index 4dbdb7b..9eb7b90 100644 --- a/.github/workflows/niks3-push.yml +++ b/.github/workflows/niks3-push.yml @@ -51,6 +51,10 @@ jobs: extra-conf: | extra-substituters = ${{ inputs.substituter-url }} extra-trusted-public-keys = ${{ inputs.substituter-public-key }} + fallback = true + connect-timeout = 5 + stalled-download-timeout = 15 + require-sigs = true - name: Build requested installables shell: bash run: | @@ -76,13 +80,18 @@ jobs: "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${audience}" \ | jq -er '.value | select(type == "string" and length > 0)')" echo "::add-mask::${token}" - echo "token=${token}" >> "$GITHUB_OUTPUT" + + umask 077 + token_file="$RUNNER_TEMP/niks3-auth-token" + printf '%s' "$token" >"$token_file" + echo "token-file=${token_file}" >> "$GITHUB_OUTPUT" - name: Push closures to niks3 shell: bash env: - NIKS3_AUTH_TOKEN: ${{ steps.oidc.outputs.token }} + NIKS3_AUTH_TOKEN_FILE: ${{ steps.oidc.outputs.token-file }} run: | set -euo pipefail + trap 'rm -f -- "$NIKS3_AUTH_TOKEN_FILE"' EXIT mapfile -t installables < <(printf '%s\n' "$NIKS3_INSTALLABLES" | sed '/^[[:space:]]*$/d') paths_output="$(nix path-info "${installables[@]}")" @@ -101,6 +110,5 @@ jobs: nix shell "$NIKS3_CLI_FLAKE_REF" -c \ niks3 push \ --server-url "$NIKS3_SERVER_URL" \ - --auth-token "$NIKS3_AUTH_TOKEN" \ --max-concurrent-uploads "$NIKS3_MAX_CONCURRENT" \ "${paths[@]}" diff --git a/README.md b/README.md index 7a92cef..dc2676c 100644 --- a/README.md +++ b/README.md @@ -190,18 +190,20 @@ Use: just gc ``` -That uses the upstream `niks3 gc` defaults: +By default, the wrapper preserves completed closures for 100 years (effectively indefinitely) and +only removes abandoned upload records: -- `--older-than 720h` (30 days) +- `--older-than 876000h` (100 years) - `--failed-uploads-older-than 6h` Override them when needed: ```sh -nix run .#gc -- --older-than 168h --failed-uploads-older-than 12h +nix run .#gc -- --older-than 720h --failed-uploads-older-than 12h ``` -This repo currently exposes GC as an on-demand command. It is not scheduled yet. +This repo currently exposes GC as an on-demand command. It is not scheduled yet. Do not introduce +age-based completed-closure deletion until the server supports and uses explicit release-root pins. ## CI Uploads @@ -291,6 +293,14 @@ jobs: - First app creation on Fly requires billing/payment information on the account. - The repo expects provider/admin and runtime secrets to come from the environment. - The repo uses OpenTofu-compatible HCL. Plain Terraform users can adapt it, but the command surface is built around `tofu`. +- `niks3` v1.4.0 is pinned by both source revision and multi-architecture image digest. Upgrade to + v1.8.0 before the paid release, but snapshot Neon and verify its database migration plus upload, + read, signing, retry, and GC behavior before deploying it. +- OIDC subjects are cache-administrator authority because accepted uploads are signed. The deployed + policy must be the exact trusted Attune main ref; owner-wide or repository-wide wildcards are not + acceptable. The tracked example encodes that fail-closed shape. +- Keep an offline backup of the signing key. R2 data is reproducible; signing-key compromise requires + key rotation, cache purge, and consumer key replacement. ## Current Limits diff --git a/infra/opentofu/cloudflare.tf b/infra/opentofu/cloudflare.tf index 20b051b..6289dca 100644 --- a/infra/opentofu/cloudflare.tf +++ b/infra/opentofu/cloudflare.tf @@ -18,8 +18,10 @@ resource "cloudflare_r2_custom_domain" "cache" { # Edge-cache the binary cache. Cloudflare does NOT cache .narinfo/.nar.zst by default (non-standard # extensions -> cf-cache-status: DYNAMIC/MISS on every pull, ~275KB/s from R2 origin). NARs and -# narinfos are content-addressed and effectively immutable, so cache hits for 7 days; 404s (every -# narinfo probe for a path not yet pushed) only for 60s so a fresh push isn't masked by a cached miss. +# narinfos are content-addressed and effectively immutable, so successful responses cache for 7 +# days; 404s (every narinfo probe for a path not yet pushed) only for 60s so a fresh push isn't masked +# by a cached miss. Every other response is no-store: a transient auth, throttling, or server failure +# must never become a week-long edge outage. resource "cloudflare_ruleset" "cache_edge" { zone_id = var.cloudflare_zone_id name = "niks3 edge caching" @@ -35,11 +37,52 @@ resource "cloudflare_ruleset" "cache_edge" { cache = true edge_ttl = { mode = "override_origin" - default = 604800 # 7d: store paths are content-addressed, safe to pin - status_code_ttl = [{ - status_code = 404 - value = 60 - }] + default = 604800 # unmatched successful/redirect responses retain the immutable-object TTL + status_code_ttl = [ + { + status_code_range = { + from = 100 + to = 199 + } + value = -1 + }, + { + status_code_range = { + from = 200 + to = 299 + } + value = 604800 + }, + { + status_code_range = { + from = 300 + to = 303 + } + value = -1 + }, + { + status_code = 304 + value = 604800 + }, + { + status_code_range = { + from = 305 + to = 403 + } + value = -1 + }, + { + status_code = 404 + value = 60 + }, + { + status_code_range = { + from = 405 + to = 999 + } + value = -1 + }, + ] } } }] diff --git a/infra/opentofu/locals.tf b/infra/opentofu/locals.tf index 4537057..7edff63 100644 --- a/infra/opentofu/locals.tf +++ b/infra/opentofu/locals.tf @@ -13,9 +13,8 @@ locals { : "${var.cloudflare_account_id}.${var.r2_jurisdiction}.r2.cloudflarestorage.com" ) - # OIDC config written to the Fly guest via [[files]]. - # When no subject patterns are configured the providers map is empty, which - # effectively disables OIDC while keeping the [[files]] stanza unconditional. + # OIDC config written to the Fly guest via [[files]]. Deploy requires at least one exact trusted + # subject; current niks3 rejects an empty provider set rather than treating it as disabled. oidc_config_json = jsonencode({ providers = length(var.oidc_github_subject_patterns) == 0 ? {} : { github = { diff --git a/infra/opentofu/stack.auto.tfvars.example.json b/infra/opentofu/stack.auto.tfvars.example.json index c670335..dc367da 100644 --- a/infra/opentofu/stack.auto.tfvars.example.json +++ b/infra/opentofu/stack.auto.tfvars.example.json @@ -18,5 +18,5 @@ "fly_swap_size_mb": 512, "niks3_s3_concurrency": 20, "niks3_enable_read_proxy": false, - "oidc_github_subject_patterns": ["repo:replace-with-your-github-owner/*:*"] + "oidc_github_subject_patterns": ["repo:replace-with-your-github-owner/replace-with-your-repository:ref:refs/heads/main"] } diff --git a/infra/opentofu/variables.tf b/infra/opentofu/variables.tf index 0f3ecb0..8b35d19 100644 --- a/infra/opentofu/variables.tf +++ b/infra/opentofu/variables.tf @@ -130,7 +130,11 @@ variable "niks3_enable_read_proxy" { } variable "oidc_github_subject_patterns" { - description = "GitHub Actions OIDC subject patterns allowed to push. Example: [\"repo:MyOrg/*:*\"]. Empty list disables GitHub OIDC." + description = "Exact GitHub Actions OIDC subjects allowed to administer the cache. Use trusted main refs; do not use owner/repository wildcards." type = list(string) - default = [] + + validation { + condition = length(var.oidc_github_subject_patterns) > 0 + error_message = "oidc_github_subject_patterns must contain at least one exact trusted subject." + } } diff --git a/nix/flake/packages.nix b/nix/flake/packages.nix index 6f0b4be..d8ac7d7 100644 --- a/nix/flake/packages.nix +++ b/nix/flake/packages.nix @@ -10,7 +10,7 @@ let mkProjectScript = import ../lib/mk-project-script.nix { inherit pkgs; }; - niks3Image = "ghcr.io/mic92/niks3:v1.4.0"; + niks3Image = "ghcr.io/mic92/niks3:v1.4.0@sha256:fcbeef12785af2048250022a02e6830efe67dcc26eeb765ce87a0a93a7a28449"; niks3Cli = inputs.niks3.packages.${system}.niks3; tfDir = "infra/opentofu"; tfVarsFile = "infra/opentofu/stack.auto.tfvars.json"; @@ -281,6 +281,7 @@ gc = mkProjectScript { name = "gc"; runtimeInputs = [ + pkgs.coreutils pkgs.jq niks3Cli ]; @@ -302,9 +303,16 @@ app_name="$(jq -r '.fly_app_name' ${tfVarsFile})" server_url="''${NIKS3_SERVER_URL:-https://$app_name.fly.dev}" - exec ${niks3Cli}/bin/niks3 gc \ + umask 077 + token_file="$(mktemp "''${TMPDIR:-/tmp}/niks3-auth-token.XXXXXX")" + trap 'rm -f -- "$token_file"' EXIT + printf '%s' "$NIKS3_API_TOKEN" >"$token_file" + unset NIKS3_API_TOKEN + + NIKS3_AUTH_TOKEN_FILE="$token_file" ${niks3Cli}/bin/niks3 gc \ --server-url "$server_url" \ - --auth-token "$NIKS3_API_TOKEN" \ + --older-than 876000h \ + --failed-uploads-older-than 6h \ "$@" ''; }; From b6a25ce50e4f98300e11780fa88a2386036b3a3e Mon Sep 17 00:00:00 2001 From: Bryce Thorpe Date: Mon, 10 Aug 2026 10:50:13 -0500 Subject: [PATCH 5/7] fix(ci): root closures and delay OIDC minting --- .github/actions/niks3-push/action.yml | 10 ++++++++-- .github/workflows/niks3-push.yml | 21 +++++++++++++++++++-- README.md | 6 ++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/actions/niks3-push/action.yml b/.github/actions/niks3-push/action.yml index a6cf8cf..fb10feb 100644 --- a/.github/actions/niks3-push/action.yml +++ b/.github/actions/niks3-push/action.yml @@ -49,6 +49,13 @@ runs: fi done + # Realize and validate the pinned client before minting the short-lived OIDC token. + niks3_cli_path="$(nix build "$NIKS3_CLI_FLAKE_REF" --no-link --print-out-paths)" + if [[ "$niks3_cli_path" != /nix/store/* ]] || [ ! -x "$niks3_cli_path/bin/niks3" ]; then + echo "niks3 CLI did not resolve to one executable store path: $niks3_cli_path" >&2 + exit 1 + fi + audience="$(jq -rn --arg value "$NIKS3_SERVER_URL" '$value | @uri')" token="$(curl -fsSL \ -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ @@ -62,8 +69,7 @@ runs: printf '%s' "$token" >"$token_file" unset token - NIKS3_AUTH_TOKEN_FILE="$token_file" nix shell "$NIKS3_CLI_FLAKE_REF" -c \ - niks3 push \ + NIKS3_AUTH_TOKEN_FILE="$token_file" "$niks3_cli_path/bin/niks3" push \ --server-url "$NIKS3_SERVER_URL" \ --max-concurrent-uploads "$NIKS3_MAX_CONCURRENT" \ "${paths[@]}" diff --git a/.github/workflows/niks3-push.yml b/.github/workflows/niks3-push.yml index 9eb7b90..6b57eac 100644 --- a/.github/workflows/niks3-push.yml +++ b/.github/workflows/niks3-push.yml @@ -68,6 +68,18 @@ jobs: fi nix build --no-link --print-build-logs "${installables[@]}" + - name: Realize the pinned niks3 client before minting a token + id: cli + shell: bash + run: | + set -euo pipefail + + cli_path="$(nix build "$NIKS3_CLI_FLAKE_REF" --no-link --print-out-paths)" + if [[ "$cli_path" != /nix/store/* ]] || [ ! -x "$cli_path/bin/niks3" ]; then + echo "niks3 CLI did not resolve to one executable store path: $cli_path" >&2 + exit 1 + fi + echo "path=${cli_path}" >> "$GITHUB_OUTPUT" - name: Fetch OIDC token id: oidc shell: bash @@ -89,10 +101,16 @@ jobs: shell: bash env: NIKS3_AUTH_TOKEN_FILE: ${{ steps.oidc.outputs.token-file }} + NIKS3_CLI_PATH: ${{ steps.cli.outputs.path }} run: | set -euo pipefail trap 'rm -f -- "$NIKS3_AUTH_TOKEN_FILE"' EXIT + if [[ "$NIKS3_CLI_PATH" != /nix/store/* ]] || [ ! -x "$NIKS3_CLI_PATH/bin/niks3" ]; then + echo "niks3 CLI is missing or invalid: $NIKS3_CLI_PATH" >&2 + exit 1 + fi + mapfile -t installables < <(printf '%s\n' "$NIKS3_INSTALLABLES" | sed '/^[[:space:]]*$/d') paths_output="$(nix path-info "${installables[@]}")" mapfile -t paths <<<"$paths_output" @@ -107,8 +125,7 @@ jobs: fi done - nix shell "$NIKS3_CLI_FLAKE_REF" -c \ - niks3 push \ + "$NIKS3_CLI_PATH/bin/niks3" push \ --server-url "$NIKS3_SERVER_URL" \ --max-concurrent-uploads "$NIKS3_MAX_CONCURRENT" \ "${paths[@]}" diff --git a/README.md b/README.md index dc2676c..5093838 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,9 @@ Minimal caller example from this repo: ```yaml jobs: cache: + permissions: + contents: read + id-token: write uses: ./.github/workflows/niks3-push.yml with: server-url: https://secbear-cache-niks3.fly.dev @@ -275,6 +278,9 @@ Example from another repository: ```yaml jobs: cache: + permissions: + contents: read + id-token: write uses: SecBear/nix-cache/.github/workflows/niks3-push.yml@ with: server-url: https://secbear-cache-niks3.fly.dev From d6549f9d2348fb94f0aced29e8d7f5a6acae065b Mon Sep 17 00:00:00 2001 From: Bryce Thorpe Date: Mon, 10 Aug 2026 11:10:15 -0500 Subject: [PATCH 6/7] infra: add resilient cache read fallback --- .github/workflows/ci.yml | 2 +- .github/workflows/niks3-push.yml | 2 +- README.md | 18 +++++++++++++----- infra/opentofu/stack.auto.tfvars.example.json | 2 +- infra/opentofu/variables.tf | 4 ++-- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b21bf2..f8745c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: - uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3.21.9 with: extra-conf: | - extra-substituters = https://cache.secbear.dev + extra-substituters = https://cache.secbear.dev https://secbear-cache-niks3.fly.dev extra-trusted-public-keys = cache.secbear.dev-1:Pbeqskasb4M7FrHn+/kfnv1PCSvF0cJhl1snZ13Jn20= fallback = true connect-timeout = 5 diff --git a/.github/workflows/niks3-push.yml b/.github/workflows/niks3-push.yml index 6b57eac..e6efaf5 100644 --- a/.github/workflows/niks3-push.yml +++ b/.github/workflows/niks3-push.yml @@ -11,7 +11,7 @@ on: required: true type: string substituter-url: - description: Public Nix binary-cache URL used before building. + description: Space-separated signed Nix binary-cache URLs used before building. required: true type: string substituter-public-key: diff --git a/README.md b/README.md index 5093838..611aaba 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,11 @@ The repo is intentionally split by control plane: - `fly/` owns the app deployment shape - `.envrc` provides the optional local bootstrap hook for secret injection -The public read path goes straight to R2. The Fly app only handles uploads, GC, and admin APIs. That keeps the running Fly VM small and cheap. OpenTofu manages only non-secret infrastructure; every real secret is injected through environment variables at runtime. +The primary public read path goes straight to R2. The Fly app handles uploads, GC, and admin APIs +and exposes a low-volume read-proxy fallback for edge failures. Consumers list R2 first, so healthy +reads still bypass the VM and the fallback does not change the normal cost or scaling path. OpenTofu +manages only non-secret infrastructure; every real secret is injected through environment variables +at runtime. ## Why This Shape @@ -266,7 +270,8 @@ jobs: uses: ./.github/workflows/niks3-push.yml with: server-url: https://secbear-cache-niks3.fly.dev - substituter-url: https://cache.secbear.dev + substituter-url: >- + https://cache.secbear.dev https://secbear-cache-niks3.fly.dev substituter-public-key: cache.secbear.dev-1:Pbeqskasb4M7FrHn+/kfnv1PCSvF0cJhl1snZ13Jn20= installables: | .#yourPackage @@ -284,7 +289,8 @@ jobs: uses: SecBear/nix-cache/.github/workflows/niks3-push.yml@ with: server-url: https://secbear-cache-niks3.fly.dev - substituter-url: https://cache.secbear.dev + substituter-url: >- + https://cache.secbear.dev https://secbear-cache-niks3.fly.dev substituter-public-key: cache.secbear.dev-1:Pbeqskasb4M7FrHn+/kfnv1PCSvF0cJhl1snZ13Jn20= installables: | .#yourPackage @@ -292,9 +298,11 @@ jobs: ## Operational Notes -- The public cache URL is the R2 custom domain, not the Fly app URL. +- The primary public cache URL is the R2 custom domain. Consumers also configure the Fly endpoint as + a signed, low-volume fallback so a stale or failed Cloudflare edge does not force a source rebuild. - The write/admin endpoint is `https://.fly.dev`. -- `niks3` read proxy stays disabled by default to keep Fly cost low. +- The `niks3` read proxy is enabled for fallback only. Keep the R2 URL first; using Fly as the primary + read path would couple downloads to the small write-plane VM and increase bandwidth cost. - The Neon project and R2 S3 API credentials are managed outside OpenTofu by design. - First app creation on Fly requires billing/payment information on the account. - The repo expects provider/admin and runtime secrets to come from the environment. diff --git a/infra/opentofu/stack.auto.tfvars.example.json b/infra/opentofu/stack.auto.tfvars.example.json index dc367da..b382bec 100644 --- a/infra/opentofu/stack.auto.tfvars.example.json +++ b/infra/opentofu/stack.auto.tfvars.example.json @@ -17,6 +17,6 @@ "fly_vm_memory": "256mb", "fly_swap_size_mb": 512, "niks3_s3_concurrency": 20, - "niks3_enable_read_proxy": false, + "niks3_enable_read_proxy": true, "oidc_github_subject_patterns": ["repo:replace-with-your-github-owner/replace-with-your-repository:ref:refs/heads/main"] } diff --git a/infra/opentofu/variables.tf b/infra/opentofu/variables.tf index 8b35d19..5840d62 100644 --- a/infra/opentofu/variables.tf +++ b/infra/opentofu/variables.tf @@ -124,9 +124,9 @@ variable "niks3_s3_concurrency" { } variable "niks3_enable_read_proxy" { - description = "Whether niks3 should proxy reads instead of redirecting clients to R2." + description = "Whether the Fly write plane also exposes a low-volume fallback read path." type = bool - default = false + default = true } variable "oidc_github_subject_patterns" { From d59a7563e2a02087b5bb0704bb908b679c5515f4 Mon Sep 17 00:00:00 2001 From: Bryce Thorpe Date: Mon, 10 Aug 2026 11:19:34 -0500 Subject: [PATCH 7/7] docs: record secondary cache miss traffic --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 611aaba..8a6ea95 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,11 @@ The repo is intentionally split by control plane: - `.envrc` provides the optional local bootstrap hook for secret injection The primary public read path goes straight to R2. The Fly app handles uploads, GC, and admin APIs -and exposes a low-volume read-proxy fallback for edge failures. Consumers list R2 first, so healthy -reads still bypass the VM and the fallback does not change the normal cost or scaling path. OpenTofu -manages only non-secret infrastructure; every real secret is injected through environment variables -at runtime. +and exposes a secondary read proxy. Consumers list R2 first, so successful R2 hits—including NAR +bodies—bypass the VM. Nix does probe the secondary cache after an ordinary primary miss, however, so +the Fly service and authenticated R2 API receive miss lookups as well as genuine edge-failure +fallback traffic. OpenTofu manages only non-secret infrastructure; every real secret is injected +through environment variables at runtime. ## Why This Shape @@ -299,10 +300,11 @@ jobs: ## Operational Notes - The primary public cache URL is the R2 custom domain. Consumers also configure the Fly endpoint as - a signed, low-volume fallback so a stale or failed Cloudflare edge does not force a source rebuild. + a signed secondary path so a stale or failed Cloudflare edge does not force a source rebuild. - The write/admin endpoint is `https://.fly.dev`. -- The `niks3` read proxy is enabled for fallback only. Keep the R2 URL first; using Fly as the primary - read path would couple downloads to the small write-plane VM and increase bandwidth cost. +- Keep the R2 URL first. Successful R2 hits never reach Fly, but every R2 miss is subsequently probed + through the read proxy; monitor that request load and remove the secondary after the Cloudflare + status policy has proven reliable if the duplicate miss traffic becomes material. - The Neon project and R2 S3 API credentials are managed outside OpenTofu by design. - First app creation on Fly requires billing/payment information on the account. - The repo expects provider/admin and runtime secrets to come from the environment.