diff --git a/.github/actions/niks3-push/action.yml b/.github/actions/niks3-push/action.yml new file mode 100644 index 0000000..fb10feb --- /dev/null +++ b/.github/actions/niks3-push/action.yml @@ -0,0 +1,75 @@ +name: Push realized Nix closures to niks3 +description: Push already-built Nix installables with a short-lived GitHub Actions OIDC token. +inputs: + installables: + description: Newline-delimited, already-realized Nix installables to push. + required: true + server-url: + description: niks3 write-plane URL. + required: true + max-concurrent-uploads: + description: niks3 client upload concurrency. + required: false + default: "30" + niks3-cli-flake-ref: + description: Immutable flake reference that provides the niks3 CLI package. + required: false + default: github:Mic92/niks3/bb87dcb1b46a1f0c9426b733f4fe325245e386fa#niks3 +runs: + using: composite + steps: + - name: Push already-realized closures + shell: bash + env: + NIKS3_SERVER_URL: ${{ inputs.server-url }} + NIKS3_INSTALLABLES: ${{ inputs.installables }} + NIKS3_CLI_FLAKE_REF: ${{ inputs.niks3-cli-flake-ref }} + NIKS3_MAX_CONCURRENT: ${{ inputs.max-concurrent-uploads }} + run: | + set -euo pipefail + + mapfile -t installables < <(printf '%s\n' "$NIKS3_INSTALLABLES" | sed '/^[[:space:]]*$/d') + if [ "${#installables[@]}" -eq 0 ]; then + echo "installables input is empty" >&2 + exit 1 + fi + + # Deliberately do not build here. The caller must publish the exact closures that already + # passed its release gates on this runner. + paths_output="$(nix path-info "${installables[@]}")" + mapfile -t paths <<<"$paths_output" + if [ "${#paths[@]}" -ne "${#installables[@]}" ]; then + echo "resolved ${#paths[@]} store paths for ${#installables[@]} installables" >&2 + exit 1 + fi + for path in "${paths[@]}"; do + if [[ "$path" != /nix/store/* ]]; then + echo "nix path-info returned an invalid store path: $path" >&2 + exit 1 + fi + done + + # Realize and validate the pinned client before minting the short-lived OIDC token. + niks3_cli_path="$(nix build "$NIKS3_CLI_FLAKE_REF" --no-link --print-out-paths)" + if [[ "$niks3_cli_path" != /nix/store/* ]] || [ ! -x "$niks3_cli_path/bin/niks3" ]; then + echo "niks3 CLI did not resolve to one executable store path: $niks3_cli_path" >&2 + exit 1 + fi + + audience="$(jq -rn --arg value "$NIKS3_SERVER_URL" '$value | @uri')" + token="$(curl -fsSL \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${audience}" \ + | jq -er '.value | select(type == "string" and length > 0)')" + echo "::add-mask::${token}" + + umask 077 + token_file="$(mktemp "${RUNNER_TEMP:-/tmp}/niks3-auth-token.XXXXXX")" + trap 'rm -f -- "$token_file"' EXIT + printf '%s' "$token" >"$token_file" + unset token + + NIKS3_AUTH_TOKEN_FILE="$token_file" "$niks3_cli_path/bin/niks3" push \ + --server-url "$NIKS3_SERVER_URL" \ + --max-concurrent-uploads "$NIKS3_MAX_CONCURRENT" \ + "${paths[@]}" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a5109c2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,23 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: monday + time: "09:00" + timezone: America/Chicago + groups: + actions: + patterns: ["*"] + open-pull-requests-limit: 5 + - package-ecosystem: nix + directory: / + schedule: + interval: monthly + time: "09:00" + timezone: America/Chicago + groups: + flake-inputs: + patterns: ["*"] + open-pull-requests-limit: 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f8745c1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +name: ci +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3.21.9 + with: + extra-conf: | + extra-substituters = https://cache.secbear.dev https://secbear-cache-niks3.fly.dev + extra-trusted-public-keys = cache.secbear.dev-1:Pbeqskasb4M7FrHn+/kfnv1PCSvF0cJhl1snZ13Jn20= + fallback = true + connect-timeout = 5 + stalled-download-timeout = 15 + require-sigs = true + - name: Evaluate every declared system + run: nix flake check --all-systems --no-build + - name: Format and statically validate repository files + run: nix build .#checks.x86_64-linux.treefmt --no-link --print-build-logs diff --git a/.github/workflows/niks3-push.yml b/.github/workflows/niks3-push.yml index 7865d77..e6efaf5 100644 --- a/.github/workflows/niks3-push.yml +++ b/.github/workflows/niks3-push.yml @@ -10,6 +10,14 @@ on: description: niks3 write-plane URL. required: true type: string + substituter-url: + description: Space-separated signed Nix binary-cache URLs used before building. + required: true + type: string + substituter-public-key: + description: Public signing key for the Nix binary cache. + required: true + type: string max-concurrent-uploads: description: niks3 client upload concurrency. required: false @@ -19,10 +27,11 @@ on: description: Flake reference that provides the niks3 CLI package. required: false type: string - default: github:Mic92/niks3/v1.4.0#niks3 + default: github:Mic92/niks3/bb87dcb1b46a1f0c9426b733f4fe325245e386fa#niks3 jobs: push: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 60 permissions: contents: read id-token: write @@ -33,9 +42,19 @@ jobs: NIKS3_MAX_CONCURRENT: ${{ inputs.max-concurrent-uploads }} steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Install Nix - uses: cachix/install-nix-action@v31 + uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3.21.9 + with: + extra-conf: | + extra-substituters = ${{ inputs.substituter-url }} + extra-trusted-public-keys = ${{ inputs.substituter-public-key }} + fallback = true + connect-timeout = 5 + stalled-download-timeout = 15 + require-sigs = true - name: Build requested installables shell: bash run: | @@ -48,31 +67,65 @@ jobs: exit 1 fi - nix build --no-link "${installables[@]}" + nix build --no-link --print-build-logs "${installables[@]}" + - name: Realize the pinned niks3 client before minting a token + id: cli + shell: bash + run: | + set -euo pipefail + + cli_path="$(nix build "$NIKS3_CLI_FLAKE_REF" --no-link --print-out-paths)" + if [[ "$cli_path" != /nix/store/* ]] || [ ! -x "$cli_path/bin/niks3" ]; then + echo "niks3 CLI did not resolve to one executable store path: $cli_path" >&2 + exit 1 + fi + echo "path=${cli_path}" >> "$GITHUB_OUTPUT" - name: Fetch OIDC token id: oidc shell: bash run: | set -euo pipefail + + audience="$(jq -rn --arg value "$NIKS3_SERVER_URL" '$value | @uri')" token="$(curl -fsSL \ -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${NIKS3_SERVER_URL}" \ - | jq -r '.value')" + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${audience}" \ + | jq -er '.value | select(type == "string" and length > 0)')" echo "::add-mask::${token}" - echo "token=${token}" >> "$GITHUB_OUTPUT" + + umask 077 + token_file="$RUNNER_TEMP/niks3-auth-token" + printf '%s' "$token" >"$token_file" + echo "token-file=${token_file}" >> "$GITHUB_OUTPUT" - name: Push closures to niks3 shell: bash env: - NIKS3_AUTH_TOKEN: ${{ steps.oidc.outputs.token }} + NIKS3_AUTH_TOKEN_FILE: ${{ steps.oidc.outputs.token-file }} + NIKS3_CLI_PATH: ${{ steps.cli.outputs.path }} run: | set -euo pipefail + trap 'rm -f -- "$NIKS3_AUTH_TOKEN_FILE"' EXIT + + if [[ "$NIKS3_CLI_PATH" != /nix/store/* ]] || [ ! -x "$NIKS3_CLI_PATH/bin/niks3" ]; then + echo "niks3 CLI is missing or invalid: $NIKS3_CLI_PATH" >&2 + exit 1 + fi mapfile -t installables < <(printf '%s\n' "$NIKS3_INSTALLABLES" | sed '/^[[:space:]]*$/d') - mapfile -t paths < <(nix path-info "${installables[@]}") + paths_output="$(nix path-info "${installables[@]}")" + mapfile -t paths <<<"$paths_output" + if [ "${#paths[@]}" -ne "${#installables[@]}" ]; then + echo "resolved ${#paths[@]} store paths for ${#installables[@]} installables" >&2 + exit 1 + fi + for path in "${paths[@]}"; do + if [[ "$path" != /nix/store/* ]]; then + echo "nix path-info returned an invalid store path: $path" >&2 + exit 1 + fi + done - nix shell "$NIKS3_CLI_FLAKE_REF" -c \ - niks3 push \ + "$NIKS3_CLI_PATH/bin/niks3" push \ --server-url "$NIKS3_SERVER_URL" \ - --auth-token "$NIKS3_AUTH_TOKEN" \ --max-concurrent-uploads "$NIKS3_MAX_CONCURRENT" \ "${paths[@]}" diff --git a/README.md b/README.md index 86f9c37..8a6ea95 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,12 @@ The repo is intentionally split by control plane: - `fly/` owns the app deployment shape - `.envrc` provides the optional local bootstrap hook for secret injection -The public read path goes straight to R2. The Fly app only handles uploads, GC, and admin APIs. That keeps the running Fly VM small and cheap. OpenTofu manages only non-secret infrastructure; every real secret is injected through environment variables at runtime. +The primary public read path goes straight to R2. The Fly app handles uploads, GC, and admin APIs +and exposes a secondary read proxy. Consumers list R2 first, so successful R2 hits—including NAR +bodies—bypass the VM. Nix does probe the secondary cache after an ordinary primary miss, however, so +the Fly service and authenticated R2 API receive miss lookups as well as genuine edge-failure +fallback traffic. OpenTofu manages only non-secret infrastructure; every real secret is injected +through environment variables at runtime. ## Why This Shape @@ -102,7 +107,10 @@ Useful follow-up commands: - `just status` - `just down` -This repo ships a repo-managed pre-commit hook under `.githooks/pre-commit` for `fmt`, `lint`, and `nix flake check --no-build`. Clones must opt in with `git config core.hooksPath .githooks`. +This repo ships a repo-managed pre-commit hook under `.githooks/pre-commit` for `fmt`, `lint`, and +`nix flake check --no-build`. Clones must opt in with `git config core.hooksPath .githooks`. One +small GitHub Actions job independently evaluates every flake system and builds the Linux treefmt +check; it does not build or deploy the infrastructure. ## Secret Model @@ -187,39 +195,85 @@ Use: just gc ``` -That uses the upstream `niks3 gc` defaults: +By default, the wrapper preserves completed closures for 100 years (effectively indefinitely) and +only removes abandoned upload records: -- `--older-than 720h` (30 days) +- `--older-than 876000h` (100 years) - `--failed-uploads-older-than 6h` Override them when needed: ```sh -nix run .#gc -- --older-than 168h --failed-uploads-older-than 12h +nix run .#gc -- --older-than 720h --failed-uploads-older-than 12h ``` -This repo currently exposes GC as an on-demand command. It is not scheduled yet. +This repo currently exposes GC as an on-demand command. It is not scheduled yet. Do not introduce +age-based completed-closure deletion until the server supports and uses explicit release-root pins. ## CI Uploads -The reusable workflow is: +There are two upload surfaces: -- `.github/workflows/niks3-push.yml` +- `.github/actions/niks3-push/action.yml` pushes already-realized installables from the caller's + current runner. Prefer this as the final step of a trusted main CI job: it publishes the exact + store that passed the gates and performs no duplicate build. +- `.github/workflows/niks3-push.yml` is a convenient reusable workflow for callers that do not + already have a Nix build runner. It restores from the public cache, builds the requested roots on + its own runner, then pushes the closure delta. -It uses GitHub Actions OIDC for authentication — no static secret is needed in calling workflows. The workflow requests an OIDC token with the niks3 write-plane URL as the audience. The server validates the token against the subject patterns configured in `oidc_github_subject_patterns`. +Both use GitHub Actions OIDC for authentication — no static secret is needed in calling workflows. +They request an OIDC token with the niks3 write-plane URL as the audience. The server validates the +token against the subject patterns configured in `oidc_github_subject_patterns`. -The workflow intentionally makes both the write-plane URL and the `niks3` CLI flake reference explicit inputs, so callers do not accidentally target this repo's live infrastructure by default. The default CLI ref is pinned to the same upstream `niks3` version this repo currently tracks. +The workflow intentionally makes the write-plane URL, public read URL, and signing key explicit +inputs, so callers do not accidentally target this repo's live infrastructure by default. It reads +from that cache before building, then uploads only the missing closure delta. The default CLI ref is +an immutable upstream commit matching the `niks3` version this repo currently tracks. + +`niks3 push` recursively discovers each requested installable's full Nix closure, so callers should +list only their expensive roots (for example the dev shell, dependency-only derivations, and final +container). Store paths, NARs, narinfos, build logs, and realisations in those closures are uploaded +transactionally; listing every transitive dependency is unnecessary. + +Run this job only after all release gates pass on a trusted branch. Pull requests should consume the +public cache read-only and must never receive cache-write authority. > **Note:** `id-token: write` permission is required, which means fork pull requests cannot push to the cache. This is intentional. +Same-run publisher example: + +```yaml +permissions: + contents: read + id-token: write + +steps: + - uses: SecBear/nix-cache/.github/actions/niks3-push@ + with: + server-url: https://secbear-cache-niks3.fly.dev + installables: | + .#yourAlreadyBuiltPackage +``` + +The publishing job must not run pull-request code. Use a distinct main-only job or reusable-workflow +caller rather than granting `id-token: write` to a PR job and relying on a conditional upload step. + +Reusable-workflow example: + Minimal caller example from this repo: ```yaml jobs: cache: + permissions: + contents: read + id-token: write uses: ./.github/workflows/niks3-push.yml with: server-url: https://secbear-cache-niks3.fly.dev + substituter-url: >- + https://cache.secbear.dev https://secbear-cache-niks3.fly.dev + substituter-public-key: cache.secbear.dev-1:Pbeqskasb4M7FrHn+/kfnv1PCSvF0cJhl1snZ13Jn20= installables: | .#yourPackage .#yourOtherPackage @@ -230,22 +284,39 @@ Example from another repository: ```yaml jobs: cache: - uses: SecBear/nix-cache/.github/workflows/niks3-push.yml@main + permissions: + contents: read + id-token: write + uses: SecBear/nix-cache/.github/workflows/niks3-push.yml@ with: server-url: https://secbear-cache-niks3.fly.dev + substituter-url: >- + https://cache.secbear.dev https://secbear-cache-niks3.fly.dev + substituter-public-key: cache.secbear.dev-1:Pbeqskasb4M7FrHn+/kfnv1PCSvF0cJhl1snZ13Jn20= installables: | .#yourPackage ``` ## Operational Notes -- The public cache URL is the R2 custom domain, not the Fly app URL. +- The primary public cache URL is the R2 custom domain. Consumers also configure the Fly endpoint as + a signed secondary path so a stale or failed Cloudflare edge does not force a source rebuild. - The write/admin endpoint is `https://.fly.dev`. -- `niks3` read proxy stays disabled by default to keep Fly cost low. +- Keep the R2 URL first. Successful R2 hits never reach Fly, but every R2 miss is subsequently probed + through the read proxy; monitor that request load and remove the secondary after the Cloudflare + status policy has proven reliable if the duplicate miss traffic becomes material. - The Neon project and R2 S3 API credentials are managed outside OpenTofu by design. - First app creation on Fly requires billing/payment information on the account. - The repo expects provider/admin and runtime secrets to come from the environment. - The repo uses OpenTofu-compatible HCL. Plain Terraform users can adapt it, but the command surface is built around `tofu`. +- `niks3` v1.4.0 is pinned by both source revision and multi-architecture image digest. Upgrade to + v1.8.0 before the paid release, but snapshot Neon and verify its database migration plus upload, + read, signing, retry, and GC behavior before deploying it. +- OIDC subjects are cache-administrator authority because accepted uploads are signed. The deployed + policy must be the exact trusted Attune main ref; owner-wide or repository-wide wildcards are not + acceptable. The tracked example encodes that fail-closed shape. +- Keep an offline backup of the signing key. R2 data is reproducible; signing-key compromise requires + key rotation, cache purge, and consumer key replacement. ## Current Limits diff --git a/infra/opentofu/cloudflare.tf b/infra/opentofu/cloudflare.tf index 20b051b..6289dca 100644 --- a/infra/opentofu/cloudflare.tf +++ b/infra/opentofu/cloudflare.tf @@ -18,8 +18,10 @@ resource "cloudflare_r2_custom_domain" "cache" { # Edge-cache the binary cache. Cloudflare does NOT cache .narinfo/.nar.zst by default (non-standard # extensions -> cf-cache-status: DYNAMIC/MISS on every pull, ~275KB/s from R2 origin). NARs and -# narinfos are content-addressed and effectively immutable, so cache hits for 7 days; 404s (every -# narinfo probe for a path not yet pushed) only for 60s so a fresh push isn't masked by a cached miss. +# narinfos are content-addressed and effectively immutable, so successful responses cache for 7 +# days; 404s (every narinfo probe for a path not yet pushed) only for 60s so a fresh push isn't masked +# by a cached miss. Every other response is no-store: a transient auth, throttling, or server failure +# must never become a week-long edge outage. resource "cloudflare_ruleset" "cache_edge" { zone_id = var.cloudflare_zone_id name = "niks3 edge caching" @@ -35,11 +37,52 @@ resource "cloudflare_ruleset" "cache_edge" { cache = true edge_ttl = { mode = "override_origin" - default = 604800 # 7d: store paths are content-addressed, safe to pin - status_code_ttl = [{ - status_code = 404 - value = 60 - }] + default = 604800 # unmatched successful/redirect responses retain the immutable-object TTL + status_code_ttl = [ + { + status_code_range = { + from = 100 + to = 199 + } + value = -1 + }, + { + status_code_range = { + from = 200 + to = 299 + } + value = 604800 + }, + { + status_code_range = { + from = 300 + to = 303 + } + value = -1 + }, + { + status_code = 304 + value = 604800 + }, + { + status_code_range = { + from = 305 + to = 403 + } + value = -1 + }, + { + status_code = 404 + value = 60 + }, + { + status_code_range = { + from = 405 + to = 999 + } + value = -1 + }, + ] } } }] diff --git a/infra/opentofu/locals.tf b/infra/opentofu/locals.tf index 4537057..7edff63 100644 --- a/infra/opentofu/locals.tf +++ b/infra/opentofu/locals.tf @@ -13,9 +13,8 @@ locals { : "${var.cloudflare_account_id}.${var.r2_jurisdiction}.r2.cloudflarestorage.com" ) - # OIDC config written to the Fly guest via [[files]]. - # When no subject patterns are configured the providers map is empty, which - # effectively disables OIDC while keeping the [[files]] stanza unconditional. + # OIDC config written to the Fly guest via [[files]]. Deploy requires at least one exact trusted + # subject; current niks3 rejects an empty provider set rather than treating it as disabled. oidc_config_json = jsonencode({ providers = length(var.oidc_github_subject_patterns) == 0 ? {} : { github = { diff --git a/infra/opentofu/stack.auto.tfvars.example.json b/infra/opentofu/stack.auto.tfvars.example.json index c670335..b382bec 100644 --- a/infra/opentofu/stack.auto.tfvars.example.json +++ b/infra/opentofu/stack.auto.tfvars.example.json @@ -17,6 +17,6 @@ "fly_vm_memory": "256mb", "fly_swap_size_mb": 512, "niks3_s3_concurrency": 20, - "niks3_enable_read_proxy": false, - "oidc_github_subject_patterns": ["repo:replace-with-your-github-owner/*:*"] + "niks3_enable_read_proxy": true, + "oidc_github_subject_patterns": ["repo:replace-with-your-github-owner/replace-with-your-repository:ref:refs/heads/main"] } diff --git a/infra/opentofu/variables.tf b/infra/opentofu/variables.tf index 0f3ecb0..5840d62 100644 --- a/infra/opentofu/variables.tf +++ b/infra/opentofu/variables.tf @@ -124,13 +124,17 @@ variable "niks3_s3_concurrency" { } variable "niks3_enable_read_proxy" { - description = "Whether niks3 should proxy reads instead of redirecting clients to R2." + description = "Whether the Fly write plane also exposes a low-volume fallback read path." type = bool - default = false + default = true } variable "oidc_github_subject_patterns" { - description = "GitHub Actions OIDC subject patterns allowed to push. Example: [\"repo:MyOrg/*:*\"]. Empty list disables GitHub OIDC." + description = "Exact GitHub Actions OIDC subjects allowed to administer the cache. Use trusted main refs; do not use owner/repository wildcards." type = list(string) - default = [] + + validation { + condition = length(var.oidc_github_subject_patterns) > 0 + error_message = "oidc_github_subject_patterns must contain at least one exact trusted subject." + } } diff --git a/nix/flake/packages.nix b/nix/flake/packages.nix index 6f0b4be..d8ac7d7 100644 --- a/nix/flake/packages.nix +++ b/nix/flake/packages.nix @@ -10,7 +10,7 @@ let mkProjectScript = import ../lib/mk-project-script.nix { inherit pkgs; }; - niks3Image = "ghcr.io/mic92/niks3:v1.4.0"; + niks3Image = "ghcr.io/mic92/niks3:v1.4.0@sha256:fcbeef12785af2048250022a02e6830efe67dcc26eeb765ce87a0a93a7a28449"; niks3Cli = inputs.niks3.packages.${system}.niks3; tfDir = "infra/opentofu"; tfVarsFile = "infra/opentofu/stack.auto.tfvars.json"; @@ -281,6 +281,7 @@ gc = mkProjectScript { name = "gc"; runtimeInputs = [ + pkgs.coreutils pkgs.jq niks3Cli ]; @@ -302,9 +303,16 @@ app_name="$(jq -r '.fly_app_name' ${tfVarsFile})" server_url="''${NIKS3_SERVER_URL:-https://$app_name.fly.dev}" - exec ${niks3Cli}/bin/niks3 gc \ + umask 077 + token_file="$(mktemp "''${TMPDIR:-/tmp}/niks3-auth-token.XXXXXX")" + trap 'rm -f -- "$token_file"' EXIT + printf '%s' "$NIKS3_API_TOKEN" >"$token_file" + unset NIKS3_API_TOKEN + + NIKS3_AUTH_TOKEN_FILE="$token_file" ${niks3Cli}/bin/niks3 gc \ --server-url "$server_url" \ - --auth-token "$NIKS3_API_TOKEN" \ + --older-than 876000h \ + --failed-uploads-older-than 6h \ "$@" ''; };