diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca61794..1bf1dd1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -101,6 +101,10 @@ jobs: bash -n scripts/tests/mirror-publish-workflow-verify.sh bash -n scripts/publish-guard.sh bash -n scripts/publish-mirror.sh + shellcheck --shell=bash --severity=warning scripts/backfill-releases.sh + shellcheck --shell=bash --severity=warning scripts/tests/backfill-releases-verify.sh + bash -n scripts/backfill-releases.sh + bash -n scripts/tests/backfill-releases-verify.sh # format.sh's own fail-closed properties. Formatters are stubbed, so this is # hermetic and needs no Go toolchain — which is why it lives in this job # rather than Lint. It exists because the first cut of format.sh reported @@ -152,6 +156,18 @@ jobs: # out of the YAML and executed with `gh` shimmed, so this is hermetic. - name: Mirror-publish workflow harness (step bodies / shape / mutations) run: bash scripts/tests/mirror-publish-workflow-verify.sh + # The one-shot historical backfill (scripts/backfill-releases.sh): dry-run + # writes nothing, --apply makes exactly the expected writes and a second + # --apply none, binaries stop at the BINARY_KEEP boundary, a binary that + # disagrees with SHA256SUMS or a forbidden string in a body refuses by + # name, a failed read is could-not-tell. `gh` is a recording fake serving + # fixtures, so this is hermetic. The second step breaks one rule per copy + # of the script and demands the same suite go red — a rule the suite + # cannot see reddens the build. + - name: Release-backfill harness (zero-write dry-run / idempotent / fail-closed) + run: bash scripts/tests/backfill-releases-verify.sh + - name: Release-backfill harness — mutations (every anchored rule is load-bearing) + run: bash scripts/tests/backfill-releases-verify.sh --mutations test: timeout-minutes: 15 diff --git a/internal/cli/auth.go b/internal/cli/auth.go index 42f7aa3..9f6c068 100644 --- a/internal/cli/auth.go +++ b/internal/cli/auth.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "os" + "sort" "time" "github.com/spf13/cobra" @@ -23,6 +24,7 @@ import ( // backend doesn't support browser sign-in yet. func newLoginCmd() *cobra.Command { var envFlag string + var force bool cmd := &cobra.Command{ Use: "login", Annotations: runtimeClassFor(classBackend), @@ -32,15 +34,23 @@ on any device (your laptop or phone), sign in the way you already do (password, Google, or GitHub), and approve the code. The CLI stores a user token in ~/.tracebloc (mode 0600). +Already signed in to this backend? login says so and exits 0 without a +browser step — so re-running it is safe in a script or a runbook, and a +headless box is never asked to approve a code it already has. Pass +--force to re-authenticate anyway: switching accounts, or replacing a +session you believe is stale. + Works on a headless / SSH box — the browser and the CLI need not share a machine. Honors HTTP(S)_PROXY / NO_PROXY for corporate-proxy networks.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - return runLogin(cmd.Context(), printerFor(cmd), envFlag) + return runLogin(cmd.Context(), printerFor(cmd), envFlag, force) }, } cmd.Flags().StringVar(&envFlag, "env", "", "backend environment: dev|stg|prod (default: $TRACEBLOC_ENV, then legacy $CLIENT_ENV, then prod)") + cmd.Flags().BoolVar(&force, "force", false, + "start a new device flow even when this machine already holds a valid session") return cmd } @@ -52,7 +62,7 @@ var ( pollAfter = time.After ) -func runLogin(ctx context.Context, p *ui.Printer, envFlag string) error { +func runLogin(ctx context.Context, p *ui.Printer, envFlag string, force bool) error { cfg, err := config.Load() if err != nil { return &exitError{code: exitFailure, err: err} @@ -67,6 +77,20 @@ func runLogin(ctx context.Context, p *ui.Printer, envFlag string) error { "unknown backend environment %q — valid values are dev, stg, prod (default). "+ "Check --env / $TRACEBLOC_ENV (or legacy $CLIENT_ENV)", env)} } + // Before asking for a browser approval, USE the credentials already on disk + // (cli#651). Without this, "re-run login to be safe" — a reasonable thing for + // a script or a runbook to do — is a hard stop on any host without a browser, + // even though the session is valid and every other command would accept it. + if !force { + reused, err := reuseStoredSession(ctx, p, cfg, env) + if err != nil { + return err + } + if reused { + return nil + } + } + client := newAPIClient(env) p.Detailf("backend %s — requesting a device code …", client.BaseURL) @@ -132,6 +156,203 @@ func runLogin(ctx context.Context, p *ui.Printer, envFlag string) error { return nil } +// storedSessionFor returns the profile KEY and the profile of the session this +// machine already holds for env, or ("", nil) when it holds none. +// +// Two arms, because "a session for env" has two shapes on disk and answering +// only the first would leave the credentials for the second unused: +// +// - the CURRENT session, when it RESOLVES to env. This is the same predicate +// `auth status --check` uses (sessionEnv, not the raw cfg.CurrentEnv), so +// login's short-circuit and the installer's probe cannot disagree about what +// "signed in to this env" means. +// - failing that, env's OWN profile. Profiles are per-env (R10), so a machine +// signed in to prod can still hold a live dev token; `login --env dev` must +// find it rather than run a flow for a credential already on disk. +// +// The KEY comes back with the profile because Profiles is keyed on the RAW env +// string — a v1-migrated config stores `"Dev"` verbatim (config.migrateV1) while +// sessionEnv normalises for the comparison. Writing the reused session back +// under the key it was FOUND under is what keeps a second, lower-cased profile +// from being minted alongside it. +func storedSessionFor(cfg *config.Config, env string) (string, *config.Profile) { + if cfg.SignedIn() && sessionEnv(cfg) == env { + return cfg.CurrentEnv, cfg.Current() + } + return profileKeyed(cfg, env) +} + +// profileKeyed finds env's own profile by FOLDING the map's keys, not by +// indexing with the already-normalised target (Bugbot, PR #658). +// +// A plain `cfg.Profiles[env]` only matches a key that is already lower-cased, so +// a live token written under `"Dev"` went unseen the moment that profile stopped +// being the current one — and the flow that followed saved a SECOND profile under +// `"dev"`, leaving the original session stranded beside it on exactly the headless +// host cli#651 is about. Arm 1 of storedSessionFor hid this: it catches the raw +// key while it is current, so the gap only opens after a `login --env` elsewhere. +// +// Exact match wins, and the fold is a tie-break scanned in sorted order — with +// both `"Dev"` and `"dev"` on disk the answer must not depend on Go's randomised +// map iteration. +func profileKeyed(cfg *config.Config, env string) (string, *config.Profile) { + if p := cfg.Profiles[env]; p != nil && p.Token != "" { + return env, p + } + keys := make([]string, 0, len(cfg.Profiles)) + for k := range cfg.Profiles { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if p := cfg.Profiles[k]; p != nil && p.Token != "" && normalizeEnv(k) == env { + return k, p + } + } + return "", nil +} + +// sessionExpired reports whether a stored profile's own recorded expiry has +// already passed, and renders it for the message. +// +// expires_at is "when known" (config.Profile) and the device grant does not +// return one today, so this is usually absent and the verdict comes from the +// backend instead. That split is exactly why the two copy paths differ: a LOCAL +// expiry we can name to the second, and a 401 we can only report as "rejected", +// because the backend returns the same status for a token that expired and one +// that was revoked. Naming a cause we cannot distinguish would be worse than +// reporting the one fact we have. +// +// An unparseable timestamp is NOT treated as expired: the backend, not a +// malformed config field, gets to invalidate a session. +func sessionExpired(prof *config.Profile) (bool, string) { + if prof.ExpiresAt == "" { + return false, "" + } + t, err := time.Parse(time.RFC3339, prof.ExpiresAt) + if err != nil || time.Now().Before(t) { + return false, "" + } + return true, t.Format(time.RFC3339) +} + +// whoAmIVerdict is what a FAILED WhoAmI says about the stored session. The three +// arms are deliberately not collapsible: only one of them is a statement about +// the credential. +type whoAmIVerdict int + +const ( + // whoAmIUnverified — no verdict was ever reached: DNS, a refused connection, + // a 5xx. NOT evidence the session is bad, and must never be reported as if it + // were: telling someone to re-authenticate during an outage sends them to a + // browser step that cannot help. + whoAmIUnverified whoAmIVerdict = iota + // whoAmIRejected — the backend refused the credential (401/403). The one arm + // where signing in again is the answer. + whoAmIRejected + // whoAmIUpgradeRequired — a 426: this CLI is below the server's version floor. + // Says nothing about the session, and no amount of re-authenticating fixes it. + whoAmIUpgradeRequired +) + +// classifyWhoAmIError turns a failed WhoAmI into that verdict, returning the +// *api.UpgradeRequiredError alongside it so the caller can surface the server's +// own version floor rather than a paraphrase. +// +// Shared because both places that probe a stored session — login's short-circuit +// and `auth status --check` — have to draw the SAME three-way distinction, and +// two hand-written copies of it drift the first time a fourth case appears +// (review on PR #658). The copy stays at the call sites: the two commands answer +// different questions ("should I start a flow?" vs "what is this exit code?") and +// say so in different words. +func classifyWhoAmIError(err error) (whoAmIVerdict, *api.UpgradeRequiredError) { + var ue *api.UpgradeRequiredError + if errors.As(err, &ue) { + return whoAmIUpgradeRequired, ue + } + var ae *api.APIError + if errors.As(err, &ae) && + (ae.StatusCode == http.StatusUnauthorized || ae.StatusCode == http.StatusForbidden) { + return whoAmIRejected, nil + } + return whoAmIUnverified, nil +} + +// reuseStoredSession is login's "you are already signed in" short-circuit +// (cli#651). It reports whether login is DONE: true means the machine holds a +// session for env that the backend just accepted, and there is nothing to sign +// in to. False means fall through to the device flow — every such path first +// says WHY, so a user who expected the short-circuit learns whether their +// session expired, was rejected, or simply couldn't be checked. +// +// The session is confirmed against the backend rather than trusted off disk: a +// token that has been revoked is still a token on disk, and reporting it as a +// live session would send the user into the next command to discover otherwise. +// When the backend can't be reached we fall through to the flow, which is what +// login does today — the flow surfaces the network failure in its own words. +func reuseStoredSession(ctx context.Context, p *ui.Printer, cfg *config.Config, env string) (bool, error) { + key, prof := storedSessionFor(cfg, env) + if prof == nil { + return false, nil + } + if expired, at := sessionExpired(prof); expired { + p.Hintf("The session saved on this machine expired at %s — signing in again.", at) + return false, nil + } + + client := newAPIClient(env) + client.Token = prof.Token + p.Detailf("backend %s — checking the session already on this machine …", client.BaseURL) + id, err := client.WhoAmI(ctx) + if err != nil { + // Ctrl-C landing during the probe is the OPERATOR, not an unverifiable + // session: fall through and we print "signing in again", then fail + // RequestDeviceCode with exit 1 — where every other interrupt in login exits + // 130 silently. Checked before the classification below because a cancelled + // context surfaces on the HTTP call as a plain error, which would otherwise + // land in the "couldn't check" arm (Bugbot, PR #658; same guard, same + // reason, as pollForToken's). + if ctx.Err() != nil { + return false, &exitError{code: exitInterrupted} + } + switch verdict, ue := classifyWhoAmIError(err); verdict { + case whoAmIUpgradeRequired: + // Not a verdict on the session, and a fresh device flow would hit the same + // floor — surface the upgrade instruction instead of burning a browser + // approval on it. + return false, &exitError{code: exitFailure, err: ue} + case whoAmIRejected: + p.Hintf("The backend rejected the session saved on this machine — it expired or was revoked. Signing in again.") + default: // whoAmIUnverified + p.Hintf("Couldn't check the session saved on this machine (%v) — signing in again.", err) + } + return false, nil + } + + // The session is live. Adopt it as the active one: `login --env dev` from a + // prod session is a request to SWITCH, and answering "already signed in" + // without moving current_env would leave every following command on prod. + // Written under the key the profile was found under, never a re-derived one. + cfg.CurrentEnv = key + prof.Email, prof.FirstName = id.Email, id.FirstName + if err := cfg.Save(); err != nil { + return false, &exitError{code: exitFailure, err: err} + } + if id.Email != "" { + p.Successf("Already signed in as %s.", id.Email) + } else { + p.Successf("Already signed in.") + } + // Visible, not demoted to Detailf, and deliberately NOT routed through + // withSignInAdvice: this is the actionable half of an outcome the user did not + // ask for. Someone who typed `login` to switch accounts needs the next step + // here, and unlike the advice withSignInAdvice guards, it contradicts nothing + // the installer prints — the installer reaches this line only on its success + // path, where its own next step is to carry on provisioning, not to re-auth. + p.Hintf("Run `tracebloc login --force` to sign in again — switching accounts, or replacing a session you believe is stale.") + return true, nil +} + // pollDisposition is what the poll loop does with a failed PollToken call. type pollDisposition int @@ -514,16 +735,12 @@ func runAuthCheck(ctx context.Context, p *ui.Printer, envFlag string) error { // A 426 means the CLI is too old, not that the session is invalid — surface // the upgrade instruction (non-silent, so it shows even without --verbose) // instead of the "re-login" advice, which wouldn't help. - var ue *api.UpgradeRequiredError - if errors.As(err, &ue) { + verdict, ue := classifyWhoAmIError(err) + if verdict == whoAmIUpgradeRequired { return &exitError{code: exitFailure, err: ue} } if p.Verbose() { - // Only a 401/403 is genuinely a rejected token (where re-login helps); a - // network/DNS/5xx failure means we couldn't verify, not that the session - // is invalid — don't send the user to re-login for an outage. - var apiErr *api.APIError - if errors.As(err, &apiErr) && (apiErr.StatusCode == http.StatusUnauthorized || apiErr.StatusCode == http.StatusForbidden) { + if verdict == whoAmIRejected { p.Hintf("Signed-in token was rejected by the backend — run `tracebloc login`.") } else { p.Hintf("Couldn't verify your session with the backend (%v).", err) diff --git a/internal/cli/auth_test.go b/internal/cli/auth_test.go index f4b9f31..f2cffd7 100644 --- a/internal/cli/auth_test.go +++ b/internal/cli/auth_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "net" "net/http" "net/http/httptest" @@ -14,6 +15,7 @@ import ( "github.com/tracebloc/cli/internal/api" "github.com/tracebloc/cli/internal/config" + "github.com/tracebloc/cli/internal/ui" ) // withTestBackend points the login command at an httptest server (via the @@ -851,3 +853,465 @@ func TestLogin_ClearsStaleIdentityOnWhoAmIFailure(t *testing.T) { t.Errorf("stale identity leaked: FirstName=%q Email=%q (want both cleared)", prof.FirstName, prof.Email) } } + +// ── login's "already signed in" short-circuit (cli#651) ──────────────────────── +// +// The bug: `login` went straight to a device code even when the machine already +// held a valid session, which on a headless host is a dead end rather than an +// inconvenience — the credentials are on disk, but the command insists on a +// browser approval it has no way to complete. +// +// Every test below asserts on whether /device/code was requested, because THAT +// is the behaviour that matters: the copy is secondary to whether a browser step +// was demanded. + +// loginBackend serves the device flow + /userinfo/, counting device-code +// requests, and lets a test decide what /userinfo/ says about the session that +// is already on disk. +func loginBackend(t *testing.T, userinfo http.HandlerFunc) *int { + t.Helper() + var codes int + withTestBackend(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/device/code": + codes++ + _, _ = w.Write([]byte(`{"device_code":"dc","user_code":"WDJB-MJHT","verification_uri":"https://x/activate","expires_in":600,"interval":5}`)) + case "/device/token": + _, _ = w.Write([]byte(`{"token":"fresh_tok"}`)) + case "/userinfo/": + userinfo(w, r) + default: + t.Errorf("unexpected request path %s", r.URL.Path) + } + }) + return &codes +} + +// okUserinfo accepts whatever token is presented. +func okUserinfo(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"email":"ds@co","first_name":"Dana","account":"Acme"}`)) +} + +// TestLogin_ReusesValidSession is the issue's reproduction: a second `login` +// against a live session must exit 0 without requesting a device code. +func TestLogin_ReusesValidSession(t *testing.T) { + codes := loginBackend(t, okUserinfo) + saveSignedIn(t, "live_tok") // CurrentEnv=dev + + out, err := runCmd(t, "login", "--env", "dev") + if err != nil { + t.Fatalf("login over a valid session should exit 0, got: %v", err) + } + if *codes != 0 { + t.Errorf("requested %d device codes; a valid session must not start a flow", *codes) + } + if !strings.Contains(out, "Already signed in as ds@co") { + t.Errorf("expected the already-signed-in line, got:\n%s", out) + } + if !strings.Contains(out, "--force") { + t.Errorf("expected the --force opt-out to be named, got:\n%s", out) + } + // The session is kept, not replaced by the flow's token. + cfg, _ := config.Load() + if got := cfg.Current().Token; got != "live_tok" { + t.Errorf("token = %q, want the existing live_tok", got) + } +} + +// TestLogin_ForceStartsAFlowOverAValidSession: --force is the opt-out for +// switching accounts or replacing a session believed stale, so it must skip the +// short-circuit entirely — including the probe. +func TestLogin_ForceStartsAFlowOverAValidSession(t *testing.T) { + codes := loginBackend(t, okUserinfo) + saveSignedIn(t, "live_tok") + + out, err := runCmd(t, "login", "--env", "dev", "--force") + if err != nil { + t.Fatalf("login --force: %v", err) + } + if *codes != 1 { + t.Errorf("requested %d device codes, want 1 under --force", *codes) + } + if strings.Contains(out, "Already signed in") { + t.Errorf("--force must not short-circuit, got:\n%s", out) + } + cfg, _ := config.Load() + if got := cfg.Current().Token; got != "fresh_tok" { + t.Errorf("token = %q, want the re-authenticated fresh_tok", got) + } +} + +// TestLogin_RejectedSessionSaysSoThenSignsIn: a 401 is the backend REJECTING the +// stored credential — name that, then run the flow the user came for. +func TestLogin_RejectedSessionSaysSoThenSignsIn(t *testing.T) { + codes := loginBackend(t, func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") == "Bearer stale_tok" { + w.WriteHeader(http.StatusUnauthorized) + return + } + okUserinfo(w, r) + }) + saveSignedIn(t, "stale_tok") + + out, err := runCmd(t, "login", "--env", "dev") + if err != nil { + t.Fatalf("login after a rejected session: %v", err) + } + if *codes != 1 { + t.Errorf("requested %d device codes, want 1 after a rejected session", *codes) + } + if !strings.Contains(out, "rejected the session saved on this machine") { + t.Errorf("expected the rejection to be named before the new flow, got:\n%s", out) + } + cfg, _ := config.Load() + if got := cfg.Current().Token; got != "fresh_tok" { + t.Errorf("token = %q, want fresh_tok", got) + } +} + +// TestLogin_UnverifiableSessionIsNotCalledRejected: a 5xx means we COULDN'T +// CHECK, which is a different situation from a rejected credential — reporting +// it as a rejection would tell the user their session is bad when the backend is +// simply down. (Same distinction runAuthCheck draws; cli#651 asks login to draw +// it too.) +func TestLogin_UnverifiableSessionIsNotCalledRejected(t *testing.T) { + codes := loginBackend(t, func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") == "Bearer live_tok" { + w.WriteHeader(http.StatusInternalServerError) // reachable but erroring + return + } + okUserinfo(w, r) + }) + saveSignedIn(t, "live_tok") + + out, err := runCmd(t, "login", "--env", "dev") + if err != nil { + t.Fatalf("login after an unverifiable session: %v", err) + } + if *codes != 1 { + t.Errorf("requested %d device codes, want 1 when the session can't be checked", *codes) + } + if !strings.Contains(out, "Couldn't check the session saved on this machine") { + t.Errorf("expected a 'couldn't check' line, got:\n%s", out) + } + if strings.Contains(out, "rejected") { + t.Errorf("a 500 must not be reported as a rejected session, got:\n%s", out) + } +} + +// TestLogin_LocallyExpiredSessionNamesTheExpiryWithoutProbing: when the stored +// profile carries an expires_at that has passed, we know the answer locally — +// say "expired" (the one cause we can actually distinguish) and don't spend a +// round-trip presenting a credential we know is dead. +func TestLogin_LocallyExpiredSessionNamesTheExpiryWithoutProbing(t *testing.T) { + // Only the OLD token's presentation counts: login's own post-flow + // confirmation hits /userinfo/ too, with the token it just obtained. + presented := false + codes := loginBackend(t, func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") == "Bearer old_tok" { + presented = true + } + okUserinfo(w, r) + }) + past := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) + if err := (&config.Config{CurrentEnv: "dev", Profiles: map[string]*config.Profile{ + "dev": {Token: "old_tok", Email: "ds@co", ExpiresAt: past}, + }}).Save(); err != nil { + t.Fatal(err) + } + + out, err := runCmd(t, "login", "--env", "dev") + if err != nil { + t.Fatalf("login after an expired session: %v", err) + } + if presented { + t.Error("must not present a locally-expired token to the backend") + } + if *codes != 1 { + t.Errorf("requested %d device codes, want 1 after an expired session", *codes) + } + if !strings.Contains(out, "expired at "+past) { + t.Errorf("expected the expiry to be named, got:\n%s", out) + } +} + +// TestLogin_UnexpiredSessionIsStillProbed guards the other side of the expiry +// arm: an expires_at in the FUTURE must not be taken as proof on its own — a +// revoked token still has an unexpired timestamp on disk. +func TestLogin_UnexpiredSessionIsStillProbed(t *testing.T) { + probed := false + codes := loginBackend(t, func(w http.ResponseWriter, r *http.Request) { + probed = true + okUserinfo(w, r) + }) + future := time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + if err := (&config.Config{CurrentEnv: "dev", Profiles: map[string]*config.Profile{ + "dev": {Token: "live_tok", Email: "ds@co", ExpiresAt: future}, + }}).Save(); err != nil { + t.Fatal(err) + } + + if _, err := runCmd(t, "login", "--env", "dev"); err != nil { + t.Fatalf("login: %v", err) + } + if !probed { + t.Error("an unexpired session must still be confirmed with the backend") + } + if *codes != 0 { + t.Errorf("requested %d device codes; the confirmed session must short-circuit", *codes) + } +} + +// TestLogin_UpgradeRequiredDoesNotStartAFlow: a 426 says this CLI is below the +// server's version floor — a device flow would hit the same floor, so surface the +// upgrade instruction instead of demanding a browser approval that cannot help. +func TestLogin_UpgradeRequiredDoesNotStartAFlow(t *testing.T) { + codes := loginBackend(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUpgradeRequired) // 426 + _, _ = w.Write([]byte(`{"error":"upgrade_required","min_version":"1.2.3"}`)) + }) + saveSignedIn(t, "live_tok") + + _, err := runCmd(t, "login", "--env", "dev") + if err == nil || !strings.Contains(err.Error(), "too old") { + t.Fatalf("a 426 must surface the upgrade message, got: %v", err) + } + if *codes != 0 { + t.Errorf("requested %d device codes; a 426 must not start a flow", *codes) + } +} + +// TestLogin_ReuseAdoptsTheTargetEnvsOwnProfile: profiles are per-env (R10), so a +// machine signed in to prod can hold a live dev token. `login --env dev` must use +// it AND switch current_env — answering "already signed in" without moving the +// pointer would leave every following command talking to prod. +func TestLogin_ReuseAdoptsTheTargetEnvsOwnProfile(t *testing.T) { + codes := loginBackend(t, func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer dev_tok" { + t.Errorf("probed with %q, want the dev profile's token", r.Header.Get("Authorization")) + } + okUserinfo(w, r) + }) + if err := (&config.Config{CurrentEnv: "prod", Profiles: map[string]*config.Profile{ + "prod": {Token: "prod_tok", Email: "ds@co"}, + "dev": {Token: "dev_tok", Email: "ds@co"}, + }}).Save(); err != nil { + t.Fatal(err) + } + + if _, err := runCmd(t, "login", "--env", "dev"); err != nil { + t.Fatalf("login --env dev over a stored dev session: %v", err) + } + if *codes != 0 { + t.Errorf("requested %d device codes; the stored dev session must be reused", *codes) + } + cfg, _ := config.Load() + if cfg.CurrentEnv != "dev" { + t.Errorf("current_env = %q, want dev (the short-circuit must still switch env)", cfg.CurrentEnv) + } + if got := cfg.Profiles["prod"].Token; got != "prod_tok" { + t.Errorf("prod token = %q, want prod_tok left intact (R10)", got) + } +} + +// TestLogin_ReuseKeepsTheRawProfileKey: Profiles is keyed on the RAW env string, +// and a v1-migrated config stores it verbatim ("Dev"). Reusing that session must +// write back under the key it was FOUND under — deriving a fresh, lower-cased key +// would mint a second profile beside the real one and strand the token. +func TestLogin_ReuseKeepsTheRawProfileKey(t *testing.T) { + codes := loginBackend(t, okUserinfo) + if err := (&config.Config{CurrentEnv: "Dev", Profiles: map[string]*config.Profile{ + "Dev": {Token: "live_tok", Email: "ds@co"}, + }}).Save(); err != nil { + t.Fatal(err) + } + + if _, err := runCmd(t, "login", "--env", "dev"); err != nil { + t.Fatalf("login: %v", err) + } + if *codes != 0 { + t.Errorf("requested %d device codes; a `Dev`-keyed session still resolves to dev", *codes) + } + cfg, _ := config.Load() + if len(cfg.Profiles) != 1 { + t.Errorf("profiles = %v, want the single existing one (no duplicate key)", cfg.Profiles) + } + if p := cfg.Profiles["Dev"]; p == nil || p.Token != "live_tok" { + t.Errorf("the `Dev` profile lost its token: %+v", cfg.Profiles) + } +} + +// TestLogin_NoStoredSessionStillSignsIn: the short-circuit must be invisible on +// the path it doesn't apply to — a machine with no session gets the device flow +// exactly as before, with no extra probe. +func TestLogin_NoStoredSessionStillSignsIn(t *testing.T) { + probes := 0 + codes := loginBackend(t, func(w http.ResponseWriter, r *http.Request) { + probes++ + okUserinfo(w, r) + }) + + if _, err := runCmd(t, "login", "--env", "dev"); err != nil { + t.Fatalf("login on a fresh machine: %v", err) + } + if *codes != 1 { + t.Errorf("requested %d device codes, want 1", *codes) + } + // One probe only: login's own post-flow confirmation. A signed-out machine + // has nothing to check beforehand. + if probes != 1 { + t.Errorf("%d /userinfo/ calls, want 1 (the post-flow confirmation only)", probes) + } +} + +// TestLogin_ReuseFindsARawKeyedProfileThatIsNotCurrent (Bugbot, PR #658) is the +// gap TestLogin_ReuseKeepsTheRawProfileKey could not see. That test keeps the +// `"Dev"` profile CURRENT, so arm 1 of storedSessionFor catches it and the map +// lookup is never exercised. Once a `login --env` elsewhere moves current_env, +// only arm 2 is left — and indexing the map with the already-normalised target +// misses `"Dev"` entirely, starting a flow and saving a second `"dev"` profile +// beside a perfectly good session. +func TestLogin_ReuseFindsARawKeyedProfileThatIsNotCurrent(t *testing.T) { + codes := loginBackend(t, func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer dev_tok" { + t.Errorf("probed with %q, want the `Dev` profile's token", got) + } + okUserinfo(w, r) + }) + // Signed in to prod; the dev session exists under a v1-migrated raw key. + if err := (&config.Config{CurrentEnv: "prod", Profiles: map[string]*config.Profile{ + "prod": {Token: "prod_tok", Email: "ds@co"}, + "Dev": {Token: "dev_tok", Email: "ds@co"}, + }}).Save(); err != nil { + t.Fatal(err) + } + + if _, err := runCmd(t, "login", "--env", "dev"); err != nil { + t.Fatalf("login --env dev over a `Dev`-keyed session: %v", err) + } + if *codes != 0 { + t.Errorf("requested %d device codes; the `Dev` session must be found and reused", *codes) + } + cfg, _ := config.Load() + if cfg.CurrentEnv != "Dev" { + t.Errorf("current_env = %q, want the key the profile was FOUND under", cfg.CurrentEnv) + } + if _, dup := cfg.Profiles["dev"]; dup { + t.Errorf("a duplicate lower-cased profile was minted: %v", cfg.Profiles) + } + if p := cfg.Profiles["Dev"]; p == nil || p.Token != "dev_tok" { + t.Errorf("the `Dev` profile lost its token: %+v", cfg.Profiles) + } +} + +// TestProfileKeyed_ExactMatchWinsAndFoldIsDeterministic: with both `"dev"` and +// `"Dev"` on disk the answer must not ride on Go's randomised map iteration. +// Exact match wins; the fold is only a tie-break, scanned in sorted order. +func TestProfileKeyed_ExactMatchWinsAndFoldIsDeterministic(t *testing.T) { + cfg := &config.Config{Profiles: map[string]*config.Profile{ + "Dev": {Token: "raw_tok"}, + "dev": {Token: "exact_tok"}, + "DEV": {Token: "shouty_tok"}, + }} + for i := 0; i < 50; i++ { + key, prof := profileKeyed(cfg, "dev") + if key != "dev" || prof.Token != "exact_tok" { + t.Fatalf("iteration %d: got (%q, %q), want the exact `dev` match", i, key, prof.Token) + } + } + // With no exact key, the sorted fold must still answer the same way every time. + delete(cfg.Profiles, "dev") + for i := 0; i < 50; i++ { + key, _ := profileKeyed(cfg, "dev") + if key != "DEV" { // "DEV" sorts before "Dev" + t.Fatalf("iteration %d: fold returned %q, want a stable sorted pick", i, key) + } + } + // A profile with no token is not a session. + cfg.Profiles = map[string]*config.Profile{"Dev": {Email: "ds@co"}} + if key, prof := profileKeyed(cfg, "dev"); prof != nil { + t.Errorf("a tokenless profile matched: (%q, %+v)", key, prof) + } +} + +// TestLogin_CancelDuringTheProbeExits130 (Bugbot, PR #658): Ctrl-C landing on the +// new session probe is the operator, not an unverifiable session. Falling through +// printed "signing in again" and then failed RequestDeviceCode with exit 1, where +// every other interrupt in login exits 130 silently. +func TestLogin_CancelDuringTheProbeExits130(t *testing.T) { + t.Setenv("TRACEBLOC_CONFIG_DIR", t.TempDir()) + codes := 0 + ctx, cancel := context.WithCancel(context.Background()) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/userinfo/": + cancel() // the operator hits Ctrl-C mid-probe + <-r.Context().Done() + case "/device/code": + codes++ + _, _ = w.Write([]byte(`{"device_code":"dc","user_code":"X","verification_uri":"https://x/a","expires_in":600,"interval":5}`)) + } + })) + t.Cleanup(srv.Close) + orig := newAPIClient + newAPIClient = func(string) *api.Client { return &api.Client{BaseURL: srv.URL, HTTP: srv.Client()} } + t.Cleanup(func() { newAPIClient = orig }) + + saveSignedIn(t, "live_tok") // CurrentEnv=dev + err := runLogin(ctx, ui.New(io.Discard, ui.WithColor(false)), "dev", false) + + if got := ExitCodeFromError(err); got != exitInterrupted { + t.Fatalf("exit code = %d, want %d (a cancelled probe is an interrupt)", got, exitInterrupted) + } + if !IsSilentError(err) { + t.Errorf("an interrupt must exit quietly, got: %v", err) + } + if codes != 0 { + t.Errorf("requested %d device codes; Ctrl-C must not start a flow", codes) + } +} + +// TestClassifyWhoAmIError pins the three-way distinction both session probes +// share (review on PR #658). The arms are not interchangeable: only +// whoAmIRejected is a statement about the credential, so a 5xx landing in it +// would send someone to re-authenticate during an outage, and a 426 landing +// there would send them to a browser step that cannot lift a version floor. +func TestClassifyWhoAmIError(t *testing.T) { + for _, tc := range []struct { + name string + err error + want whoAmIVerdict + wantMin string // the server's floor, when the verdict carries one + }{ + {"401 is a rejection", &api.APIError{StatusCode: http.StatusUnauthorized}, whoAmIRejected, ""}, + {"403 is a rejection", &api.APIError{StatusCode: http.StatusForbidden}, whoAmIRejected, ""}, + {"426 is a version floor", &api.UpgradeRequiredError{MinVersion: "1.2.3"}, whoAmIUpgradeRequired, "1.2.3"}, + {"500 is not a verdict", &api.APIError{StatusCode: http.StatusInternalServerError}, whoAmIUnverified, ""}, + {"404 is not a rejection", &api.APIError{StatusCode: http.StatusNotFound}, whoAmIUnverified, ""}, + {"429 is not a rejection", &api.APIError{StatusCode: http.StatusTooManyRequests}, whoAmIUnverified, ""}, + {"a transport error is not a verdict", errors.New("dial tcp: no such host"), whoAmIUnverified, ""}, + {"a cancelled context is not a rejection", context.Canceled, whoAmIUnverified, ""}, + // Wrapped, because both call sites get their error back through the api + // client's own fmt.Errorf wrapping — matching on the concrete type only + // would silently demote every real verdict to "unverified". + {"wrapped 401 still a rejection", fmt.Errorf("confirming: %w", + &api.APIError{StatusCode: http.StatusUnauthorized}), whoAmIRejected, ""}, + {"wrapped 426 still a version floor", fmt.Errorf("confirming: %w", + &api.UpgradeRequiredError{MinVersion: "9.9.9"}), whoAmIUpgradeRequired, "9.9.9"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, ue := classifyWhoAmIError(tc.err) + if got != tc.want { + t.Errorf("verdict = %d, want %d", got, tc.want) + } + switch { + case tc.wantMin != "": + if ue == nil || ue.MinVersion != tc.wantMin { + t.Errorf("upgrade error = %+v, want MinVersion %q", ue, tc.wantMin) + } + case ue != nil: + t.Errorf("upgrade error = %+v, want nil", ue) + } + }) + } +} diff --git a/internal/cli/client.go b/internal/cli/client.go index d69dbf5..63cb335 100644 --- a/internal/cli/client.go +++ b/internal/cli/client.go @@ -164,12 +164,21 @@ func clientPrompter() prompter { // and the file is hand-written in fixtures) failed `auth status --check --env dev` // against a session that works perfectly. func sessionEnv(cfg *config.Config) string { - if e := strings.ToLower(strings.TrimSpace(cfg.CurrentEnv)); e != "" { + if e := normalizeEnv(cfg.CurrentEnv); e != "" { return e } return api.ResolveEnv("") } +// normalizeEnv is the trim+lower-case sessionEnv applies, as a pure function, so +// the one other place that has to COMPARE a raw stored env string — the profile +// lookup in auth.go, which reads the Profiles map's own keys — folds it exactly +// the same way. Two hand-rolled copies of this is how a `"Dev"` key stops +// matching a `dev` target. +func normalizeEnv(env string) string { + return strings.ToLower(strings.TrimSpace(env)) +} + // knownSessionEnv resolves the session env like sessionEnv and then REJECTS an // unrecognised value, instead of letting api.BaseURL fall it back to prod. Every // path that attaches the stored TOKEN — authedClient, logout's server-side revoke, diff --git a/internal/cli/testdata/golden/07-login.golden b/internal/cli/testdata/golden/07-login.golden index db8da7e..1b152ec 100644 --- a/internal/cli/testdata/golden/07-login.golden +++ b/internal/cli/testdata/golden/07-login.golden @@ -15,6 +15,12 @@ on any device (your laptop or phone), sign in the way you already do (password, Google, or GitHub), and approve the code. The CLI stores a user token in ~/.tracebloc (mode 0600). +Already signed in to this backend? login says so and exits 0 without a +browser step — so re-running it is safe in a script or a runbook, and a +headless box is never asked to approve a code it already has. Pass +--force to re-authenticate anyway: switching accounts, or replacing a +session you believe is stale. + Works on a headless / SSH box — the browser and the CLI need not share a machine. Honors HTTP(S)_PROXY / NO_PROXY for corporate-proxy networks. @@ -23,6 +29,7 @@ Usage: Flags: --env string backend environment: dev|stg|prod (default: $TRACEBLOC_ENV, then legacy $CLIENT_ENV, then prod) + --force start a new device flow even when this machine already holds a valid session -h, --help help for login Global Flags: diff --git a/internal/cli/testdata/golden/zz-all-strings.golden b/internal/cli/testdata/golden/zz-all-strings.golden index e4ccd85..1559a4c 100644 --- a/internal/cli/testdata/golden/zz-all-strings.golden +++ b/internal/cli/testdata/golden/zz-all-strings.golden @@ -136,6 +136,8 @@ screen. %s/%d are runtime placeholders. "A training run is allocated up to:" "Add --help to any command for the flags." "Add/resize a node to meet the job's requests, or lower RESOURCE_REQUESTS on jobs-manager." +"Already signed in as %s." +"Already signed in." "Already signed out." "Applies to your next training run; a run already going keeps its size." "Applying the resource change…" @@ -173,6 +175,7 @@ screen. %s/%d are runtime placeholders. "Copying %s" "Correlation id: %s" "Couldn't check for active training runs (%v) — continuing; the confirmation below still guards you." +"Couldn't check the session saved on this machine (%v) — signing in again." "Couldn't clear the stored active-client pointer (%v) — the on-disk config still names the revoked client; run `tracebloc logout` or remove it by hand." "Couldn't connect to your secure environment — check your kubeconfig/context." "Couldn't determine this client's namespace — skipped the Helm uninstall. If a release is still installed, re-run with --namespace ." @@ -352,6 +355,7 @@ screen. %s/%d are runtime placeholders. "Review" "Revoked this machine's credential — your secure environment %q stays on tracebloc as a record." "Run '%s --help' for the available commands." +"Run `tracebloc login --force` to sign in again — switching accounts, or replacing a session you believe is stale." "SELECT '%s',%s,COUNT(*),%s,%s FROM `%s`.`%s`" "SELECT r.table_name, COALESCE(r.task,'') FROM `%s`.`%s` r JOIN (SELECT table_name, MAX(started_at) ms FROM `%s`.`%s` WHERE task IS NOT NULL GROUP BY table_name) m ON r.table_name = m.table_name AND r.started_at = m.ms WHERE r.task IS NOT NULL" "SELECT table_name FROM information_schema.tables WHERE table_schema='%s' ORDER BY table_name" @@ -390,11 +394,13 @@ screen. %s/%d are runtime placeholders. "Target verified with tracebloc: %s (namespace %s) — cluster %s." "Task:" "Text a folder with labels.csv + texts/ e.g. %s" +"The backend rejected the session saved on this machine — it expired or was revoked. Signing in again." "The column holding the duration / time-to-event. e.g. time, tenure_days" "The dataset's catalog metadata is kept as a record on tracebloc, marked unavailable — never removed." "The ingestion hasn't started yet (usually a slow image pull or a busy cluster). It's queued to run once the cluster can schedule it — check on it with the command below." "The name you provided was only control characters — auto-naming this client instead." "The number of landmark points each sample is annotated with — dataset-specific. e.g. 17 for COCO human pose" +"The session saved on this machine expired at %s — signing in again." "The size your images already are, as WxH — tracebloc checks every image matches and never resizes. Press Enter to read it from your first image. e.g. 224x224" "The tracebloc CLI (your local data & config are kept — --keep-data)" "This CLI is out of date — update it: %s" @@ -475,6 +481,7 @@ screen. %s/%d are runtime placeholders. "authorized — confirming the token with the backend …" "auto-detect" "backend" +"backend %s — checking the session already on this machine …" "backend %s — requesting a device code …" "backfilling the cluster anchor onto the existing client: %w" "bucket bins the target before it leaves the cluster" diff --git a/scripts/RELEASE_CHECKLIST.md b/scripts/RELEASE_CHECKLIST.md index 38d6ec2..fdec18b 100644 --- a/scripts/RELEASE_CHECKLIST.md +++ b/scripts/RELEASE_CHECKLIST.md @@ -48,6 +48,44 @@ have to reverse-engineer the surface area on release day. pinned to the mirror's current default-branch head — the mirror's default branch keeps the last stable release. +8. Releases that predate the mirror are carried over ONCE, by hand, with + `scripts/backfill-releases.sh` (the workflow only publishes releases + cut after it exists). The decision it implements: every published + release gets its tag, its GitHub release and its text assets + (`install.sh`, `install.ps1`, `SHA256SUMS`, anything else SHA256SUMS + does not list); the binaries and their `.sig`/`.cert` only for the + newest `BINARY_KEEP` releases (default 10) — older pinned binary + URLs 404 on the mirror, and the answer is "re-run the installer". + Prereleases are skipped unless `--include-prerelease`. Mirror tags + are annotated RELEASE MARKERS on the mirror's default-branch head, + carrying the original date and message — the mirror has no source + commit to point at, and the annotation says so. Release notes are + the same fixed text the workflow writes (`--notes fixed`, the + default) plus a footer naming the original publish date — the + historical bodies are GitHub's generated pull-request lists, and + nearly every one carries strings the guard's report tier counts, + which the mirror should not repeat. `--notes source` carries the + source body instead, as an explicit opt-in. Every text asset and + every release body goes through `publish-guard.sh` first; every + binary is checked against the source's `SHA256SUMS`; anything + already on the mirror with the same SHA256 is skipped, so a re-run + writes nothing. Dry-run is the default: + + ```bash + MIRROR_REPO= scripts/backfill-releases.sh # plan + MIRROR_REPO= scripts/backfill-releases.sh --apply # write + # resume after a failure, or redo one release: + MIRROR_REPO= scripts/backfill-releases.sh --apply --from-tag vX.Y.Z + MIRROR_REPO= scripts/backfill-releases.sh --apply --only-tag vX.Y.Z + ``` + + Needs `gh` (token with write on the mirror), `jq`, `gitleaks`; set + `BACKFILL_EXTRA_FORBIDDEN` to a file with the private needle list + the workflow gets from its secret, or the string scan runs without + them. Exit 1 means at least one release was refused (the table says + which and why); exit 2 means a read did not complete and nothing was + written. The script's header carries the full contract. + GitHub Releases plus the cosign-verified `install.sh` are the install path — a Homebrew tap and the `install.tracebloc.io` vanity URL were considered and dropped diff --git a/scripts/backfill-releases.sh b/scripts/backfill-releases.sh new file mode 100644 index 0000000..46026ff --- /dev/null +++ b/scripts/backfill-releases.sh @@ -0,0 +1,575 @@ +#!/usr/bin/env bash +# ============================================================================= +# backfill-releases.sh — one-shot backfill of this repository's HISTORICAL +# releases onto the public deliverable mirror. +# +# .github/workflows/mirror-publish.yml publishes each NEW release to the +# mirror as it is cut. Releases that existed before the mirror did are carried +# over once, by this script, run by a human. Idempotent: a re-run over an +# already-backfilled mirror reads everything and writes nothing. +# +# THE DECISION (taken once, stated here so nobody re-derives it): +# * every published release gets its tag and its GitHub release on the +# mirror, with its TEXT assets — install.sh, install.ps1, SHA256SUMS and +# any other asset SHA256SUMS does not list; +# * BINARIES (the files SHA256SUMS lists) and their cosign companions +# (.sig, .cert) are carried only for the newest +# BINARY_KEEP releases (default 10). Older pinned binary URLs 404 on the +# mirror; the documented answer is "re-run the installer"; +# * prereleases and drafts are skipped unless --include-prerelease. +# +# HOW MIRROR TAGS ARE ANCHORED: the mirror holds a README, not the source, so +# no tag can point at the commit a release was built from. Each tag is created +# as an ANNOTATED tag on the mirror's default-branch head. The annotation +# carries the ORIGINAL date (the source tag's tagger date when the source tag +# is annotated, the release's created_at otherwise) and the original message, +# and says in plain words that the tag is a RELEASE MARKER on the mirror, not +# a source snapshot. A tag already on the mirror is accepted only if it points +# at a commit the mirror has; a dangling one is refused, never repointed. +# +# RELEASE NOTES: --notes fixed (DEFAULT) writes the same fixed text the +# workflow writes for new releases. Historical source bodies are GitHub's +# generated ones — merged pull requests by title — and nearly every one +# carries strings the guard's report tier counts, which the public mirror +# should not repeat; so the source body is an explicit opt-in: --notes source +# carries it, and it then goes through the guard's forbidden-string scan like +# any text asset (a hit refuses the release and names the tier). Either way a +# footer names the original publish date (GitHub does not let a created +# release carry a past date, so the footer and the tag annotation are where +# the date survives). +# +# WHAT IS REUSED: scripts/publish-mirror.sh `target` decides the mirror name +# (unset, malformed, or equal to the source is refused there — one rule, one +# place); scripts/publish-guard.sh scans every text asset and the notes with +# the repo's own .publish-forbidden (and the private needles from +# BACKFILL_EXTRA_FORBIDDEN, when given) plus gitleaks, before anything is +# uploaded. Binaries are opaque to a string scan by design; each one is +# verified against the SOURCE release's SHA256SUMS before upload and refused +# on a mismatch, naming the asset. Companions (.sig/.cert) are downloaded +# with their binary under --apply and scanned like text assets then. +# +# Usage: +# MIRROR_REPO=NAME scripts/backfill-releases.sh [--dry-run | --apply] +# [--from-tag TAG | --only-tag TAG] [--include-prerelease] +# [--notes fixed|source] [--strict] +# +# Environment: +# SOURCE_REPO OWNER/REPO to read releases from (default: the repository +# `gh repo view` reports for the current checkout) +# MIRROR_REPO bare repository name in the source's organisation; REQUIRED +# BINARY_KEEP how many of the newest releases carry binaries (default 10) +# BACKFILL_EXTRA_FORBIDDEN +# file of extra refuse-tier needles for the guard (the +# private list the workflow gets from a secret); optional +# PUBLISH_MIRROR_GIT_NAME / PUBLISH_MIRROR_GIT_EMAIL +# tagger identity on the mirror tags (default github-actions[bot]) +# GH_TOKEN gh's; must be able to write the mirror under --apply +# +# Modes: +# --dry-run DEFAULT. Every read runs, the text assets and notes of +# each release that would change are downloaded and put +# through the guard, the plan is printed, nothing is written. +# --apply performs the writes: tag, release, uploads. +# --from-tag TAG resume: process TAG and every release newer than it +# (releases are processed oldest to newest). +# --only-tag TAG process TAG alone. +# Both keep the BINARY_KEEP decision of the FULL list, so a +# partial run carries the same binaries a full run would. +# +# Exit 0 done (every release created or already present); 1 at least one +# release was REFUSED (the table says which and why; the rest went ahead); +# 2 COULD NOT TELL — a read that did not complete, a tool missing, an input +# malformed. "Cannot tell" stops the run at once and never writes. +# ============================================================================= +set -euo pipefail + +SCRIPTS_DIR="${BACKFILL_SCRIPTS_DIR:-$(cd "$(dirname "$0")" && pwd)}" +REPO_ROOT="$(cd "$SCRIPTS_DIR/.." && pwd)" +PUBLISH_MIRROR="$SCRIPTS_DIR/publish-mirror.sh" +PUBLISH_GUARD="$SCRIPTS_DIR/publish-guard.sh" +FORBIDDEN_LIST="$REPO_ROOT/.publish-forbidden" + +# die2 REASON — could-not-tell: the reason, then exit 2. The reason goes to +# STDERR on purpose: most callers (jq_of above all) sit inside "$(...)", where +# stdout is the variable being assigned — a stdout reason would be captured +# into it and never seen, leaving a bare exit 2. On stderr it reaches the +# operator either way, and the substitution's status 2 aborts the assignment +# under `set -e`. That abort is the ONLY thing ending the parent, so never +# put a die2-capable "$(...)" inside an && / || list or a `[ ]` test, where +# `set -e` is suspended — hoist it into its own assignment first (the +# per-release block does). +die2() { echo "::error::backfill-releases: COULD NOT TELL — $1 (nothing more is written)" >&2; exit 2; } # mutation-anchor: die2-stderr +note() { echo "backfill-releases: $1"; } + +# ---- arguments ----------------------------------------------------------------- +APPLY=0; FROM_TAG=""; ONLY_TAG=""; INCLUDE_PRE=0; NOTES_MODE=fixed; STRICT=0 # mutation-anchor: notes-default-fixed +while [ "$#" -gt 0 ]; do + case "$1" in + --dry-run) APPLY=0; shift ;; + --apply) APPLY=1; shift ;; + --from-tag) FROM_TAG="${2:-}"; shift 2 ;; + --only-tag) ONLY_TAG="${2:-}"; shift 2 ;; + --include-prerelease) INCLUDE_PRE=1; shift ;; + --notes) NOTES_MODE="${2:-}"; shift 2 ;; + --strict) STRICT=1; shift ;; + -h|--help) sed -n '2,/^# ====/p' "$0" | sed 's/^# \{0,2\}//'; exit 0 ;; + *) die2 "unknown argument '$1' (see --help)" ;; + esac +done +[ -z "$FROM_TAG" ] || [ -z "$ONLY_TAG" ] || die2 "--from-tag and --only-tag exclude each other" +case "$NOTES_MODE" in fixed|source) ;; *) die2 "--notes must be 'fixed' or 'source', not '$NOTES_MODE'" ;; esac +BINARY_KEEP="${BINARY_KEEP:-10}" +[[ "$BINARY_KEEP" =~ ^[0-9]+$ ]] || die2 "BINARY_KEEP '$BINARY_KEEP' is not a non-negative integer" +TAG_RE='^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$' +COMPANION_RE='\.(sig|cert)$' # .sig / .cert travel with their binary +[ -z "$FROM_TAG" ] || [[ "$FROM_TAG" =~ $TAG_RE ]] || die2 "--from-tag '$FROM_TAG' is not a release tag" +[ -z "$ONLY_TAG" ] || [[ "$ONLY_TAG" =~ $TAG_RE ]] || die2 "--only-tag '$ONLY_TAG' is not a release tag" + +# ---- tools ----------------------------------------------------------------------- +for t in gh jq git awk; do command -v "$t" >/dev/null 2>&1 || die2 "'$t' is not on PATH"; done +command -v "${PUBLISH_GUARD_GITLEAKS:-gitleaks}" >/dev/null 2>&1 || die2 "'${PUBLISH_GUARD_GITLEAKS:-gitleaks}' is not on PATH — the guard treats a missing scanner as could-not-tell, so nothing could be uploaded" +[ -f "$PUBLISH_MIRROR" ] || die2 "$PUBLISH_MIRROR is missing" +[ -f "$PUBLISH_GUARD" ] || die2 "$PUBLISH_GUARD is missing" +[ -r "$FORBIDDEN_LIST" ] || die2 "$FORBIDDEN_LIST is missing or unreadable — the scan has no rules" +if [ -n "${BACKFILL_EXTRA_FORBIDDEN:-}" ]; then + [ -s "$BACKFILL_EXTRA_FORBIDDEN" ] || die2 "BACKFILL_EXTRA_FORBIDDEN '$BACKFILL_EXTRA_FORBIDDEN' is missing or empty" +fi +if command -v sha256sum >/dev/null 2>&1; then + sha256_of() { sha256sum "$1" | cut -d' ' -f1; } +elif command -v shasum >/dev/null 2>&1; then + sha256_of() { shasum -a 256 "$1" | cut -d' ' -f1; } +else + die2 "neither sha256sum nor shasum is on PATH" +fi + +TMP="$(mktemp -d "${TMPDIR:-/tmp}/backfill-releases.XXXXXX")" && [ -d "$TMP" ] || die2 "could not create a scratch directory" +trap 'rm -rf "$TMP"' EXIT + +# ---- gh wrappers ----------------------------------------------------------------- +# gh_read OUTFILE ARGS... — a READ that must complete. Any failure is +# could-not-tell: an unreadable list is never an empty list. +gh_read() { + local out="$1"; shift + if ! gh "$@" >"$out" 2>"$TMP/gh.err"; then + die2 "gh $* failed: $(tr '\n' ' ' <"$TMP/gh.err")" + fi +} +# gh_read_maybe OUTFILE ARGS... — a READ where "not there" is an answer. +# Returns 0 on success, 1 on a clear HTTP 404 or the HTTP 409 GitHub gives for +# a commit read on an EMPTY repository; anything else is could-not-tell. +gh_read_maybe() { + local out="$1"; shift + if gh "$@" >"$out" 2>"$TMP/gh.err"; then return 0; fi + grep -qE 'HTTP 404|HTTP 409' "$TMP/gh.err" && return 1 + die2 "gh $* failed: $(tr '\n' ' ' <"$TMP/gh.err")" +} +# gh_write OUTFILE ARGS... — a WRITE (apply only). A failed write is fatal: +# the mirror may now be half-changed and the human decides, with the table. +gh_write() { + local out="$1"; shift + [ "$APPLY" -eq 1 ] || die2 "internal: gh_write reached in dry-run (gh $*)" + if ! gh "$@" >"$out" 2>"$TMP/gh.err"; then + die2 "gh $* failed: $(tr '\n' ' ' <"$TMP/gh.err") — re-run to resume; completed steps are skipped" + fi +} +# jq_of FILE FILTER — jq over a file that MUST parse; a malformed answer is +# could-not-tell, not an empty one. +jq_of() { jq -r "$2" "$1" 2>"$TMP/jq.err" || die2 "could not parse $1 with '$2': $(tr '\n' ' ' <"$TMP/jq.err")"; } + +# ---- source and mirror --------------------------------------------------------------- +SRC="${SOURCE_REPO:-}" +if [ -z "$SRC" ]; then + gh_read "$TMP/self.json" repo view --json nameWithOwner + SRC="$(jq_of "$TMP/self.json" '.nameWithOwner')" +fi +[[ "$SRC" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die2 "source repository '$SRC' is not OWNER/REPO" + +# The mirror name is decided by publish-mirror.sh `target`, so an unset or +# malformed name and a mirror equal to the source are refused by the same rule +# the workflow applies. Run directly, output captured to a file: its +# ::error:: line is the reason, and its exit status is ours. +rc=0 +bash "$PUBLISH_MIRROR" target --mirror "${MIRROR_REPO:-}" --source-repo "$SRC" >"$TMP/target.out" 2>&1 || rc=$? +if [ "$rc" -ne 0 ]; then cat "$TMP/target.out"; echo "::error::backfill-releases: REFUSED — the mirror target was refused above"; exit "$rc"; fi +MIRROR="$(tail -1 "$TMP/target.out")" + +gh_read "$TMP/mirror.json" api "repos/$MIRROR" +MIRROR_FULL="$(jq_of "$TMP/mirror.json" '.full_name')" +MIRROR_BRANCH="$(jq_of "$TMP/mirror.json" '.default_branch')" +if [ "$(printf '%s' "$MIRROR_FULL" | tr '[:upper:]' '[:lower:]')" = "$(printf '%s' "$SRC" | tr '[:upper:]' '[:lower:]')" ]; then + echo "::error::backfill-releases: REFUSED — mirror '$MIRROR_FULL' resolves to the source repository"; exit 1 +fi +[ -n "$MIRROR_BRANCH" ] && [ "$MIRROR_BRANCH" != null ] || die2 "mirror '$MIRROR' reports no default branch" +# Every mirror tag is anchored here. An empty mirror has no head: that is a +# refusal with instructions, not a guess. +if ! gh_read_maybe "$TMP/head.json" api "repos/$MIRROR/commits/$MIRROR_BRANCH"; then + echo "::error::backfill-releases: REFUSED — mirror '$MIRROR' has no commit on '$MIRROR_BRANCH' to anchor tags to; publish the README first"; exit 1 +fi +MIRROR_HEAD="$(jq_of "$TMP/head.json" '.sha')" +[[ "$MIRROR_HEAD" =~ ^[0-9a-f]{40}$ ]] || die2 "mirror head '$MIRROR_HEAD' is not a commit sha" + +# ---- the release list, derived from the API ----------------------------------------- +# --paginate concatenates one JSON array per page; `jq -s add` joins them. +gh_read "$TMP/src-pages.json" api --paginate "repos/$SRC/releases" # mutation-anchor: releases-read-fail-closed +jq -s 'add // []' "$TMP/src-pages.json" >"$TMP/src-releases.json" 2>"$TMP/jq.err" || die2 "release list of '$SRC' did not parse: $(tr '\n' ' ' <"$TMP/jq.err")" +# Newest first by created_at; drafts never; prereleases only when asked. +FILTER='[ .[] | select(.draft == false) | select($pre == 1 or .prerelease == false) ] | sort_by(.created_at) | reverse' # mutation-anchor: prerelease-filter +jq --argjson pre "$INCLUDE_PRE" "$FILTER" "$TMP/src-releases.json" >"$TMP/releases.json" 2>"$TMP/jq.err" || die2 "could not filter the release list: $(tr '\n' ' ' <"$TMP/jq.err")" +N_ALL="$(jq_of "$TMP/releases.json" 'length')" +[ "$N_ALL" -gt 0 ] || die2 "'$SRC' has no published release matching the filter — nothing to backfill is not a clean run" +jq -r '.[].tag_name' "$TMP/releases.json" >"$TMP/tags-newest-first.txt" +while IFS= read -r t; do [[ "$t" =~ $TAG_RE ]] || die2 "release tag '$t' on '$SRC' is not a release tag"; done <"$TMP/tags-newest-first.txt" +# The newest BINARY_KEEP of the FULL filtered list carry binaries — decided +# before --from-tag/--only-tag narrow the run, so a partial run agrees with a +# full one. +head -n "$BINARY_KEEP" "$TMP/tags-newest-first.txt" >"$TMP/tags-with-binaries.txt" # mutation-anchor: binary-keep +NEWEST_STABLE="$(jq_of "$TMP/releases.json" '[ .[] | select(.prerelease == false) ] | .[0].tag_name // ""')" +carries_binaries() { grep -qxF -- "$1" "$TMP/tags-with-binaries.txt"; } + +# Processing order: oldest to newest, so the mirror's release order reads like +# the source's and the newest stable release is created last. +sed -n '1!G;h;$p' "$TMP/tags-newest-first.txt" >"$TMP/tags-ordered.txt" +if [ -n "$ONLY_TAG" ]; then + grep -qxF -- "$ONLY_TAG" "$TMP/tags-ordered.txt" || die2 "--only-tag '$ONLY_TAG' is not a release of '$SRC' matching the filter" + printf '%s\n' "$ONLY_TAG" >"$TMP/tags-run.txt" +elif [ -n "$FROM_TAG" ]; then + grep -qxF -- "$FROM_TAG" "$TMP/tags-ordered.txt" || die2 "--from-tag '$FROM_TAG' is not a release of '$SRC' matching the filter" + awk -v t="$FROM_TAG" 'f || $0 == t { f = 1; print }' "$TMP/tags-ordered.txt" >"$TMP/tags-run.txt" +else + cp "$TMP/tags-ordered.txt" "$TMP/tags-run.txt" +fi +N_RUN="$(grep -c . "$TMP/tags-run.txt" || true)" + +# ---- what the mirror already has ---------------------------------------------------- +gh_read "$TMP/mirror-pages.json" api --paginate "repos/$MIRROR/releases" +jq -s 'add // []' "$TMP/mirror-pages.json" >"$TMP/mirror-releases.json" 2>"$TMP/jq.err" || die2 "release list of '$MIRROR' did not parse" +gh_read "$TMP/mirror-tag-pages.json" api --paginate "repos/$MIRROR/git/matching-refs/tags/" +jq -s 'add // []' "$TMP/mirror-tag-pages.json" >"$TMP/mirror-tags.json" 2>"$TMP/jq.err" || die2 "tag list of '$MIRROR' did not parse" + +MODE=dry-run; [ "$APPLY" -eq 1 ] && MODE=apply +note "source $SRC → mirror $MIRROR ($MIRROR_BRANCH @ ${MIRROR_HEAD:0:12}); $N_ALL release(s) match the filter, $N_RUN in this run; binaries for the newest $BINARY_KEEP; notes=$NOTES_MODE; mode=$MODE" +[ "$STRICT" -eq 0 ] || note "--strict: the guard's [strings-report] tier refuses" + +# ---- the guard's scratch source tree ------------------------------------------------- +# publish-guard.sh stages a tree from a git checkout by design. The backfill has +# no tree to publish, so it hands the guard a one-file scratch checkout and puts +# what matters — the text assets and the notes — in --assets, where guards +# 2–4 (forbidden paths, forbidden strings, gitleaks) read them. +SCRATCH="$TMP/scratch-src"; mkdir -p "$SCRATCH" +git -C "$SCRATCH" init -q || die2 "could not init the guard's scratch checkout" +printf 'backfill scratch tree\n' >"$SCRATCH/README.md" +# This runs on a human's machine: a global commit.gpgsign=true would try to +# sign the scratch commit as backfill@localhost, fail, and end the run before +# a single release is planned. The scratch commit is never published — unsigned. +git -C "$SCRATCH" add README.md && git -C "$SCRATCH" -c user.name=backfill -c user.email=backfill@localhost -c commit.gpgsign=false commit -q -m scratch || die2 "could not commit the guard's scratch checkout" # mutation-anchor: scratch-commit-unsigned +printf 'README.md\n' >"$TMP/include.txt" + +# run_guard ASSETS_DIR OUT_DIR — the guard over ASSETS_DIR. Returns the guard's +# exit status (0 clean, 1 refused, 2 could not tell); its output is in OUT_DIR.log. +run_guard() { + local assets="$1" out="$2" rc=0 + local -a args=(--source "$SCRATCH" --include "$TMP/include.txt" --forbidden "$FORBIDDEN_LIST" --out "$out" --assets "$assets") + [ -z "${BACKFILL_EXTRA_FORBIDDEN:-}" ] || args+=(--extra-forbidden "$BACKFILL_EXTRA_FORBIDDEN") + [ "$STRICT" -eq 0 ] || args+=(--strict) + bash "$PUBLISH_GUARD" "${args[@]}" >"$out.log" 2>&1 || rc=$? + return "$rc" +} + +# ---- per-release helpers ------------------------------------------------------------------- +# Decision files: one line per asset, `nameactionsha`, action one of +# upload | skip | compare | refuse. `compare` means the mirror has the asset and +# the source's digest is unknown, so the download is hashed before deciding. +count_action() { awk -F'\t' -v a="$2" '$2 == a { n++ } END { print n + 0 }' "$1"; } +in_sums() { awk -F'\t' -v n="$1" '$2 == n { f = 1 } END { exit !f }' "$R/sums.txt"; } +sum_of() { awk -F'\t' -v n="$1" '$2 == n { print $1; exit }' "$R/sums.txt"; } +mirror_digest() { # NAME → sha256 hex; "" when absent; "?" when present without a digest + [ "$MREL_PRESENT" -eq 1 ] || return 0 + awk -F'\t' -v n="$1" '$1 == n { print ($2 == "" ? "?" : $2); exit }' "$R/mirror-assets.tsv" +} +# decide LIST NAME EXPECTED_SHA — EXPECTED_SHA may be "" (unknown). +decide() { + local list="$1" name="$2" expected="$3" have + have="$(mirror_digest "$name")" + if [ "$have" = "?" ]; then die2 "$TAG: mirror asset '$name' has no digest — cannot tell whether it matches"; fi + if [ -z "$have" ]; then printf '%s\tupload\t%s\n' "$name" "$expected" >>"$list"; return 0; fi + if [ -z "$expected" ]; then printf '%s\tcompare\t%s\n' "$name" "$have" >>"$list"; return 0; fi + if [ "$have" = "$expected" ]; then printf '%s\tskip\t%s\n' "$name" "$expected" >>"$list"; return 0; fi # mutation-anchor: idempotent-skip + [ -n "$REFUSAL" ] || REFUSAL="asset '$name' is on the mirror with SHA256 $have but the source release says $expected — a published asset is never replaced" + printf '%s\trefuse\t%s\n' "$name" "$expected" >>"$list" +} +# resolve_compares LIST DIR — hash each `compare` download; equal → skip, +# different → REFUSAL (a published asset is never replaced). +resolve_compares() { + local list="$1" dir="$2" aname action have got + while IFS=$'\t' read -r aname action have; do + [ "$action" = compare ] || continue + [ -f "$dir/$aname" ] || die2 "$TAG: '$aname' did not download from '$SRC'" + got="$(sha256_of "$dir/$aname")" + if [ "$got" = "$have" ]; then + awk -F'\t' -v OFS='\t' -v n="$aname" '$1 == n && $2 == "compare" { $2 = "skip" } { print }' "$list" >"$list.new" && mv "$list.new" "$list" + else + [ -n "$REFUSAL" ] || REFUSAL="asset '$aname' is on the mirror with SHA256 $have but the source's is $got — a published asset is never replaced" + fi + done <"$list" +} +# download_listed LIST DIR — `gh release download` of every upload/compare +# entry in LIST into DIR (one call; a pattern that matches nothing is an error +# gh reports, which is could-not-tell here). +download_listed() { + local list="$1" dir="$2" f + local -a pats=() + while IFS= read -r f; do pats+=(--pattern "$f"); done < <(awk -F'\t' '$2 == "upload" || $2 == "compare" { print $1 }' "$list") + [ "${#pats[@]}" -gt 0 ] || return 0 + gh_read "$dir.log" release download "$TAG" --repo "$SRC" --dir "$dir" "${pats[@]}" +} +cols() { # → TEXT_COL / BIN_COL from the decision files + local tu ts bu bs cu cs + tu="$(count_action "$R/upload-text.txt" upload)"; ts="$(count_action "$R/upload-text.txt" skip)" + TEXT_COL="$tu up/$ts skip" + BIN_COL="-" + [ "$WANT_BIN" -eq 1 ] || return 0 + bu="$(count_action "$R/upload-bin.txt" upload)"; bs="$(count_action "$R/upload-bin.txt" skip)" + cu="$(count_action "$R/upload-companion.txt" upload)"; cs="$(count_action "$R/upload-companion.txt" skip)" + BIN_COL="$bu up/$bs skip (+$cu/$cs sig+cert)" +} + +# Table rows: tag | kind | tag-action | release-action | text | binaries | verdict +: >"$TMP/table.txt" +N_REFUSED=0; N_CREATED=0; N_SKIPPED=0 +# For the `latest` fallback after the loop: was the newest stable release +# refused before its own POST (the one carrying make_latest=true), and which +# stable release did this run create last (= newest, the run is oldest-first). +NEWEST_STABLE_REFUSED=0; LAST_STABLE_CREATED=""; LAST_STABLE_CREATED_ID="" +row() { printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$@" >>"$TMP/table.txt"; } +refuse() { # TAG REASON — the release is refused, the run goes on + echo "::error::backfill-releases: REFUSED $1 — $2" + # Refusing a release the mirror already has leaves `latest` where it is; + # refusing the newest stable BEFORE it is created leaves nothing marked. + if [ "$1" = "$NEWEST_STABLE" ] && [ "$REL_ACTION" = create ]; then NEWEST_STABLE_REFUSED=1; fi + N_REFUSED=$((N_REFUSED + 1)); cols; row "$1" "$KIND" "$TAG_ACTION" "$REL_ACTION" "$TEXT_COL" "$BIN_COL" refused +} + +# ---- per-release work ------------------------------------------------------------------- +while IFS= read -r TAG; do + R="$TMP/r-$TAG"; mkdir -p "$R/text" "$R/bin" "$R/sums" "$R/guard-assets" + jq --arg t "$TAG" '.[] | select(.tag_name == $t)' "$TMP/releases.json" >"$R/release.json" + # Own assignment, not `[ "$(jq_of …)" = true ]`: inside a test the + # substitution's exit 2 is swallowed and a malformed release.json would + # silently read as "stable". + PRERELEASE="$(jq_of "$R/release.json" '.prerelease')" + KIND=stable; [ "$PRERELEASE" = true ] && KIND=prerelease + NAME="$(jq_of "$R/release.json" '.name // .tag_name')" + CREATED="$(jq_of "$R/release.json" '.created_at')" + PUBLISHED="$(jq_of "$R/release.json" '.published_at // .created_at')" + jq -r '.body // ""' "$R/release.json" >"$R/body.md" + WANT_BIN=0; carries_binaries "$TAG" && WANT_BIN=1 + TAG_ACTION=create; REL_ACTION=create; REFUSAL="" + : >"$R/upload-text.txt"; : >"$R/upload-bin.txt"; : >"$R/upload-companion.txt" + + # Assets, classified. SHA256SUMS is the authority on what a binary is: the + # names it lists are binaries, .sig / .cert their companions, + # everything else a text asset. A release without SHA256SUMS has no binaries + # this tool can vouch for, so every asset is text and none is a binary. + jq -r '.assets[] | [.name, (.digest // "")] | @tsv' "$R/release.json" >"$R/assets.tsv" + : >"$R/sums.txt" + if awk -F'\t' '$1 == "SHA256SUMS" { f = 1 } END { exit !f }' "$R/assets.tsv"; then + gh_read "$R/sums.log" release download "$TAG" --repo "$SRC" --dir "$R/sums" --pattern SHA256SUMS + [ -s "$R/sums/SHA256SUMS" ] || die2 "$TAG: SHA256SUMS did not download from '$SRC'" + awk 'NF >= 2 { print $1 "\t" $NF }' "$R/sums/SHA256SUMS" >"$R/sums.txt" + fi + : >"$R/text.txt"; : >"$R/bin.txt"; : >"$R/companion.txt" + while IFS=$'\t' read -r aname adigest; do + if in_sums "$aname"; then printf '%s\t%s\n' "$aname" "$adigest" >>"$R/bin.txt" + elif [[ "$aname" =~ $COMPANION_RE ]] && in_sums "${aname%.*}"; then printf '%s\t%s\n' "$aname" "$adigest" >>"$R/companion.txt" + else printf '%s\t%s\n' "$aname" "$adigest" >>"$R/text.txt"; fi + done <"$R/assets.tsv" + + # -- mirror state for this tag ---------------------------------------------------------- + jq --arg t "$TAG" '[ .[] | select(.tag_name == $t) ] | .[0] // empty' "$TMP/mirror-releases.json" >"$R/mirror-release.json" + MREL_PRESENT=0; [ -s "$R/mirror-release.json" ] && MREL_PRESENT=1 + : >"$R/mirror-assets.tsv" + [ "$MREL_PRESENT" -eq 0 ] || jq -r '.assets[] | [.name, ((.digest // "") | ltrimstr("sha256:"))] | @tsv' "$R/mirror-release.json" >"$R/mirror-assets.tsv" + jq --arg r "refs/tags/$TAG" '[ .[] | select(.ref == $r) ] | .[0] // empty' "$TMP/mirror-tags.json" >"$R/mirror-tag.json" + MTAG_PRESENT=0; [ -s "$R/mirror-tag.json" ] && MTAG_PRESENT=1 + if [ "$MTAG_PRESENT" -eq 1 ]; then + # The tag exists: it must point at a commit the mirror has. Dereference an + # annotated tag first; a dangling tag is refused, never repointed. + OBJ_SHA="$(jq_of "$R/mirror-tag.json" '.object.sha')"; OBJ_TYPE="$(jq_of "$R/mirror-tag.json" '.object.type')" + if [ "$OBJ_TYPE" = tag ]; then + gh_read "$R/mirror-tagobj.json" api "repos/$MIRROR/git/tags/$OBJ_SHA" + OBJ_SHA="$(jq_of "$R/mirror-tagobj.json" '.object.sha')"; OBJ_TYPE="$(jq_of "$R/mirror-tagobj.json" '.object.type')" + fi + if [ "$OBJ_TYPE" != commit ] || ! gh_read_maybe "$R/mirror-tagcommit.json" api "repos/$MIRROR/git/commits/$OBJ_SHA"; then + REFUSAL="tag '$TAG' exists on the mirror but points at $OBJ_TYPE $OBJ_SHA, which the mirror does not have — a dangling tag is not repointed" + fi + TAG_ACTION=present + fi + [ "$MREL_PRESENT" -eq 0 ] || REL_ACTION=present + + # -- assets: skip / upload / compare / refuse, per asset -------------------------------------- + # An asset already on the mirror is skipped when its SHA256 equals the + # source's, refused when it differs (a published asset is never replaced), + # uploaded when absent. The mirror's digest comes from the API; a mirror + # asset without one cannot be compared, and "cannot compare" is not "equal". + while IFS=$'\t' read -r aname adigest; do decide "$R/upload-text.txt" "$aname" "${adigest#sha256:}"; done <"$R/text.txt" + if [ "$WANT_BIN" -eq 1 ]; then + while IFS=$'\t' read -r aname _; do decide "$R/upload-bin.txt" "$aname" "$(sum_of "$aname")"; done <"$R/bin.txt" + while IFS=$'\t' read -r aname adigest; do decide "$R/upload-companion.txt" "$aname" "${adigest#sha256:}"; done <"$R/companion.txt" + fi + if [ -n "$REFUSAL" ]; then refuse "$TAG" "$REFUSAL"; continue; fi + N_TODO="$(cat "$R"/upload-*.txt | awk -F'\t' '$2 == "upload" || $2 == "compare" { n++ } END { print n + 0 }')" + if [ "$TAG_ACTION" = present ] && [ "$REL_ACTION" = present ] && [ "$N_TODO" -eq 0 ]; then + note "$TAG: already on the mirror, every asset matches — nothing to do" + N_SKIPPED=$((N_SKIPPED + 1)); cols; row "$TAG" "$KIND" present present "$TEXT_COL" "$BIN_COL" skipped; continue + fi + + # -- downloads ------------------------------------------------------------------------------ + # Text assets to upload or compare come down in both modes (the guard reads + # them). Binaries and companions come down under --apply only: they are large + # and opaque to the scan; each binary is checked against SHA256SUMS at once. + download_listed "$R/upload-text.txt" "$R/text" + resolve_compares "$R/upload-text.txt" "$R/text" + if [ -n "$REFUSAL" ]; then refuse "$TAG" "$REFUSAL"; continue; fi + if [ "$APPLY" -eq 1 ] && [ "$WANT_BIN" -eq 1 ]; then + cat "$R/upload-bin.txt" "$R/upload-companion.txt" >"$R/upload-binlike.txt" + download_listed "$R/upload-binlike.txt" "$R/bin" + while IFS=$'\t' read -r aname action expected; do + [ "$action" = upload ] || continue + [ -f "$R/bin/$aname" ] || die2 "$TAG: binary '$aname' did not download from '$SRC'" + got="$(sha256_of "$R/bin/$aname")" + [ "$got" = "$expected" ] || { REFUSAL="binary '$aname' hashes to $got but the source release's SHA256SUMS says $expected — not uploaded"; break; } # mutation-anchor: sha-check + done <"$R/upload-bin.txt" + if [ -n "$REFUSAL" ]; then refuse "$TAG" "$REFUSAL"; continue; fi + resolve_compares "$R/upload-companion.txt" "$R/bin" + if [ -n "$REFUSAL" ]; then refuse "$TAG" "$REFUSAL"; continue; fi + fi + + # -- notes -------------------------------------------------------------------------------- + if [ "$REL_ACTION" = create ]; then + if [ "$NOTES_MODE" = fixed ]; then + { + echo "tracebloc CLI $TAG." + echo + echo "Install with the one-liner in the README, or download a binary below and" + echo "verify it against SHA256SUMS and its cosign .sig/.cert (recipe in the README)." + } >"$R/notes.md" + else + cp "$R/body.md" "$R/notes.md" + fi + { + echo + echo "---" + echo "Backfilled release marker: originally published $PUBLISHED. The tag \`$TAG\` on this repository points at the default branch, not at the sources this release was built from." + if [ "$WANT_BIN" -eq 0 ]; then + echo "Binaries are carried only for the newest $BINARY_KEEP releases; re-run the installer to get a current build." + fi + } >>"$R/notes.md" + fi + + # -- the guard: text assets, companions and notes, before any write -------------------------- + while IFS= read -r f; do cp "$R/text/$f" "$R/guard-assets/$f"; done < <(awk -F'\t' '$2 == "upload" { print $1 }' "$R/upload-text.txt") + if [ "$APPLY" -eq 1 ] && [ "$WANT_BIN" -eq 1 ]; then + while IFS= read -r f; do cp "$R/bin/$f" "$R/guard-assets/$f"; done < <(awk -F'\t' '$2 == "upload" { print $1 }' "$R/upload-companion.txt") + fi + [ "$REL_ACTION" != create ] || cp "$R/notes.md" "$R/guard-assets/RELEASE_NOTES.md" + if [ -n "$(ls -A "$R/guard-assets")" ]; then + rc=0; run_guard "$R/guard-assets" "$R/guard-out" || rc=$? + case "$rc" in + 0) ;; + 1) REFUSAL="the guard refused the notes or a text asset: $(grep -E 'REFUSED' "$R/guard-out.log" | sed 's/^::error::publish-guard: //' | paste -sd';' -)" ;; # mutation-anchor: guard-refusal + *) cat "$R/guard-out.log"; die2 "$TAG: the guard could not tell (exit $rc)" ;; + esac + if [ -n "$REFUSAL" ]; then grep -E 'REFUSED|^ ' "$R/guard-out.log" | sed 's/^/ /'; refuse "$TAG" "$REFUSAL"; continue; fi + fi + + cols + if [ "$WANT_BIN" -eq 1 ]; then BIN_PLAN="$BIN_COL"; else BIN_PLAN="none (older than the newest $BINARY_KEEP)"; fi + PLAN="tag: $TAG_ACTION | release: $REL_ACTION | text: $TEXT_COL | binaries: $BIN_PLAN" + if [ "$APPLY" -eq 0 ]; then + note "$TAG [$KIND] would: $PLAN" + N_CREATED=$((N_CREATED + 1)); row "$TAG" "$KIND" "$TAG_ACTION" "$REL_ACTION" "$TEXT_COL" "$BIN_COL" planned; continue + fi + + # -- writes ----------------------------------------------------------------------------------- + note "$TAG [$KIND]: $PLAN" + if [ "$TAG_ACTION" = create ]; then + # The original tag's date and message, when it is annotated; the release's + # created_at otherwise. Read from the source, never invented. + gh_read "$R/src-ref.json" api "repos/$SRC/git/ref/tags/$TAG" + SRC_OBJ_TYPE="$(jq_of "$R/src-ref.json" '.object.type')"; SRC_OBJ_SHA="$(jq_of "$R/src-ref.json" '.object.sha')" + TAG_DATE="$CREATED"; ORIG_MSG="" + if [ "$SRC_OBJ_TYPE" = tag ]; then + gh_read "$R/src-tagobj.json" api "repos/$SRC/git/tags/$SRC_OBJ_SHA" + TAG_DATE="$(jq_of "$R/src-tagobj.json" '.tagger.date // empty')"; [ -n "$TAG_DATE" ] || TAG_DATE="$CREATED" + ORIG_MSG="$(jq_of "$R/src-tagobj.json" '.message // ""')" + fi + { + echo "Release $TAG" + echo + echo "Mirror release marker for $TAG: this tag points at the mirror's default-branch head, not at the sources the release was built from. Original tag date: $TAG_DATE." + if [ -n "$ORIG_MSG" ]; then echo; echo "--- original tag message ---"; printf '%s\n' "$ORIG_MSG"; fi + } >"$R/tag-message.txt" + gh_write "$R/tagobj.json" api -X POST "repos/$MIRROR/git/tags" \ + -f "tag=$TAG" -F "message=@$R/tag-message.txt" -f "object=$MIRROR_HEAD" -f type=commit \ + -f "tagger[name]=${PUBLISH_MIRROR_GIT_NAME:-github-actions[bot]}" \ + -f "tagger[email]=${PUBLISH_MIRROR_GIT_EMAIL:-github-actions[bot]@users.noreply.github.com}" \ + -f "tagger[date]=$TAG_DATE" + TAGOBJ_SHA="$(jq_of "$R/tagobj.json" '.sha')" + [[ "$TAGOBJ_SHA" =~ ^[0-9a-f]{40}$ ]] || die2 "$TAG: the created tag object has no sha" + gh_write "$R/ref.json" api -X POST "repos/$MIRROR/git/refs" -f "ref=refs/tags/$TAG" -f "sha=$TAGOBJ_SHA" + fi + if [ "$REL_ACTION" = create ]; then + LATEST=false; [ "$TAG" = "$NEWEST_STABLE" ] && LATEST=true + PRE=false; [ "$KIND" = prerelease ] && PRE=true + gh_write "$R/created.json" api -X POST "repos/$MIRROR/releases" \ + -f "tag_name=$TAG" -f "name=$NAME" -F "body=@$R/notes.md" -F "prerelease=$PRE" -F draft=false -f "make_latest=$LATEST" + fi + UPLOADS=() + while IFS= read -r f; do UPLOADS+=("$f"); done < <( + awk -F'\t' -v d="$R/text" '$2 == "upload" { print d "/" $1 }' "$R/upload-text.txt" + awk -F'\t' -v d="$R/bin" '$2 == "upload" { print d "/" $1 }' "$R/upload-bin.txt" "$R/upload-companion.txt" + ) + if [ "${#UPLOADS[@]}" -gt 0 ]; then + gh_write "$R/upload.log" release upload "$TAG" "${UPLOADS[@]}" --repo "$MIRROR" + fi + if [ "$KIND" = stable ] && [ "$REL_ACTION" = create ]; then + LAST_STABLE_CREATED="$TAG"; LAST_STABLE_CREATED_ID="$(jq_of "$R/created.json" '.id')" + [[ "$LAST_STABLE_CREATED_ID" =~ ^[0-9]+$ ]] || die2 "$TAG: the created release has no numeric id" + fi + N_CREATED=$((N_CREATED + 1)) + row "$TAG" "$KIND" "$TAG_ACTION" "$REL_ACTION" "$TEXT_COL" "$BIN_COL" "done" +done <"$TMP/tags-run.txt" + +# ---- latest, when the newest stable release was refused ----------------------------- +# make_latest=true travels on the newest stable release's own POST; every older +# release is created with make_latest=false. Refused before that POST, the newest +# stable leaves the mirror's releases/latest answering 404 until a human re-runs +# --only-tag for it. Until then the newest stable release this run DID write is +# marked latest — that re-run's POST moves `latest` forward again. +if [ "$APPLY" -eq 1 ] && [ "$NEWEST_STABLE_REFUSED" -eq 1 ]; then # mutation-anchor: latest-fallback + if [ -n "$LAST_STABLE_CREATED" ]; then + note "latest: $NEWEST_STABLE was refused — marking $LAST_STABLE_CREATED, the newest stable release written in this run, as latest until $NEWEST_STABLE is re-run" + # -f, not -F: make_latest is a STRING enum ("true"/"false"/"legacy") in the + # releases API; a typed boolean is a 422, which here would be a die2 in the + # very case this fallback exists for. Same reason the create path uses -f. + gh_write "$TMP/latest.json" api -X PATCH "repos/$MIRROR/releases/$LAST_STABLE_CREATED_ID" -f make_latest=true # mutation-anchor: latest-string-typed + else + echo "::warning::backfill-releases: $NEWEST_STABLE was refused and this run wrote no stable release — nothing is newly marked latest; re-run --only-tag $NEWEST_STABLE once the refusal is fixed" + fi +fi + +# ---- report ------------------------------------------------------------------------- +echo +echo "backfill-releases: $MODE report — $SRC → $MIRROR" +{ + printf 'TAG\tKIND\tTAG-ON-MIRROR\tRELEASE\tTEXT ASSETS\tBINARIES\tVERDICT\n' + cat "$TMP/table.txt" +} | column -t -s "$(printf '\t')" 2>/dev/null || cat "$TMP/table.txt" +echo +VERB=written; [ "$APPLY" -eq 1 ] || VERB=planned +echo "backfill-releases: $N_RUN release(s) in this run — $N_CREATED $VERB, $N_SKIPPED already complete, $N_REFUSED refused" +if [ "$N_REFUSED" -gt 0 ]; then + echo "::error::backfill-releases: REFUSED — $N_REFUSED release(s) were refused (see the table); the rest went ahead" + exit 1 +fi +exit 0 diff --git a/scripts/tests/backfill-releases-verify.sh b/scripts/tests/backfill-releases-verify.sh new file mode 100644 index 0000000..2b573e4 --- /dev/null +++ b/scripts/tests/backfill-releases-verify.sh @@ -0,0 +1,570 @@ +#!/usr/bin/env bash +# ============================================================================= +# backfill-releases-verify.sh — pin the properties of +# scripts/backfill-releases.sh, the one-shot historical backfill of releases +# to the public mirror. +# +# Offline. `gh` is a recording FAKE on PATH that serves a source repository's +# releases, tags and assets from a fixture directory and keeps the mirror's +# state (tags, releases, uploaded assets with their digests) in JSON files it +# mutates on every write, so a second run sees what the first one created. +# Every call is logged; a WRITE is any `api -X POST` or `release upload` line. +# gitleaks is a stub (the guard needs one on PATH; the real scanner is the +# workflow's business). +# +# Pinned: dry-run writes nothing; --apply makes exactly the expected writes, +# oldest release first, newest stable release last and marked latest; a second +# --apply writes nothing; the BINARY_KEEP boundary (the 10th newest carries +# binaries, the 11th does not — and the count is over the FILTERED list, so +# --include-prerelease shifts it); a binary whose SHA256 disagrees with the +# source's SHA256SUMS is refused by name and nothing of that release is +# written; an unset mirror and a mirror equal to the source are refused by +# publish-mirror's rule; prereleases are excluded by default; a read that +# fails is could-not-tell (exit 2) with the failing call named, never an empty +# list, and a read that returns non-JSON is could-not-tell naming the file and +# filter even though it is parsed inside a "$(...)" substitution (the reason +# travels on stderr, so the variable never swallows it); the DEFAULT notes are the workflow's fixed text (the source body is +# never written or scanned unless --notes source asks for it); under --notes +# source a refuse-tier needle in a release body refuses that release naming +# the tier; a mirror asset already present with a different digest is refused, +# never replaced; a mirror tag that dangles is refused, never repointed; +# --only-tag / --from-tag narrow the run without changing the binary decision; +# the mirror tag carries the original date and, for an annotated source tag, +# the original message; when the newest stable release is refused, the newest +# stable release the run did write is marked latest (one PATCH) so +# releases/latest never 404s behind a refusal; the guard's scratch commit is +# made unsigned, so a global commit.gpgsign cannot end the run. +# +# --mutations: copies the script, breaks ONE rule per copy at its +# `# mutation-anchor: NAME` line, proves the mutation landed (the anchor was +# found exactly once and the copy differs), runs THIS suite against the copy +# and demands red. A mutation the suite survives is a vacuous test, reported +# as a failure of this harness. +# +# Environment (mutation runs only): BACKFILL_UNDER_TEST names the script copy +# to test instead of ../backfill-releases.sh. +# ============================================================================= +set -uo pipefail + +SELF="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")" +SELF_DIR="$(cd "$(dirname "$0")" && pwd)" +SCRIPTS_DIR="$(cd "$SELF_DIR/.." && pwd)" +REAL="$SCRIPTS_DIR/backfill-releases.sh" +BACKFILL="${BACKFILL_UNDER_TEST:-$REAL}" +[ -f "$REAL" ] || { printf 'backfill-releases-verify: %s missing — refusing to report clean\n' "$REAL" >&2; exit 2; } +[ -f "$BACKFILL" ] || { printf 'backfill-releases-verify: %s missing — refusing to report clean\n' "$BACKFILL" >&2; exit 2; } +for t in jq git awk; do command -v "$t" >/dev/null 2>&1 || { printf 'backfill-releases-verify: %s is not on PATH — refusing to report clean\n' "$t" >&2; exit 2; }; done + +PASS=0 +FAIL=0 +ok() { printf ' ok %s\n' "$1"; PASS=$((PASS+1)); } +bad() { printf ' FAIL %s\n' "$1"; FAIL=$((FAIL+1)); } + +ROOT="$(mktemp -d "${TMPDIR:-/tmp}/backfill-releases-verify.XXXXXX")" +trap 'rm -rf "$ROOT"' EXIT +if command -v sha256sum >/dev/null 2>&1; then sha256_of() { sha256sum "$1" | cut -d' ' -f1; }; else sha256_of() { shasum -a 256 "$1" | cut -d' ' -f1; }; fi + +# ============================================================================= +# --mutations mode: break one rule per copy, demand red. +# ============================================================================= +if [ "${1:-}" = "--mutations" ]; then + echo "== backfill-releases.sh mutations ==" + # NAME|REPLACEMENT — the line carrying `# mutation-anchor: NAME` becomes REPLACEMENT. + # Each replacement is a valid program that drops exactly the rule the anchor names. + MUTATIONS=( + 'releases-read-fail-closed|gh api --paginate "repos/$SRC/releases" >"$TMP/src-pages.json" 2>/dev/null || printf "[]" >"$TMP/src-pages.json"' + 'prerelease-filter|FILTER='"'"'[ .[] | select(.draft == false) ] | sort_by(.created_at) | reverse'"'" + 'binary-keep|cp "$TMP/tags-newest-first.txt" "$TMP/tags-with-binaries.txt"' + 'idempotent-skip|:' + 'sha-check|:' + 'guard-refusal| 1) ;;' + 'notes-default-fixed|APPLY=0; FROM_TAG=""; ONLY_TAG=""; INCLUDE_PRE=0; NOTES_MODE=source; STRICT=0' + 'die2-stderr|die2() { echo "::error::backfill-releases: COULD NOT TELL — $1 (nothing more is written)"; exit 2; }' + 'latest-fallback|if false; then' + 'latest-string-typed| gh_write "$TMP/latest.json" api -X PATCH "repos/$MIRROR/releases/$LAST_STABLE_CREATED_ID" -F make_latest=true' + 'scratch-commit-unsigned|git -C "$SCRATCH" add README.md && git -C "$SCRATCH" -c user.name=backfill -c user.email=backfill@localhost commit -q -m scratch || die2 "could not commit the guard'"'"'s scratch checkout"' + ) + MUT_PASS=0; MUT_FAIL=0 + # Every mutant is prepared and PROVEN to have landed first; then the suite + # runs against all of them in parallel (each run is ~a minute of forks). + NAMES=() + for entry in "${MUTATIONS[@]}"; do + name="${entry%%|*}"; repl="${entry#*|}" + copy="$ROOT/mutant-$name.sh" + n="$(grep -c -- "# mutation-anchor: $name\$" "$REAL")" + if [ "$n" -ne 1 ]; then bad "mutation $name: anchor found $n time(s) in $REAL, need exactly 1"; MUT_FAIL=$((MUT_FAIL+1)); continue; fi + awk -v a="# mutation-anchor: $name" -v r="$repl" 'index($0, a) && substr($0, length($0) - length(a) + 1) == a { print r; next } { print }' "$REAL" >"$copy" + if cmp -s "$REAL" "$copy"; then bad "mutation $name: the copy is identical to the original — the mutation did not apply"; MUT_FAIL=$((MUT_FAIL+1)); continue; fi + if ! bash -n "$copy" 2>"$ROOT/mutant.err"; then bad "mutation $name: the mutant does not parse: $(cat "$ROOT/mutant.err")"; MUT_FAIL=$((MUT_FAIL+1)); continue; fi + NAMES+=("$name") + done + for name in "${NAMES[@]+"${NAMES[@]}"}"; do + ( if BACKFILL_UNDER_TEST="$ROOT/mutant-$name.sh" bash "$SELF" >"$ROOT/mutant-$name.log" 2>&1; then echo green; else echo red; fi >"$ROOT/mutant-$name.verdict" ) & + done + wait + for name in "${NAMES[@]+"${NAMES[@]}"}"; do + out="$ROOT/mutant-$name.log" + if [ "$(cat "$ROOT/mutant-$name.verdict" 2>/dev/null)" = red ]; then + ok "mutation $name: caught — $(grep -c '^ FAIL' "$out") test(s) reddened: $(grep '^ FAIL' "$out" | head -2 | sed -E 's/^ FAIL ([^(:]*).*/\1/' | paste -sd'|' -)"; MUT_PASS=$((MUT_PASS+1)) + else + bad "mutation $name: the suite stayed GREEN against the mutant — a test is vacuous"; MUT_FAIL=$((MUT_FAIL+1)) + sed 's/^/ /' "$out" | tail -5 + fi + done + echo + printf 'backfill-releases-verify --mutations: %d caught, %d survived\n' "$MUT_PASS" "$MUT_FAIL" + [ "$MUT_FAIL" -eq 0 ] && [ "$MUT_PASS" -ge 4 ] + exit $? +fi + +# ============================================================================= +# The fake gh and the gitleaks stub +# ============================================================================= +SHIM="$ROOT/shim"; mkdir -p "$SHIM" +cat >"$SHIM/gitleaks" <<'EOF' +#!/usr/bin/env bash +[ "${1:-}" = version ] && { echo "gitleaks-stub"; exit 0; } +exit 0 +EOF +cat >"$SHIM/gh" <<'EOF' +#!/usr/bin/env bash +# Recording fake gh. FIX: fixtures (read-only). STATE: the mirror, mutated by writes. +set -uo pipefail +FIX="${FAKE_GH_FIX:?}"; STATE="${FAKE_GH_STATE:?}" +printf '%s\n' "$*" >>"${GH_LOG:?}" +if [ -n "${FAKE_GH_FAIL_RE:-}" ] && [[ "$*" =~ $FAKE_GH_FAIL_RE ]]; then echo "gh: Internal Server Error (HTTP 500)" >&2; exit 1; fi +# A call that "succeeds" with a body that is not JSON — the answer jq must refuse. +if [ -n "${FAKE_GH_GARBLE_RE:-}" ] && [[ "$*" =~ $FAKE_GH_GARBLE_RE ]]; then echo 'not json'; exit 0; fi +SRC="$(cat "$FIX/src-repo")"; MIRROR="$(cat "$FIX/mirror-repo")"; HEAD_SHA="$(cat "$FIX/mirror-head")" +if command -v sha256sum >/dev/null 2>&1; then sha() { sha256sum "$1" | cut -d' ' -f1; }; else sha() { shasum -a 256 "$1" | cut -d' ' -f1; }; fi +fake_sha() { printf '%s' "$1" | { command -v sha256sum >/dev/null 2>&1 && sha256sum || shasum -a 256; } | cut -c1-40; } # a 40-hex git object id +notfound() { echo "gh: Not Found (HTTP 404)" >&2; exit 1; } +fieldval() { # KEY from -f/-F pairs; @file is read + local k="$1" f v + for f in "${FIELDS[@]+"${FIELDS[@]}"}"; do + case "$f" in "$k="*) v="${f#*=}"; case "$v" in @*) cat "${v#@}" ;; *) printf '%s' "$v" ;; esac; return 0 ;; esac + done + return 1 +} +# make_latest is a STRING enum ("true"/"false"/"legacy") in the releases API. A +# `-F make_latest=true` is a JSON boolean, which GitHub answers with 422 — so +# does this fake, on both release calls, instead of quietly accepting it. +make_latest_typed() { local t; for t in "${TYPED[@]+"${TYPED[@]}"}"; do [ "$t" = make_latest ] && return 0; done; return 1; } +reject_typed_make_latest() { ! make_latest_typed || { echo "gh: HTTP 422: Invalid request. For 'properties/make_latest', true is not a string. (https://docs.github.com/rest/releases/releases)" >&2; exit 1; }; } +cmd="${1:-}"; shift || true +case "$cmd" in + repo) + printf '{"nameWithOwner":"%s"}\n' "$SRC" ;; + api) + METHOD=GET; PATHP=""; FIELDS=(); TYPED=() + while [ "$#" -gt 0 ]; do + case "$1" in + -X) METHOD="$2"; shift 2 ;; + --paginate) shift ;; + -f) FIELDS+=("$2"); shift 2 ;; + -F) FIELDS+=("$2"); TYPED+=("${2%%=*}"); shift 2 ;; # -F types true/false/numbers as JSON, like real gh + *) PATHP="$1"; shift ;; + esac + done + case "$METHOD $PATHP" in + "GET repos/$SRC/releases") + # Two "pages": the fixture is split so --paginate's concatenated-arrays shape is exercised. + jq -c '.[0:7]' "$FIX/src-releases.json"; jq -c '.[7:]' "$FIX/src-releases.json" ;; + "GET repos/$SRC/git/ref/tags/"*) + t="${PATHP##*/}"; jq -e --arg r "refs/tags/$t" '.[] | select(.ref == $r)' "$FIX/src-tags.json" >/dev/null || notfound + jq --arg r "refs/tags/$t" '.[] | select(.ref == $r)' "$FIX/src-tags.json" ;; + "GET repos/$SRC/git/tags/"*) + s="${PATHP##*/}"; jq -e --arg s "$s" '.[] | select(.sha == $s)' "$FIX/src-tagobjs.json" >/dev/null || notfound + jq --arg s "$s" '.[] | select(.sha == $s)' "$FIX/src-tagobjs.json" ;; + "GET repos/$MIRROR") + printf '{"full_name":"%s","default_branch":"main","visibility":"public"}\n' "$MIRROR" ;; + "GET repos/$MIRROR/commits/main") + [ -z "${FAKE_GH_EMPTY_MIRROR:-}" ] || { echo "gh: Git Repository is empty. (HTTP 409)" >&2; exit 1; } + printf '{"sha":"%s"}\n' "$HEAD_SHA" ;; + "GET repos/$MIRROR/releases") + cat "$STATE/mirror-releases.json" ;; + "GET repos/$MIRROR/git/matching-refs/tags/") + cat "$STATE/mirror-tags.json" ;; + "GET repos/$MIRROR/git/commits/"*) + s="${PATHP##*/}"; [ "$s" = "$HEAD_SHA" ] || notfound; printf '{"sha":"%s"}\n' "$s" ;; + "GET repos/$MIRROR/git/tags/"*) + s="${PATHP##*/}"; jq -e --arg s "$s" '.[] | select(.sha == $s)' "$STATE/mirror-tagobjs.json" >/dev/null || notfound + jq --arg s "$s" '.[] | select(.sha == $s)' "$STATE/mirror-tagobjs.json" ;; + "POST repos/$MIRROR/git/tags") + tag="$(fieldval tag)"; msg="$(fieldval message)"; obj="$(fieldval object)"; date="$(fieldval 'tagger[date]')" + s="$(fake_sha "tagobj:$tag")" + jq --arg s "$s" --arg tag "$tag" --arg msg "$msg" --arg obj "$obj" --arg date "$date" \ + '. + [{sha: $s, tag: $tag, message: $msg, tagger: {date: $date}, object: {sha: $obj, type: "commit"}}]' "$STATE/mirror-tagobjs.json" >"$STATE/t.json" && mv "$STATE/t.json" "$STATE/mirror-tagobjs.json" + printf '{"sha":"%s"}\n' "$s" ;; + "POST repos/$MIRROR/git/refs") + ref="$(fieldval ref)"; s="$(fieldval sha)" + jq -e --arg r "$ref" '.[] | select(.ref == $r)' "$STATE/mirror-tags.json" >/dev/null && { echo "gh: Reference already exists (HTTP 422)" >&2; exit 1; } + jq --arg r "$ref" --arg s "$s" '. + [{ref: $r, object: {sha: $s, type: "tag"}}]' "$STATE/mirror-tags.json" >"$STATE/t.json" && mv "$STATE/t.json" "$STATE/mirror-tags.json" + printf '{"ref":"%s"}\n' "$ref" ;; + "POST repos/$MIRROR/releases") + reject_typed_make_latest + tag="$(fieldval tag_name)"; name="$(fieldval name)"; body="$(fieldval body)"; pre="$(fieldval prerelease)"; latest="$(fieldval make_latest)" + jq -e --arg r "refs/tags/$tag" '.[] | select(.ref == $r)' "$STATE/mirror-tags.json" >/dev/null || { echo "gh: fake: release for '$tag' before its tag (HTTP 422)" >&2; exit 1; } + id="$(jq 'length + 1' "$STATE/mirror-releases.json")" + jq --argjson id "$id" --arg tag "$tag" --arg name "$name" --arg body "$body" --arg pre "$pre" --arg latest "$latest" \ + '. + [{id: $id, tag_name: $tag, name: $name, body: $body, prerelease: ($pre == "true"), make_latest: $latest, draft: false, assets: []}]' "$STATE/mirror-releases.json" >"$STATE/t.json" && mv "$STATE/t.json" "$STATE/mirror-releases.json" + printf '{"id":%s,"tag_name":"%s"}\n' "$id" "$tag" ;; + "PATCH repos/$MIRROR/releases/"*) + reject_typed_make_latest + id="${PATHP##*/}"; latest="$(fieldval make_latest)" + [[ "$id" =~ ^[0-9]+$ ]] || notfound + jq -e --argjson id "$id" '.[] | select(.id == $id)' "$STATE/mirror-releases.json" >/dev/null || notfound + jq --argjson id "$id" --arg latest "$latest" 'map(if .id == $id then .make_latest = $latest else . end)' "$STATE/mirror-releases.json" >"$STATE/t.json" && mv "$STATE/t.json" "$STATE/mirror-releases.json" + printf '{"id":%s}\n' "$id" ;; + *) echo "gh: fake: unhandled $METHOD $PATHP" >&2; exit 1 ;; + esac ;; + release) + sub="${1:-}"; shift || true + case "$sub" in + download) + tag="$1"; shift; repo=""; dir=""; pats=() + while [ "$#" -gt 0 ]; do case "$1" in --repo) repo="$2"; shift 2 ;; --dir) dir="$2"; shift 2 ;; --pattern) pats+=("$2"); shift 2 ;; *) echo "gh: fake: unknown download arg $1" >&2; exit 1 ;; esac; done + [ "$repo" = "$SRC" ] || { echo "gh: fake: download from '$repo' is not the source" >&2; exit 1; } + mkdir -p "$dir" + for p in "${pats[@]}"; do + [ -f "$FIX/assets/$tag/$p" ] || { echo "gh: no assets match the file pattern ($p)" >&2; exit 1; } + [ ! -e "$dir/$p" ] || { echo "gh: fake: $dir/$p already exists (gh refuses without --clobber)" >&2; exit 1; } + cp "$FIX/assets/$tag/$p" "$dir/$p" + done ;; + upload) + tag="$1"; shift; repo=""; files=() + while [ "$#" -gt 0 ]; do case "$1" in --repo) repo="$2"; shift 2 ;; *) files+=("$1"); shift ;; esac; done + [ "$repo" = "$MIRROR" ] || { echo "gh: fake: upload to '$repo' is not the mirror" >&2; exit 1; } + jq -e --arg t "$tag" '.[] | select(.tag_name == $t)' "$STATE/mirror-releases.json" >/dev/null || { echo "gh: release not found" >&2; exit 1; } + for f in "${files[@]}"; do + [ -f "$f" ] || { echo "gh: fake: $f is not a file" >&2; exit 1; } + n="$(basename "$f")"; d="$(sha "$f")" + jq -e --arg t "$tag" --arg n "$n" '.[] | select(.tag_name == $t) | .assets[] | select(.name == $n)' "$STATE/mirror-releases.json" >/dev/null && { echo "gh: fake: asset '$n' already on '$tag' (HTTP 422)" >&2; exit 1; } + jq --arg t "$tag" --arg n "$n" --arg d "sha256:$d" 'map(if .tag_name == $t then .assets += [{name: $n, digest: $d}] else . end)' "$STATE/mirror-releases.json" >"$STATE/t.json" && mv "$STATE/t.json" "$STATE/mirror-releases.json" + done ;; + *) echo "gh: fake: unhandled release $sub" >&2; exit 1 ;; + esac ;; + *) echo "gh: fake: unhandled command $cmd" >&2; exit 1 ;; +esac +EOF +chmod +x "$SHIM/gh" "$SHIM/gitleaks" + +# ============================================================================= +# Fixtures: 12 stable releases v0.1.0..v0.1.11 (created a day apart) and one +# newer prerelease v0.1.12-rc.1. Each carries install.sh, install.ps1, +# SHA256SUMS, two "binaries" and their .sig/.cert. v0.1.3's source tag is +# annotated. The fixture list is deliberately NOT in date order. +# ============================================================================= +STABLE=(v0.1.0 v0.1.1 v0.1.2 v0.1.3 v0.1.4 v0.1.5 v0.1.6 v0.1.7 v0.1.8 v0.1.9 v0.1.10 v0.1.11) +PRE=v0.1.12-rc.1 +ALL=("${STABLE[@]}" "$PRE") +HEAD_SHA=1111111111111111111111111111111111111111 +ANNOT_TAG=v0.1.3; ANNOT_DATE=2025-12-31T10:00:00Z; ANNOT_MSG="tracebloc CLI v0.1.3 original annotation" + +# build_fixtures DIR [CORRUPT_SUMS_TAG] [BAD_BODY_TAG] +build_fixtures() { + local fix="$1" corrupt="${2:-}" badbody="${3:-}" i tag pre body created f + mkdir -p "$fix/assets" + printf 'acme/src' >"$fix/src-repo"; printf 'acme/mirror' >"$fix/mirror-repo"; printf '%s' "$HEAD_SHA" >"$fix/mirror-head" + : >"$fix/releases.ndjson"; : >"$fix/tags.ndjson"; printf '[]' >"$fix/src-tagobjs.json" + i=0 + for tag in "${ALL[@]}"; do + i=$((i + 1)); mkdir -p "$fix/assets/$tag" + printf '#!/bin/sh\necho install %s\n' "$tag" >"$fix/assets/$tag/install.sh" + printf 'Write-Host install %s\n' "$tag" >"$fix/assets/$tag/install.ps1" + for f in "tracebloc-$tag-linux-amd64" "tracebloc-$tag-darwin-arm64"; do + printf 'binary %s\n' "$f" >"$fix/assets/$tag/$f" + printf 'MEUCIQ%s\n' "$f" >"$fix/assets/$tag/$f.sig" + printf -- '-----BEGIN CERTIFICATE-----\n%s\n-----END CERTIFICATE-----\n' "$f" >"$fix/assets/$tag/$f.cert" + done + ( cd "$fix/assets/$tag" && for f in tracebloc-*; do case "$f" in *.sig|*.cert) ;; *) printf '%s %s\n' "$(sha256_of "$f")" "$f" ;; esac; done ) >"$fix/assets/$tag/SHA256SUMS" + if [ "$tag" = "$corrupt" ]; then + awk -v n="tracebloc-$tag-darwin-arm64" '$2 == n { $1 = "0000000000000000000000000000000000000000000000000000000000000000" } { print $1 " " $2 }' "$fix/assets/$tag/SHA256SUMS" >"$fix/s" && mv "$fix/s" "$fix/assets/$tag/SHA256SUMS" + fi + pre=false; [ "$tag" = "$PRE" ] && pre=true + created="$(printf '2026-01-%02dT12:00:00Z' "$i")" + body="## What's Changed\n* fix: something in $tag by @dev in https://github.com/acme/src/pull/$i" + [ "$tag" != "$badbody" ] || body="$body\n* ops: moved to role arn:aws:iam::000000000000:role/planted" + ( cd "$fix/assets/$tag" && for f in *; do printf '%s\t%s\t%s\n' "$f" "sha256:$(sha256_of "$f")" "$(wc -c <"$f" | tr -d ' ')"; done ) \ + | jq -R -s -c --arg tag "$tag" --arg pre "$pre" --arg created "$created" --arg body "$(printf "$body")" ' + split("\n") | map(select(length > 0) | split("\t") | {name: .[0], digest: .[1], size: (.[2]|tonumber)}) as $assets + | {tag_name: $tag, name: $tag, body: $body, draft: false, prerelease: ($pre == "true"), created_at: $created, published_at: $created, target_commitish: "develop", assets: $assets}' >>"$fix/releases.ndjson" + if [ "$tag" = "$ANNOT_TAG" ]; then + jq -n -c --arg t "$tag" '{ref: ("refs/tags/" + $t), object: {sha: "3333333333333333333333333333333333333333", type: "tag"}}' >>"$fix/tags.ndjson" + jq -n --arg m "$ANNOT_MSG" --arg d "$ANNOT_DATE" '[{sha: "3333333333333333333333333333333333333333", message: $m, tagger: {name: "dev", date: $d}, object: {sha: "cccccccccccccccccccccccccccccccccccccccc", type: "commit"}}]' >"$fix/src-tagobjs.json" + else + jq -n -c --arg t "$tag" --argjson i "$i" '{ref: ("refs/tags/" + $t), object: {sha: ("c" * 39 + ($i|tostring)|.[0:40]), type: "commit"}}' >>"$fix/tags.ndjson" + fi + done + # Shuffle the release order (newest in the middle) so sorting is the script's, not the fixture's. + jq -s '[.[12], .[3], .[0], .[11], .[7], .[1], .[10], .[2], .[9], .[4], .[8], .[5], .[6]]' "$fix/releases.ndjson" >"$fix/src-releases.json" + jq -s '.' "$fix/tags.ndjson" >"$fix/src-tags.json" +} +fresh_state() { # DIR — an empty mirror state + mkdir -p "$1"; printf '[]' >"$1/mirror-releases.json"; printf '[]' >"$1/mirror-tags.json"; printf '[]' >"$1/mirror-tagobjs.json" +} + +FIX="$ROOT/fix"; build_fixtures "$FIX" +FIX_CORRUPT="$ROOT/fix-corrupt"; build_fixtures "$FIX_CORRUPT" v0.1.11 +FIX_BADBODY="$ROOT/fix-badbody"; build_fixtures "$FIX_BADBODY" "" v0.1.5 + +# run STATE_DIR FIX_DIR ARGS... — the script with the fake gh; OUTPUT, RC, GH_LOG set. +run() { + local state="$1" fix="$2"; shift 2 + GH_LOG="$ROOT/gh-$RANDOM$RANDOM.log"; : >"$GH_LOG" + OUTPUT="$(PATH="$SHIM:$PATH" GH_LOG="$GH_LOG" FAKE_GH_FIX="$fix" FAKE_GH_STATE="$state" PUBLISH_GUARD_GITLEAKS="$SHIM/gitleaks" \ + BACKFILL_SCRIPTS_DIR="$SCRIPTS_DIR" SOURCE_REPO="${SOURCE_REPO-acme/src}" MIRROR_REPO="${MIRROR_REPO-mirror}" BINARY_KEEP="${BINARY_KEEP-10}" \ + bash "$BACKFILL" "$@" 2>&1)"; RC=$? +} +has() { [[ "$OUTPUT" == *"$1"* ]]; } +hasg() { [[ "$OUTPUT" == *$1* ]]; } # $1 is a glob: `a*b`; escape [ ] as \[ \] +writes() { grep -cE '^(api -X (POST|PATCH)|release upload) ' "$GH_LOG" || true; } +posts() { grep -c "^api -X POST repos/acme/mirror/$1 " "$GH_LOG" || true; } +verdicts() { printf '%s\n' "$OUTPUT" | awk -v v="$1" '$NF == v && $1 ~ /^v[0-9]/ { n++ } END { print n + 0 }'; } + +echo "== backfill-releases.sh harness ==" + +# ---- 1. dry-run (the default) writes nothing --------------------------------------------- +S1="$ROOT/s1"; fresh_state "$S1" +run "$S1" "$FIX" +if [ "$RC" -eq 0 ] && [ "$(writes)" -eq 0 ] && [ "$(verdicts planned)" -eq 12 ] && has "12 release(s) match the filter, 12 in this run; binaries for the newest 10" && has "mode=dry-run" \ + && has "v0.1.0 [stable] would: tag: create | release: create | text: 3 up/0 skip | binaries: none (older than the newest 10)" \ + && has "v0.1.11 [stable] would: tag: create | release: create | text: 3 up/0 skip | binaries: 2 up/0 skip (+4/0 sig+cert)" \ + && grep -q '^release download v0.1.11 --repo acme/src --dir .*/text --pattern' "$GH_LOG" && ! grep -q -- '--pattern tracebloc-' "$GH_LOG"; then + ok "dry-run (default): every read runs, text assets are fetched for the guard, no binary is fetched, zero writes, 12 rows planned" +else bad "dry-run (rc=$RC writes=$(writes) planned=$(verdicts planned)): $OUTPUT"; fi +if [ "$(jq length "$S1/mirror-releases.json")" -eq 0 ] && [ "$(jq length "$S1/mirror-tags.json")" -eq 0 ]; then ok "dry-run: the mirror state is untouched"; else bad "dry-run touched the mirror state"; fi + +# ---- 2. --apply makes exactly the expected writes ----------------------------------------- +S2="$ROOT/s2"; fresh_state "$S2" +run "$S2" "$FIX" --apply +first_rel="$(grep '^api -X POST repos/acme/mirror/releases ' "$GH_LOG" | head -1)" +last_rel="$(grep '^api -X POST repos/acme/mirror/releases ' "$GH_LOG" | tail -1)" +if [ "$RC" -eq 0 ] && [ "$(writes)" -eq 48 ] && [ "$(posts git/tags)" -eq 12 ] && [ "$(posts git/refs)" -eq 12 ] && [ "$(posts releases)" -eq 12 ] \ + && [ "$(grep -c '^release upload ' "$GH_LOG")" -eq 12 ] && [ "$(verdicts "done")" -eq 12 ] && has "12 release(s) in this run — 12 written, 0 already complete, 0 refused"; then + ok "apply: 12 releases → exactly 48 writes (tag object, ref, release, one upload call each), all 12 done" +else bad "apply (rc=$RC writes=$(writes) tags=$(posts git/tags) refs=$(posts git/refs) rel=$(posts releases)): $OUTPUT"; fi +if [[ "$first_rel" == *"-f tag_name=v0.1.0 "*"-f make_latest=false"* ]] && [[ "$last_rel" == *"-f tag_name=v0.1.11 "*"-f make_latest=true"* ]] \ + && [ "$(grep -c -- '-f make_latest=true' "$GH_LOG")" -eq 1 ] && ! grep -q 'v0.1.12-rc.1' "$GH_LOG"; then + ok "apply: oldest first, newest stable last and the only make_latest=true; the prerelease is never touched" +else bad "apply order/latest: first='$first_rel' last='$last_rel'"; fi +if [ "$(jq -r '.[] | .tag_name' "$S2/mirror-releases.json" | paste -sd' ' -)" = "${STABLE[*]}" ] && [ "$(jq -r '[.[] | select(.prerelease)] | length' "$S2/mirror-releases.json")" -eq 0 ] \ + && [ "$(jq -r '.[] | select(.tag_name == "v0.1.11") | .assets | length' "$S2/mirror-releases.json")" -eq 9 ] && [ "$(jq -r '.[] | select(.tag_name == "v0.1.0") | .assets | length' "$S2/mirror-releases.json")" -eq 3 ]; then + ok "apply: the mirror holds the 12 stable releases; the newest carries 9 assets (3 text + 2 binaries + 4 sig/cert), the oldest 3" +else bad "apply mirror state: $(jq -c '[.[] | {tag_name, n: (.assets|length)}]' "$S2/mirror-releases.json")"; fi +tag0="$(jq -r '.[] | select(.tag == "v0.1.0")' "$S2/mirror-tagobjs.json")"; tag3="$(jq -r '.[] | select(.tag == "v0.1.3")' "$S2/mirror-tagobjs.json")" +if [ "$(printf '%s' "$tag0" | jq -r .object.sha)" = "$HEAD_SHA" ] && [ "$(printf '%s' "$tag0" | jq -r .tagger.date)" = "2026-01-01T12:00:00Z" ] \ + && [[ "$(printf '%s' "$tag0" | jq -r .message)" == *"Mirror release marker for v0.1.0"*"not at the sources"* ]] \ + && [ "$(printf '%s' "$tag3" | jq -r .tagger.date)" = "$ANNOT_DATE" ] && [[ "$(printf '%s' "$tag3" | jq -r .message)" == *"--- original tag message ---"*"$ANNOT_MSG"* ]] \ + && [ "$(jq -r '[.[] | .object.sha] | unique | length' "$S2/mirror-tagobjs.json")" -eq 1 ]; then + ok "tags: every mirror tag is an annotated marker on the mirror head; the date is the release's, or the source tag's own when annotated, with its message carried" +else bad "tags: v0.1.0=$(printf '%s' "$tag0" | jq -c .) v0.1.3=$(printf '%s' "$tag3" | jq -c .)"; fi +body0="$(jq -r '.[] | select(.tag_name == "v0.1.0") | .body' "$S2/mirror-releases.json")"; body11="$(jq -r '.[] | select(.tag_name == "v0.1.11") | .body' "$S2/mirror-releases.json")" +if [[ "$body0" == "tracebloc CLI v0.1.0."*"verify it against SHA256SUMS"*"originally published 2026-01-01T12:00:00Z"*"re-run the installer"* ]] && [[ "$body0" != *"What's Changed"* ]] && [[ "$body0" != *"acme/src/pull/"* ]] \ + && [[ "$body11" == "tracebloc CLI v0.1.11."*"originally published 2026-01-12T12:00:00Z"* ]] && [[ "$body11" != *"acme/src/pull/"* ]] && [[ "$body11" != *"re-run the installer"* ]] && has "notes=fixed"; then + ok "notes (default): the workflow's fixed text plus an original-date footer, no trace of the source body; the installer hint appears only where binaries are not carried" +else bad "notes default: body0='$body0' body11='$body11'"; fi +# The tag is created before its release (the fake refuses the other order) and the release before its uploads. +if [ "$(grep -nE '^(api -X POST repos/acme/mirror/(git/tags|git/refs|releases)|release upload) ' "$GH_LOG" | head -4 | sed -E 's/^[0-9]+://; s/ .*//' | paste -sd' ' -)" = "api api api release" ]; then + ok "apply: per release the order is tag object, ref, release, upload" +else bad "apply order: $(head -8 "$GH_LOG")"; fi +# --notes source is the explicit opt-in that carries the source body. +S2B="$ROOT/s2b"; fresh_state "$S2B" +run "$S2B" "$FIX" --apply --notes source +body0="$(jq -r '.[] | select(.tag_name == "v0.1.0") | .body' "$S2B/mirror-releases.json")"; body11="$(jq -r '.[] | select(.tag_name == "v0.1.11") | .body' "$S2B/mirror-releases.json")" +if [ "$RC" -eq 0 ] && has "notes=source" && [[ "$body0" == "## What's Changed"*"acme/src/pull/1"*"originally published 2026-01-01T12:00:00Z"*"re-run the installer"* ]] && [[ "$body0" != *"tracebloc CLI v0.1.0."* ]] \ + && [[ "$body11" == "## What's Changed"*"acme/src/pull/12"*"originally published 2026-01-12T12:00:00Z"* ]] && [[ "$body11" != *"re-run the installer"* ]]; then + ok "--notes source: the source body is carried with the same original-date footer, and only when asked for" +else bad "notes source (rc=$RC): body0='$body0' body11='$body11'"; fi +run "$S2B" "$FIX" --notes generated +if [ "$RC" -eq 2 ] && has "--notes must be 'fixed' or 'source', not 'generated'" && [ "$(wc -l <"$GH_LOG" | tr -d ' ')" -eq 0 ]; then + ok "--notes with anything else is could-not-tell before any gh call" +else bad "notes bogus (rc=$RC): $OUTPUT"; fi + +# ---- 3. a second --apply writes nothing --------------------------------------------------- +before="$(cat "$S2/mirror-releases.json" "$S2/mirror-tags.json" | sha256_of /dev/stdin)" +run "$S2" "$FIX" --apply +after="$(cat "$S2/mirror-releases.json" "$S2/mirror-tags.json" | sha256_of /dev/stdin)" +if [ "$RC" -eq 0 ] && [ "$(writes)" -eq 0 ] && [ "$(verdicts skipped)" -eq 12 ] && has "12 release(s) in this run — 0 written, 12 already complete, 0 refused" && [ "$before" = "$after" ] \ + && ! grep -q '^release download .* --pattern install' "$GH_LOG"; then + ok "idempotent: a second --apply over the same mirror makes zero writes, downloads no asset it can compare by digest, and reports 12 already complete" +else bad "idempotent (rc=$RC writes=$(writes) skipped=$(verdicts skipped)): $OUTPUT"; fi + +# ---- 4. BINARY_KEEP boundary --------------------------------------------------------------- +# Newest 10 of the 12 stable: v0.1.2 (10th) carries binaries, v0.1.1 (11th) does not. +if [ "$(jq -r '.[] | select(.tag_name == "v0.1.2") | .assets | length' "$S2/mirror-releases.json")" -eq 9 ] && [ "$(jq -r '.[] | select(.tag_name == "v0.1.1") | .assets | length' "$S2/mirror-releases.json")" -eq 3 ] \ + && [ "$(jq -r '.[] | select(.tag_name == "v0.1.1") | [.assets[].name] | sort | join(" ")' "$S2/mirror-releases.json")" = "SHA256SUMS install.ps1 install.sh" ]; then + ok "BINARY_KEEP=10: the 10th newest (v0.1.2) carries binaries + sig/cert, the 11th (v0.1.1) carries exactly the three text assets" +else bad "binary-keep boundary: v0.1.2=$(jq -c '.[] | select(.tag_name == "v0.1.2") | [.assets[].name]' "$S2/mirror-releases.json") v0.1.1=$(jq -c '.[] | select(.tag_name == "v0.1.1") | [.assets[].name]' "$S2/mirror-releases.json")"; fi +S4="$ROOT/s4"; fresh_state "$S4" +BINARY_KEEP=1 run "$S4" "$FIX" +if [ "$RC" -eq 0 ] && hasg "v0.1.11 \[stable\] would: *binaries: 2 up/0 skip" && hasg "v0.1.10 \[stable\] would: *binaries: none (older than the newest 1)"; then + ok "BINARY_KEEP is read: with 1, only the newest release carries binaries" +else bad "BINARY_KEEP=1 (rc=$RC): $OUTPUT"; fi +BINARY_KEEP=ten run "$S4" "$FIX" +if [ "$RC" -eq 2 ] && has "BINARY_KEEP 'ten' is not a non-negative integer"; then ok "BINARY_KEEP that is not a number is could-not-tell"; else bad "BINARY_KEEP=ten (rc=$RC): $OUTPUT"; fi + +# ---- 5. a binary that disagrees with SHA256SUMS is refused by name --------------------------- +S5="$ROOT/s5"; fresh_state "$S5" +run "$S5" "$FIX_CORRUPT" --apply +if [ "$RC" -eq 1 ] && has "REFUSED v0.1.11 — binary 'tracebloc-v0.1.11-darwin-arm64' hashes to " && has "but the source release's SHA256SUMS says 0000000000000000000000000000000000000000000000000000000000000000 — not uploaded" \ + && [ "$(verdicts refused)" -eq 1 ] && [ "$(verdicts "done")" -eq 11 ] && ! grep -q 'v0.1.11' <(grep -E '^(api -X POST|release upload) ' "$GH_LOG") && has "1 release(s) were refused" \ + && [ "$(jq -r '[.[] | select(.tag_name == "v0.1.11")] | length' "$S5/mirror-releases.json")" -eq 0 ] && [ "$(jq length "$S5/mirror-releases.json")" -eq 11 ]; then + ok "sha mismatch: the binary is named, nothing of that release is written (no tag, no release, no upload), the other 11 go ahead, exit 1" +else bad "sha mismatch (rc=$RC refused=$(verdicts refused) done=$(verdicts "done")): $OUTPUT"; fi +# v0.1.11 is the newest stable, so its refusal is the "no latest" case: every +# other release was created with make_latest=false. The newest stable the run +# did write (v0.1.10) must be marked latest, by exactly one PATCH. +if [ "$(grep -c '^api -X PATCH repos/acme/mirror/releases/' "$GH_LOG")" -eq 1 ] && has "latest: v0.1.11 was refused — marking v0.1.10, the newest stable release written in this run, as latest until v0.1.11 is re-run" \ + && [ "$(jq -r '.[] | select(.tag_name == "v0.1.10") | .make_latest' "$S5/mirror-releases.json")" = true ] && [ "$(jq -r '[.[] | select(.make_latest == "true")] | length' "$S5/mirror-releases.json")" -eq 1 ]; then + ok "sha mismatch on the newest stable: the newest stable release the run did write is marked latest — releases/latest does not 404 behind a refusal" +else bad "latest fallback (patches=$(grep -c '^api -X PATCH' "$GH_LOG" || true)): $(jq -c '[.[] | {tag_name, make_latest}]' "$S5/mirror-releases.json")"; fi +run "$S5" "$FIX_CORRUPT" +if [ "$RC" -eq 0 ] && [ "$(verdicts planned)" -eq 1 ] && [ "$(verdicts skipped)" -eq 11 ] && [ "$(writes)" -eq 0 ]; then + ok "sha mismatch: a dry-run afterwards plans only the refused release (binaries are checked at apply, not fetched for a plan)" +else bad "sha mismatch dry-run after (rc=$RC planned=$(verdicts planned) skipped=$(verdicts skipped)): $OUTPUT"; fi + +# ---- 6. mirror unset / equal to the source: publish-mirror's rule ---------------------------- +S6="$ROOT/s6"; fresh_state "$S6" +MIRROR_REPO='' run "$S6" "$FIX" --apply; a="$RC"; o1="$OUTPUT"; l1="$(wc -l <"$GH_LOG" | tr -d ' ')" +MIRROR_REPO=src run "$S6" "$FIX" --apply; b="$RC"; o2="$OUTPUT"; l2="$(wc -l <"$GH_LOG" | tr -d ' ')" +if [ "$a" -eq 1 ] && [[ "$o1" == *"publish-mirror: REFUSED — no mirror repository is configured (MIRROR_REPO is unset)"* ]] && [[ "$o1" == *"backfill-releases: REFUSED — the mirror target was refused above"* ]] \ + && [ "$b" -eq 1 ] && [[ "$o2" == *"publish-mirror: REFUSED — mirror 'acme/src' is this repository"* ]] && [ "$l1" -eq 0 ] && [ "$l2" -eq 0 ]; then + ok "mirror unset, or equal to the source, is refused by publish-mirror's own rule before any gh call" +else bad "mirror target (a=$a b=$b calls=$l1/$l2): $o1 / $o2"; fi +MIRROR_REPO=SRC run "$S6" "$FIX" +if [ "$RC" -eq 1 ] && has "is this repository"; then ok "mirror equal to the source is refused case-insensitively"; else bad "mirror case (rc=$RC): $OUTPUT"; fi + +# ---- 7. source derived from gh repo view when SOURCE_REPO is unset ---------------------------- +S7="$ROOT/s7"; fresh_state "$S7" +SOURCE_REPO='' run "$S7" "$FIX" +if [ "$RC" -eq 0 ] && has "source acme/src → mirror acme/mirror" && [ "$(head -1 "$GH_LOG")" = "repo view --json nameWithOwner" ]; then + ok "SOURCE_REPO unset: the source is what gh repo view reports, never a hardcoded name" +else bad "source derivation (rc=$RC): $(head -2 "$GH_LOG") / $OUTPUT"; fi +if ! grep -qE 'tracebloc/cli|"tracebloc"' "$REAL"; then ok "the script hardcodes no repository name"; else bad "a repository name is hardcoded in $REAL"; fi + +# ---- 8. prereleases: excluded by default, included on request, and they move the boundary ----- +S8="$ROOT/s8"; fresh_state "$S8" +run "$S8" "$FIX" --include-prerelease +if [ "$RC" -eq 0 ] && [ "$(verdicts planned)" -eq 13 ] && has "13 release(s) match the filter" && has "v0.1.12-rc.1 [prerelease] would: tag: create | release: create | text: 3 up/0 skip | binaries: 2 up/0 skip" \ + && hasg "v0.1.2 \[stable\] would: *binaries: none (older than the newest 10)" && hasg "v0.1.3 \[stable\] would: *binaries: 2 up/0 skip"; then + ok "--include-prerelease: the rc is planned, and being the newest it takes a binary slot — v0.1.2 drops out of the newest 10" +else bad "include-prerelease (rc=$RC planned=$(verdicts planned)): $OUTPUT"; fi +run "$S8" "$FIX" --include-prerelease --apply +if [ "$RC" -eq 0 ] && [ "$(grep -c -- '-F prerelease=true' "$GH_LOG")" -eq 1 ] && [ "$(grep -c -- '-f tag_name=v0.1.12-rc.1 ' "$GH_LOG")" -eq 1 ] \ + && [[ "$(grep -- '-f tag_name=v0.1.12-rc.1 ' "$GH_LOG")" == *"-f make_latest=false"* ]] && [[ "$(grep -- '-f tag_name=v0.1.11 ' "$GH_LOG")" == *"-f make_latest=true"* ]]; then + ok "--include-prerelease --apply: the rc is created as a prerelease and is never make_latest; the newest STABLE is" +else bad "include-prerelease apply (rc=$RC): $(grep -- 'tag_name=v0.1.1' "$GH_LOG")"; fi + +# ---- 9. a read that fails is could-not-tell, naming the call ---------------------------------- +S9="$ROOT/s9"; fresh_state "$S9" +FAKE_GH_FAIL_RE='^api --paginate repos/acme/src/releases$' run "$S9" "$FIX" --apply; a="$RC"; o1="$OUTPUT"; w1="$(writes)" +FAKE_GH_FAIL_RE='^api --paginate repos/acme/mirror/releases$' run "$S9" "$FIX" --apply; b="$RC"; o2="$OUTPUT"; w2="$(writes)" +FAKE_GH_FAIL_RE='^release download v0.1.4 ' run "$S9" "$FIX" --apply; c="$RC"; o3="$OUTPUT" +if [ "$a" -eq 2 ] && [[ "$o1" == *"COULD NOT TELL — gh api --paginate repos/acme/src/releases failed: gh: Internal Server Error (HTTP 500)"* ]] && [ "$w1" -eq 0 ] \ + && [ "$b" -eq 2 ] && [[ "$o2" == *"COULD NOT TELL — gh api --paginate repos/acme/mirror/releases failed"* ]] && [ "$w2" -eq 0 ] \ + && [ "$c" -eq 2 ] && [[ "$o3" == *"COULD NOT TELL — gh release download v0.1.4 --repo acme/src"*"failed"* ]]; then + ok "a failing read (source list, mirror list, an asset download) is exit 2 naming the call — never 'no releases', never a write" +else bad "api failure (a=$a b=$b c=$c w=$w1/$w2): $o1 / $o2 / $o3"; fi +if [ "$(jq -r '[.[] | select(.tag_name | test("^v0.1.[0-3]$"))] | length' "$S9/mirror-releases.json")" -eq 4 ]; then + S9B="$ROOT/s9b"; fresh_state "$S9B" + FAKE_GH_FAIL_RE='^release download v0.1.4 ' run "$S9B" "$FIX" --apply + run "$S9B" "$FIX" --apply + if [ "$RC" -eq 0 ] && [ "$(verdicts skipped)" -eq 4 ] && [ "$(verdicts "done")" -eq 8 ] && [ "$(jq length "$S9B/mirror-releases.json")" -eq 12 ]; then + ok "resume after a mid-run failure: the completed releases are skipped, the rest are written, the mirror ends complete" + else bad "resume (rc=$RC skipped=$(verdicts skipped) done=$(verdicts "done")): $OUTPUT"; fi +else bad "mid-run failure did not stop at v0.1.4: $(jq -c '[.[].tag_name]' "$S9/mirror-releases.json")"; fi +FAKE_GH_EMPTY_MIRROR=1 run "$S9" "$FIX" --apply +if [ "$RC" -eq 1 ] && has "REFUSED — mirror 'acme/mirror' has no commit on 'main' to anchor tags to; publish the README first" && [ "$(writes)" -eq 0 ]; then + ok "an empty mirror is refused with instructions — tags are never anchored to an invented commit" +else bad "empty mirror (rc=$RC): $OUTPUT"; fi +# A read that returns garbage is parsed inside "$(jq_of …)" — a subshell. The +# reason must still reach the operator (die2 writes it to stderr; on stdout it +# would be captured into the variable and lost) and the run must still end 2. +S9C="$ROOT/s9c"; fresh_state "$S9C" +FAKE_GH_GARBLE_RE='^api repos/acme/mirror$' run "$S9C" "$FIX" --apply +if [ "$RC" -eq 2 ] && [[ "$OUTPUT" == *"COULD NOT TELL — could not parse "*"/mirror.json with '.full_name':"* ]] && [ "$(writes)" -eq 0 ]; then + ok "a read that returns non-JSON, parsed inside a \$(...) substitution, is exit 2 naming the file and filter — the reason reaches the operator, not the variable" +else bad "garbled read in a substitution (rc=$RC writes=$(writes)): $OUTPUT"; fi + +# ---- 10. under --notes source a refuse-tier needle in a body refuses that release, naming the tier; +# the default never reads the body into the notes, so the same release goes through ------- +S10="$ROOT/s10"; fresh_state "$S10" +run "$S10" "$FIX_BADBODY" --apply --notes source +if [ "$RC" -eq 1 ] && has "REFUSED v0.1.5 — the guard refused the notes or a text asset: [forbidden-strings] REFUSED — [strings-refuse] needle 'arn:aws:' found in 1 staged line(s)" \ + && has "assets/RELEASE_NOTES.md:" && ! has "role/planted" && [ "$(verdicts refused)" -eq 1 ] && [ "$(verdicts "done")" -eq 11 ] \ + && ! grep -q 'v0.1.5' <(grep -E '^(api -X POST|release upload) ' "$GH_LOG") && [ "$(jq -r '[.[] | select(.tag_name == "v0.1.5")] | length' "$S10/mirror-releases.json")" -eq 0 ]; then + ok "--notes source, forbidden string in a body: the release is refused naming the tier and the notes file, the text is not echoed, nothing of it is written, exit 1" +else bad "bad body under --notes source (rc=$RC refused=$(verdicts refused)): $OUTPUT"; fi +run "$S10" "$FIX_BADBODY" --apply +if [ "$RC" -eq 0 ] && [ "$(verdicts "done")" -eq 1 ] && [[ "$(jq -r '.[] | select(.tag_name == "v0.1.5") | .body' "$S10/mirror-releases.json")" == "tracebloc CLI v0.1.5."*"originally published 2026-01-06T12:00:00Z"* ]] \ + && ! grep -q 'role/planted' "$S10/mirror-releases.json"; then + ok "default notes: the same release goes through with the workflow's fixed text plus the date footer; the planted body never reaches the mirror" +else bad "bad body under the default notes (rc=$RC): $OUTPUT"; fi +S10B="$ROOT/s10b"; fresh_state "$S10B" +run "$S10B" "$FIX_BADBODY" --apply +if [ "$RC" -eq 0 ] && [ "$(verdicts "done")" -eq 12 ] && [ "$(jq length "$S10B/mirror-releases.json")" -eq 12 ] && ! grep -q 'role/planted' "$S10B/mirror-releases.json"; then + ok "default notes on a fresh mirror: all 12 written, none refused — a bad source body is not a reason to hold up a release the mirror never quotes" +else bad "fresh mirror, default notes (rc=$RC done=$(verdicts "done")): $OUTPUT"; fi + +# ---- 11. present-but-different is refused, never replaced; a dangling mirror tag is refused ------ +S11="$ROOT/s11"; fresh_state "$S11" +jq -n '[{id: 1, tag_name: "v0.1.9", name: "v0.1.9", body: "x", prerelease: false, draft: false, assets: [{name: "install.sh", digest: "sha256:deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"}]}]' >"$S11/mirror-releases.json" +jq -n --arg h "$HEAD_SHA" '[{ref: "refs/tags/v0.1.9", object: {sha: $h, type: "commit"}}, {ref: "refs/tags/v0.1.7", object: {sha: "9999999999999999999999999999999999999999", type: "commit"}}]' >"$S11/mirror-tags.json" +run "$S11" "$FIX" --apply +if [ "$RC" -eq 1 ] && has "REFUSED v0.1.9 — asset 'install.sh' is on the mirror with SHA256 deadbeef" && has "a published asset is never replaced" \ + && has "REFUSED v0.1.7 — tag 'v0.1.7' exists on the mirror but points at commit 9999999999999999999999999999999999999999, which the mirror does not have — a dangling tag is not repointed" \ + && [ "$(verdicts refused)" -eq 2 ] && [ "$(verdicts "done")" -eq 10 ] && ! grep -qE 'v0.1.(7|9)' <(grep -E '^(api -X POST|release upload) ' "$GH_LOG"); then + ok "present-with-a-different-digest and a dangling mirror tag are each refused by name, nothing of those releases is written, the other 10 go ahead" +else bad "present/dangling (rc=$RC refused=$(verdicts refused) done=$(verdicts "done")): $OUTPUT"; fi + +# ---- 12. --only-tag / --from-tag narrow the run, not the binary decision -------------------------- +S12="$ROOT/s12"; fresh_state "$S12" +run "$S12" "$FIX" --apply --only-tag v0.1.1 +if [ "$RC" -eq 0 ] && [ "$(writes)" -eq 4 ] && has "12 release(s) match the filter, 1 in this run" && [ "$(jq -r '.[0].assets | length' "$S12/mirror-releases.json")" -eq 3 ]; then + ok "--only-tag: one release, 4 writes; v0.1.1 stays outside the newest 10 even when it is the only release in the run" +else bad "only-tag (rc=$RC writes=$(writes)): $OUTPUT"; fi +run "$S12" "$FIX" --apply --from-tag v0.1.10 +if [ "$RC" -eq 0 ] && [ "$(writes)" -eq 8 ] && has "2 in this run" && [ "$(jq -r '[.[].tag_name] | join(" ")' "$S12/mirror-releases.json")" = "v0.1.1 v0.1.10 v0.1.11" ]; then + ok "--from-tag: that release and every newer one, oldest first" +else bad "from-tag (rc=$RC writes=$(writes)): $OUTPUT / $(jq -c '[.[].tag_name]' "$S12/mirror-releases.json")"; fi +run "$S12" "$FIX" --only-tag v9.9.9; a="$RC"; o1="$OUTPUT" +run "$S12" "$FIX" --only-tag v0.1.12-rc.1; b="$RC"; o2="$OUTPUT" +run "$S12" "$FIX" --from-tag v0.1.1 --only-tag v0.1.2; c="$RC"; o3="$OUTPUT" +if [ "$a" -eq 2 ] && [[ "$o1" == *"--only-tag 'v9.9.9' is not a release of 'acme/src' matching the filter"* ]] && [ "$b" -eq 2 ] && [[ "$o2" == *"is not a release of 'acme/src' matching the filter"* ]] \ + && [ "$c" -eq 2 ] && [[ "$o3" == *"--from-tag and --only-tag exclude each other"* ]]; then + ok "an unknown tag, a filtered-out prerelease, or both flags at once are could-not-tell" +else bad "tag flags (a=$a b=$b c=$c): $o1 / $o2 / $o3"; fi + +# ---- 13. the guard is really consulted: a strict run refuses the report tier ---------------------- +S13="$ROOT/s13"; fresh_state "$S13" +FIX_REPORT="$ROOT/fix-report"; build_fixtures "$FIX_REPORT" +jq '(.[] | select(.tag_name == "v0.1.6") | .body) |= . + "\n* tested against https://dev-api.tracebloc.io"' "$FIX_REPORT/src-releases.json" >"$FIX_REPORT/t.json" && mv "$FIX_REPORT/t.json" "$FIX_REPORT/src-releases.json" +run "$S13" "$FIX_REPORT" --notes source; a="$RC"; o1="$OUTPUT" +run "$S13" "$FIX_REPORT" --notes source --strict; b="$RC"; o2="$OUTPUT" +run "$S13" "$FIX_REPORT" --strict; c="$RC"; o3="$OUTPUT" +if [ "$a" -eq 0 ] && [ "$b" -eq 1 ] && [[ "$o2" == *"REFUSED v0.1.6 — the guard refused"*"[strings-report (strict)] needle 'dev-api\.tracebloc\.io' found in 1 staged line(s)"* ]]; then + ok "--strict is passed to the guard: under --notes source a report-tier needle in a body is counted, and refuses under --strict, tier named" +else bad "strict (a=$a b=$b): $o1 / $o2"; fi +if [ "$c" -eq 0 ] && [ "$(printf '%s\n' "$o3" | awk '$NF == "planned" && $1 ~ /^v[0-9]/ { n++ } END { print n + 0 }')" -eq 12 ] && [[ "$o3" != *"dev-api"* ]]; then + ok "--strict with the default notes: the report-tier body is never staged, so all 12 are planned — the reason fixed notes are the default" +else bad "strict default notes (c=$c): $o3"; fi + +# ---- 14. the guard's scratch commit is unsigned even under a global commit.gpgsign ----------- +# The script runs on a human's machine. A global gpgsign that cannot sign as +# backfill@localhost must not end the run before a release is planned. +S14="$ROOT/s14"; fresh_state "$S14" +cat >"$ROOT/gpg-fail" <<'EOF' +#!/usr/bin/env bash +echo "gpg: signing failed: No secret key" >&2; exit 2 +EOF +chmod +x "$ROOT/gpg-fail" +printf '[commit]\n\tgpgsign = true\n[gpg]\n\tprogram = %s\n' "$ROOT/gpg-fail" >"$ROOT/gitconfig-gpgsign" +GIT_CONFIG_GLOBAL="$ROOT/gitconfig-gpgsign" run "$S14" "$FIX" +if [ "$RC" -eq 0 ] && [ "$(verdicts planned)" -eq 12 ] && ! has "could not commit the guard's scratch checkout"; then + ok "a global commit.gpgsign=true with a failing signer does not end the run — the guard's scratch commit is made unsigned, all 12 planned" +else bad "gpgsign (rc=$RC planned=$(verdicts planned)): $OUTPUT"; fi + +echo +printf 'backfill-releases-verify: %d passed, %d failed\n' "$PASS" "$FAIL" +[ "$FAIL" -eq 0 ] && [ "$PASS" -ge 30 ]