Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions .abcd/config/reading-presets.json
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,10 @@
"test"
],
"window": {
"tokens_est": 1020000,
"measured_tokens_est": 1009052,
"measured_bytes": 3884854,
"measured_at": "e1bbef2864f42c2fa8c21f7eaf97d0747580fc68"
"tokens_est": 1040000,
"measured_tokens_est": 1021262,
"measured_bytes": 3931862,
"measured_at": "1dff258e9ea7ae29841faaefc47966913fbbd954"
}
},
"entailment": {
Expand Down Expand Up @@ -133,9 +133,9 @@
],
"window": {
"tokens_est": 350000,
"measured_tokens_est": 340918,
"measured_bytes": 1312538,
"measured_at": "e1bbef2864f42c2fa8c21f7eaf97d0747580fc68"
"measured_tokens_est": 343062,
"measured_bytes": 1320792,
"measured_at": "1dff258e9ea7ae29841faaefc47966913fbbd954"
}
},
"comparative": {
Expand Down Expand Up @@ -216,10 +216,10 @@
"test"
],
"window": {
"tokens_est": 1030000,
"measured_tokens_est": 1018088,
"measured_bytes": 3919642,
"measured_at": "e1bbef2864f42c2fa8c21f7eaf97d0747580fc68"
"tokens_est": 1050000,
"measured_tokens_est": 1030298,
"measured_bytes": 3966650,
"measured_at": "1dff258e9ea7ae29841faaefc47966913fbbd954"
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
schema_version: 1
id: "iss-2609260319483365"
slug: "the-race-lane-runs-internal-surface-cli-at-the-edge-of-go"
severity: "minor"
category: "bug"
source: "agent-finding"
found_during: "autonomous run A resumed 2026-09-25"
origin: researcher-authored
production_mode: hand-written
found_at: ".github/workflows/ci.yml"
resolution: "The race lane runs under a declared go test -race -timeout 20m in ci.yml's check job, release.yml's verify job (and its scaffold template) and make preflight, which lifts go test's 10m per-package default: a slow package no longer fails at ten minutes, the failure that ejected PR #717. On merge-group runs the merge queue's 30-minute check response cap (the main ruleset, mirrored in .abcd/work/rulesets/main-protection.json) is the outer ceiling, so ci.yml's check job stays at 30 minutes, equal to that cap; on the macOS leg the slowest package starts about 18 minutes in, so a genuine hang there is still cancelled at 30 without a goroutine dump. Raising the queue's cap is an admin act on the live ruleset, left to the technical facilitator. release.yml's verify job is not a merge-queue job; its ceiling rises from 15 to 35 minutes so the package timeout fires there before the job is cancelled. TestRaceLaneBudgetIsDeclaredAndFitsItsJob holds the three commands to one explicit timeout, the check job's timeout-minutes at or below the mirror's merge-queue cap, and verify's timeout-minutes to the package timeout plus its measured headroom (the check-job half corrected in 2f7b903b)."
impact: internal
resolved_by:
commit: "f0994f70"
---

The race lane runs internal/surface/cli at the edge of go test's default 10-minute per-package timeout on the macOS runner, so a slow runner ejects an unrelated pull request from the merge queue. PR #717 was ejected by merge_group run 36212607621 (job check (macos-latest), step Test (race, internal)): panic: test timed out after 10m0s and FAIL github.com/intentdriven/abcd/internal/surface/cli 600.120s, with no test hung; the package's total wall time crossed the default. The same package under -race on macOS took 417.9s (run 36211489533), 472.0s (36211390371) and 567.9s (36209973364), and 324-355s on ubuntu. ci.yml's race step, release.yml's verify race step and the Makefile preflight all run go test -race ./internal/... with no -timeout, so the ceiling is the implicit default rather than a budget sized from measurement. The job ceilings around the step are as tight: the macOS check job took 19.0, 21.4, 25.9 and (failing) 28.1 minutes against timeout-minutes: 30, so lifting the package timeout alone moves the failure to the job cap, where it arrives with no goroutine dump; and release.yml's verify job, ubuntu only with an uncached toolchain, took 13.2 minutes of its 15 on 2026-09-24 (run 35963282477, cli race 302s), since when the ubuntu race step has grown from 9.8 to about 12 minutes, so the next release's verify job is at risk of the same cancellation.

## Grounds

- pursued: the macOS race step stops failing when internal/surface/cli's total -race wall time crosses 600s, while a genuinely hung test still fails with a goroutine dump at 20m in release.yml's verify job; shown wrong by a merge-queue ejection from a package-timeout panic under 20m on a passing package, by a verify job cancelled at its ceiling before the race step's own timeout could fire, or by a check job set above the merge queue's response cap. A merge-group check cancelled at the 30-minute cap does not show it wrong: that ceiling is the queue's, and the record says so
25 changes: 24 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,17 @@ jobs:
check:
needs: changes
if: ${{ !cancelled() }}
# Equal to the merge queue's check response timeout (30 minutes, recorded in
# .abcd/work/rulesets/main-protection.json), which is the outer ceiling on a
# merge-group run: the queue fails the group when this required check has
# not concluded by then, so a job ceiling above it is unreachable there. The
# race step's -timeout lifts go test's 10m per-package default, so a slow
# package runs on to this ceiling instead of failing at ten minutes; it does
# not fit inside the ceiling on the macOS leg, which spends about 18m before
# the slowest package starts, so a hang there is cancelled without a
# goroutine dump. Raising the queue's cap is an admin act on the live
# ruleset, left to the technical facilitator; this job may follow it then.
# TestRaceLaneBudgetIsDeclaredAndFitsItsJob holds this at or below the cap.
timeout-minutes: 30
strategy:
fail-fast: false
Expand Down Expand Up @@ -283,9 +294,21 @@ jobs:

# Only a source change can introduce a data race, and the plain lane above
# has already run the same tests over the record.
#
# -timeout is a declared budget, not go test's 10m per-package default:
# internal/surface/cli under -race on the macOS runner measured 418-568s
# passing, and a slow runner crossed 600s with no test hung, ejecting an
# unrelated pull request from the merge queue (iss-2609260319483365). 20m
# is about twice the slowest passing run, so a real hang still fails with
# a goroutine dump where the enclosing ceiling leaves room for it: in
# release.yml's verify job it does, while this job's 30 minutes, held to
# the merge queue's response cap, are the binding ceiling here (see the
# job header). The Makefile's preflight recipe and release.yml's verify
# job carry the same flag; TestRaceLaneBudgetIsDeclaredAndFitsItsJob
# holds all three.
- name: Test (race, internal)
if: needs.changes.outputs.inert != 'true'
run: go test -race ./internal/...
run: go test -race -timeout 20m ./internal/...

# Drift gate for the .abcd/development design record. Blocking: a
# blocker-severity finding exits non-zero and fails the job; warn-level
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,10 @@ jobs:
# never the default-branch tip or the re-resolvable tag name — so the gate
# exercises exactly the commit whose binaries will ship.
verify:
timeout-minutes: 15
# The race step's own -timeout (20m, sized in ci.yml) must fire before this
# does, or a hang reads as a cancelled job with no goroutine dump: 20m for
# the slowest package plus about 10m this job spends before it starts.
timeout-minutes: 35
runs-on: ubuntu-latest
permissions:
contents: read
Expand Down Expand Up @@ -136,7 +139,7 @@ jobs:
run: go test ./...

- name: Test (race, internal)
run: go test -race ./internal/...
run: go test -race -timeout 20m ./internal/...

- name: Record-lint (design-record drift gate)
run: go run ./cmd/record-lint
Expand Down
3 changes: 2 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,8 @@ irreversible; guessing downward costs nothing.**
`lint-decisions`, `record-lint`, `issue-drift`, `docs-lint`, `site-render`),
both tagged eval
lanes (`smoke`, `evals-cold-reading`), plus `go build ./...`,
`go vet ./...`, `go test ./...`, and `go test -race ./internal/...`. The load
`go vet ./...`, `go test ./...`, and
`go test -race -timeout 20m ./internal/...`. The load
check runs first (`load-check`, a warning, never a failure) and is not a gate:
it exits 0 whatever it finds. The eval
lanes are named separately because their files carry a build tag, so
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ preflight: load-check lint-reviews lint-issues lint-decisions record-lint issue-
go build ./...
go vet ./...
go test ./...
go test -race ./internal/...
go test -race -timeout 20m ./internal/...
@scripts/preflight-receipt.sh mint "$(PREFLIGHT_BEGAN)"

# The push receipt (iss-2608290810036869, iss-2608210738378295). The pre-push hook
Expand Down
9 changes: 8 additions & 1 deletion internal/core/launch/scaffold/templates/release.yml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,14 @@ jobs:
<%- end %>
<%- end %>
verify:
<%- if .Abcd %>
# The race step's own -timeout (20m, sized in ci.yml) must fire before this
# does, or a hang reads as a cancelled job with no goroutine dump: 20m for
# the slowest package plus about 10m this job spends before it starts.
timeout-minutes: 35
<%- else %>
timeout-minutes: 15
<%- end %>
runs-on: ubuntu-latest
permissions:
contents: read
Expand Down Expand Up @@ -193,7 +200,7 @@ jobs:
<%- if .Abcd %>

- name: Test (race, internal)
run: go test -race ./internal/...
run: go test -race -timeout 20m ./internal/...
<%- else %>

# A generic module may have no internal tree, so the race leg runs over the
Expand Down
176 changes: 176 additions & 0 deletions internal/core/lint/racelanebudget_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
package lint_test

import (
"encoding/json"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"
"time"
)

// The race lane's time budget is declared, not inherited (iss-2609260319483365).
//
// `go test` kills a test binary that runs past -timeout, and with no flag the
// ceiling is ten minutes per package. internal/surface/cli under -race on the
// macOS runner measured 417.9s, 472.0s and 567.9s on three passing runs, and
// a merge-group run on a slow runner crossed 600s with no test hung and ejected
// an unrelated pull request from the queue. The lane's ceiling was the default
// rather than a budget anyone had sized, so it is written down here, in three
// places that must agree: the merge gate (ci.yml), the release gate
// (release.yml's verify job) and the local pre-push gate (make preflight).
//
// The enclosing ceilings are the other half, and they differ by job.
//
// ci.yml's check job is a required context, and on a merge-group run the merge
// queue fails the group when a required check has not concluded within the
// ruleset's check_response_timeout_minutes. That cap is the outer ceiling on
// the event the lane was ejected on, and a job ceiling above it is unreachable
// there: the queue gives up first, with no job failure to read. So the check
// job's timeout-minutes may not exceed the cap the ruleset mirror records. The
// step's -timeout still earns its place under that cap: it lifts the ten-minute
// per-package default, so a slow package fails on the job's clock rather than
// at ten minutes. Raising the queue's cap is an admin act on the live ruleset,
// left to the technical facilitator; this test reads the mirror, so the day the
// cap is raised the check job may follow it.
//
// release.yml's verify job is not a merge-queue job, so its own timeout-minutes
// is its ceiling. A package timeout that reaches past it never fires: the
// runner cancels the job first, and a cancellation carries no goroutine dump,
// so a real hang would read as a slow runner. verify therefore holds the
// package timeout plus the time its slowest package waits before it starts:
// the steps ahead of the race step, and the packages `go test` runs ahead of
// it inside the step.
func TestRaceLaneBudgetIsDeclaredAndFitsItsJob(t *testing.T) {
root := filepath.Join("..", "..", "..")

recipe, ok := makeRecipe(readRepoFile(t, root, "Makefile"), "preflight")
if !ok {
t.Fatal("Makefile declares no `preflight:` recipe")
}
local := raceTimeout(t, "Makefile preflight", recipe)

// raceJob returns a workflow job's block and the -timeout its race step
// carries, holding that timeout to the local gate's.
raceJob := func(file, job string) (string, time.Duration, bool) {
where := file + " job " + job
block, ok := workflowJobBlock(readRepoFile(t, root, file), job)
if !ok {
t.Errorf("%s: no such job; the parser or the workflow changed shape", where)
return "", 0, false
}
step, ok := workflowStepBlock(block, "Test (race, internal)")
if !ok {
t.Errorf("%s: no `Test (race, internal)` step", where)
return "", 0, false
}
pkg := raceTimeout(t, where, step)
if pkg != local {
t.Errorf("%s runs the race lane under -timeout %s but make preflight uses %s; "+
"local and CI must judge the lane against one budget", where, pkg, local)
}
return block, pkg, true
}

if check, _, ok := raceJob(".github/workflows/ci.yml", "check"); ok {
const where = ".github/workflows/ci.yml job check"
capMin := jobTimeoutMinutes(t, where, check)
queue := mergeQueueResponseTimeout(t, root)
if capMin > queue {
t.Errorf("%s: timeout-minutes is %s, above the merge queue's %s check response "+
"timeout in %s; on a merge-group run the queue fails the group first, so the "+
"job ceiling is unreachable there and any budget sized against it is false. "+
"Raise the live ruleset's cap (an admin act) before the job's",
where, capMin, queue, rulesetMirror)
}
}

// The last release (run 35963282477, ubuntu, uncached toolchain): 2.8
// minutes before the race step, 4.8 inside it before internal/surface/cli
// started, 0.5 after it; the ubuntu race step has grown by about two
// minutes since. Rounded up to 10.
const verifyHeadroom = 10 * time.Minute
if verify, pkg, ok := raceJob(".github/workflows/release.yml", "verify"); ok {
const where = ".github/workflows/release.yml job verify"
capMin := jobTimeoutMinutes(t, where, verify)
if need := pkg + verifyHeadroom; capMin < need {
t.Errorf("%s: timeout-minutes is %s, below the package timeout %s plus %s of headroom (%s); "+
"the runner would cancel the job before go test could report a hang",
where, capMin, pkg, verifyHeadroom, need)
}
}
}

// rulesetMirror is the tree's record of the live branch ruleset on main.
const rulesetMirror = ".abcd/work/rulesets/main-protection.json"

// mergeQueueResponseTimeout reads the merge_queue rule's
// check_response_timeout_minutes from the ruleset mirror, failing the test
// when the mirror declares no merge queue or no positive cap.
func mergeQueueResponseTimeout(t *testing.T, root string) time.Duration {
t.Helper()
var ruleset struct {
Rules []struct {
Type string `json:"type"`
Parameters struct {
CheckResponseTimeoutMinutes int `json:"check_response_timeout_minutes"`
} `json:"parameters"`
} `json:"rules"`
}
if err := json.Unmarshal([]byte(readRepoFile(t, root, rulesetMirror)), &ruleset); err != nil {
t.Fatalf("decoding %s: %v", rulesetMirror, err)
}
for _, r := range ruleset.Rules {
if r.Type != "merge_queue" {
continue
}
if r.Parameters.CheckResponseTimeoutMinutes <= 0 {
t.Fatalf("%s: the merge_queue rule declares no positive check_response_timeout_minutes", rulesetMirror)
}
return time.Duration(r.Parameters.CheckResponseTimeoutMinutes) * time.Minute
}
t.Fatalf("%s declares no merge_queue rule; the check job's ceiling is sized against it", rulesetMirror)
return 0
}

// raceTimeout returns the -timeout the one `go test -race` command in text
// carries, failing the test when there is none.
func raceTimeout(t *testing.T, where, text string) time.Duration {
t.Helper()
var cmds []string
for _, l := range strings.Split(text, "\n") {
l = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(l), "run:"))
if strings.HasPrefix(l, "go test -race") {
cmds = append(cmds, l)
}
}
if len(cmds) != 1 {
t.Fatalf("%s: want one `go test -race` command, found %d", where, len(cmds))
}
m := regexp.MustCompile(`\s-timeout[ =](\S+)`).FindStringSubmatch(cmds[0])
if m == nil {
t.Fatalf("%s: `%s` sets no -timeout, so the lane inherits go test's 10m default "+
"per package, which internal/surface/cli under -race has already crossed", where, cmds[0])
}
d, err := time.ParseDuration(m[1])
if err != nil {
t.Fatalf("%s: -timeout %q: %v", where, m[1], err)
}
return d
}

// jobTimeoutMinutes reads the job-level `timeout-minutes:` (four-space indent,
// directly under the job key), failing the test when the job declares none.
func jobTimeoutMinutes(t *testing.T, where, job string) time.Duration {
t.Helper()
m := regexp.MustCompile(`(?m)^ timeout-minutes: (\d+)\s*$`).FindStringSubmatch(job)
if m == nil {
t.Fatalf("%s declares no job-level timeout-minutes", where)
}
n, err := strconv.Atoi(m[1])
if err != nil {
t.Fatalf("%s: timeout-minutes %q: %v", where, m[1], err)
}
return time.Duration(n) * time.Minute
}
Loading