From 2fe4cd7795617f88cbcb0f812e3886743c508966 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 12:03:03 -0700 Subject: [PATCH 01/18] sweep dw3-authoritative-rearm-outcome: DW-40 via bmad-loop --- CHANGELOG.md | 2 + docs/FEATURES.md | 7 +- src/bmad_loop/cli.py | 38 ++++----- src/bmad_loop/runs.py | 49 ++++++++++- src/bmad_loop/tui/app.py | 33 ++++---- tests/test_cli.py | 148 +++++++++++++++++++++++++--------- tests/test_engine_worktree.py | 2 +- tests/test_resolve.py | 78 +++++++++++++++--- tests/test_runs.py | 30 +++++-- tests/test_tui_app.py | 107 ++++++++++++++++++++---- 10 files changed, 385 insertions(+), 109 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 223a7c1a..aae706db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -263,6 +263,8 @@ breaking changes may land in a minor release. unrecoverably, on a gitignored ledger. The mark and append legs now re-anchor only when the preimage they wrote over is still the bytes the run last claimed; the restore skips and journals `ledger-restore-skipped-diverged`. +- Make successful escalation re-arms return authoritative ordered notices and a resume-hold + verdict, so a corrupt journal cannot hide a persisted hold from the CLI or TUI gesture. - **An accepted spec reached through a link out of the unit worktree no longer counts as delivered.** The pre-dispatch check asked only whether a file existed at the mounted path, and that probe follows symlinks, so a spec directory the checkout carries as a link diff --git a/docs/FEATURES.md b/docs/FEATURES.md index ea18aeaf..e07fbd4b 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -162,7 +162,12 @@ Result` section. Every other spec keeps warn-and-continue, and the record says w All of these warnings reach the TUI's re-arm as well as `resolve`'s — both route every kind through one shared table, so neither surface can silently learn a kind the other drops, though each still owns where it calls the echo from and the TUI drops the trailing "before - resuming" advice, since it otherwise resumes in the same gesture. Each re-arm also bumps a per-task + resuming" advice, since it otherwise resumes in the same gesture. A successful re-arm returns + those rendered notices and its hold verdict as one immutable authoritative outcome, captured + only after each journal append succeeds and in append order. The CLI and TUI consume that + outcome directly, so an unreadable journal cannot erase a successfully appended hold from the + combined re-arm/resume gesture; best-effort journal diffing remains only for diagnostics already + appended by a call that aborts before it can return an outcome. Each re-arm also bumps a per-task **generation**, so the re-minted session id cannot collide with the abandoned attempt's record — ids already on disk keep their exact spelling, since the suffix appears only above generation zero (#705). Sweep migration and triage tasks make the same rollover automatically when an diff --git a/src/bmad_loop/cli.py b/src/bmad_loop/cli.py index 00943c05..f1084369 100644 --- a/src/bmad_loop/cli.py +++ b/src/bmad_loop/cli.py @@ -2964,7 +2964,7 @@ def _resolve_restore_patch( return str(patch), None -def _echo_rearm_events(run_dir: Path, before: list[dict[str, Any]] | None) -> bool: +def _echo_rearm_events(run_dir: Path, before: list[dict[str, Any]] | None) -> None: """Surface the events a just-completed re-arm journaled: the residue of the restore attempt it abandoned — the `stale-restore-*` records AND `rearm-commits-probe-failed`, all written by `runs._stale_restore_residue` — and the `rearm-*` records the status @@ -2993,33 +2993,30 @@ def _echo_rearm_events(run_dir: Path, before: list[dict[str, Any]] | None) -> bo the whole degrade is journal-only — the invisibility #640(b) exists to end, not to relocate. - Returns True when one of those records HOLDS the resume - (`runs.rearm_holds_the_resume`): the caller re-arms and resumes in a single gesture, - and a record proving the re-drive cannot route has to break that gesture, or its own - "before resuming" imperative is already unactionable the moment it prints. The - question is asked here because this is the one walk over the entries the re-arm - added, and the answer has to survive the `finally` it is computed in.""" + This is abort-only diagnostic recovery: a raised call has no authoritative outcome, + so the journal is the only place to recover records that were appended before the + abort. It deliberately does not infer a resume hold for a call that did not succeed.""" after = runs.journal_entries_or_none(run_dir) if before is None or after is None: # Either end of the diff is unreadable, so there is no trustworthy "new since # the re-arm" window. Skip rather than guess: this runs from a `finally`, and a # raise here would replace the `RearmError` the operator needs, while treating a # failed read as "no entries seen" would replay the whole journal as new. The - # hold degrades with the echo, for the same reason: an unproven hold is a guess, - # and this is what the gesture did before either existed. - return False - holds = False + return for entry in after[len(before) :]: - # asked of every entry, BEFORE the routing table can drop it — a `None` notice - # means "nothing to print here", never "nothing to decide here" - holds = runs.rearm_holds_the_resume(entry) or holds notice = runs.rearm_event_notice(entry) if notice is None: continue severity, message, next_step = notice tail = f"; {next_step}" if next_step else "" print(f"{severity}: {message}{tail}", file=sys.stderr) - return holds + + +def _echo_rearm_notices(notices: tuple[runs.RearmNotice, ...]) -> None: + """Render a successful re-arm's authoritative notices in append order.""" + for notice in notices: + tail = f"; {notice.next_step}" if notice.next_step else "" + print(f"{notice.severity}: {notice.message}{tail}", file=sys.stderr) def cmd_resolve(args: argparse.Namespace) -> int: @@ -3308,9 +3305,9 @@ def cmd_resolve(args: argparse.Namespace) -> int: if (moved := runs.restamp_code_root(run_dir, paths.repo_root)) is not None: print(f"warning: {moved}", file=sys.stderr) before_entries = runs.journal_entries_or_none(run_dir) - hold_resume = False + outcome: runs.RearmOutcome | None = None try: - runs.rearm_escalation( + outcome = runs.rearm_escalation( run_dir, story_key, restore_patch=restore_patch, @@ -3333,7 +3330,10 @@ def cmd_resolve(args: argparse.Namespace) -> int: # `rearm-commits-probe-failed` when it could not), whose whole point is that # nothing else will tell the human. An abort is when that residue matters most: the # re-arm half-ran and the operator has to decide what to do with the tree. - hold_resume = _echo_rearm_events(run_dir, before_entries) + if outcome is None: + _echo_rearm_events(run_dir, before_entries) + assert outcome is not None + _echo_rearm_notices(outcome.notices) print( f"re-armed {story_key}" + (" (restoring the attempted change for review)" if restore_patch else "") @@ -3341,7 +3341,7 @@ def cmd_resolve(args: argparse.Namespace) -> int: if args.resume is False: print(f"resume when ready: bmad-loop resume {args.run_id}") return 0 - if hold_resume: + if outcome.hold_resume: # The re-arm SUCCEEDED — the task is armed and persisted — so this is a 0, and it # stops the GESTURE, not the run. `--resume` does not override it: that flag # skips the confirmation prompt, while the hold is not a question but a proof diff --git a/src/bmad_loop/runs.py b/src/bmad_loop/runs.py index 9d513e3e..c453d793 100644 --- a/src/bmad_loop/runs.py +++ b/src/bmad_loop/runs.py @@ -4029,6 +4029,45 @@ def restamp_code_root(run_dir: Path, repo_root: Path) -> str | None: ) +@dataclass(frozen=True) +class RearmNotice: + """One operator-facing notice produced by a successful re-arm.""" + + severity: Literal["note", "warning"] + message: str + next_step: str + + +@dataclass(frozen=True) +class RearmOutcome: + """Authoritative result of a successfully persisted escalation re-arm.""" + + story_key: str + notices: tuple[RearmNotice, ...] + hold_resume: bool + + +class _RearmJournal(Journal): + """Journal writer that captures successful re-arm notices at append time.""" + + def __init__(self, run_dir: Path): + super().__init__(run_dir) + self.notices: list[RearmNotice] = [] + self.hold_resume = False + + def append(self, kind: str, **fields: Any) -> None: + # Capture only after the durable append succeeds. The synthetic entry contains + # every producer-supplied field the shared classifiers consume; Journal's + # self-minted timestamp/log fields are not part of either contract. + super().append(kind, **fields) + entry = {"kind": kind, **fields} + self.hold_resume = rearm_holds_the_resume(entry) or self.hold_resume + rendered = rearm_event_notice(entry) + if rendered is not None: + severity, message, next_step = rendered + self.notices.append(RearmNotice(severity, message, next_step)) + + def rearm_escalation( run_dir: Path, story_key: str | None = None, @@ -4037,7 +4076,7 @@ def rearm_escalation( isolated_redrive: bool, resolution_recorded: bool, project_root: Path | None = None, -) -> str: +) -> RearmOutcome: """Re-arm an escalation-paused story so the next resume re-drives it. Flips the escalated task out of its terminal ESCALATED phase back to @@ -4150,7 +4189,9 @@ def rearm_escalation( The generation bump stays UNCONDITIONAL beside the gated stamp: it answers session-id reuse (#705), which an abandoned attempt needs exactly as much as a resolved one. - Returns the re-armed story key. Raises RearmError when the run is not paused at + Returns the authoritative re-arm outcome: the story key, the ordered notices + whose journal appends succeeded during this call, and whether one of the appended + records holds the combined re-arm/resume gesture. Raises RearmError when the run is not paused at the escalation stage, the target story is not escalated, or a supplied `restore_patch` fails `validate_restore_latch` (the shared precondition set — sentinel wedge, spec-less escalation, worktree isolation). @@ -4184,7 +4225,7 @@ def rearm_escalation( # names a directory that is no longer there. See the `project_root` note above. live_project = project_root if project_root is not None else Path(state.project) - journal = Journal(run_dir) + journal = _RearmJournal(run_dir) # Read before the unconditional overwrite below: they describe the restore # attempt this re-arm is abandoning, and the residue block needs both. old_latch = task.restore_patch @@ -4858,7 +4899,7 @@ def rearm_escalation( baseline=task.baseline_commit or "", restore=bool(restore_patch), ) - return key + return RearmOutcome(key, tuple(journal.notices), journal.hold_resume) def journal_entries_or_none(run_dir: Path) -> list[dict[str, Any]] | None: diff --git a/src/bmad_loop/tui/app.py b/src/bmad_loop/tui/app.py index 893726e1..767a2ffd 100644 --- a/src/bmad_loop/tui/app.py +++ b/src/bmad_loop/tui/app.py @@ -868,7 +868,7 @@ def _do_replan(self, run_id: str, spec_path: Path, confine_root: Path) -> None: self.notify("plan reset to draft — the next dispatch re-plans") self._do_resume(run_id) - def _echo_rearm_events(self, run_dir: Path, before: list[dict[str, Any]] | None) -> bool: + def _echo_rearm_events(self, run_dir: Path, before: list[dict[str, Any]] | None) -> None: """Toast the re-arm records `cli._echo_rearm_events` prints, same table. Reads through `runs.journal_entries_or_none`, shared with the CLI so the two @@ -880,24 +880,26 @@ def _echo_rearm_events(self, run_dir: Path, before: list[dict[str, Any]] | None) The table's `next_step` is deliberately dropped: it reads "... before resuming", and this path resumes in the same gesture. - Returns True when a record HOLDS that gesture (`runs.rearm_holds_the_resume`), - which is the one case where the dropped imperative was load-bearing rather than - moot — `_do_rearm` stops instead of resuming, and says so in its own words. + This is abort-only diagnostic recovery. A raised call has no authoritative + outcome, so this path must not infer a hold from partial journal residue. """ after = runs.journal_entries_or_none(run_dir) if before is None or after is None: - return False - holds = False + return for entry in after[len(before) :]: - # before the routing table can drop it: a `None` notice means "nothing to - # toast", never "nothing to decide" - holds = runs.rearm_holds_the_resume(entry) or holds notice = runs.rearm_event_notice(entry) if notice is None: continue severity, message, _next_step = notice self.notify(message, severity="warning" if severity == "warning" else "information") - return holds + + def _echo_rearm_notices(self, notices: tuple[runs.RearmNotice, ...]) -> None: + """Toast a successful re-arm's authoritative notices in append order.""" + for notice in notices: + self.notify( + notice.message, + severity="warning" if notice.severity == "warning" else "information", + ) def _do_rearm( self, run_id: str, run_dir: Path, story_key: str, *, restore_recorded: bool = False @@ -966,9 +968,9 @@ def _do_rearm( if (moved := runs.restamp_code_root(run_dir, paths.repo_root)) is not None: self.notify(moved, severity="warning") before_entries = runs.journal_entries_or_none(run_dir) - hold_resume = False + outcome: runs.RearmOutcome | None = None try: - runs.rearm_escalation( + outcome = runs.rearm_escalation( run_dir, story_key, isolated_redrive=isolation == "worktree", @@ -1001,7 +1003,10 @@ def _do_rearm( # path even after they were unified on routing — and an abort is when the # residue matters most: the re-arm half-ran and the operator has to decide # what to do with the tree. - hold_resume = self._echo_rearm_events(run_dir, before_entries) + if outcome is None: + self._echo_rearm_events(run_dir, before_entries) + assert outcome is not None + self._echo_rearm_notices(outcome.notices) if restore_recorded: self.notify( "recorded restore patch NOT honored — this re-arm re-drives from " @@ -1009,7 +1014,7 @@ def _do_rearm( severity="warning", ) self.notify(f"re-armed {story_key}") - if hold_resume: + if outcome.hold_resume: # The half of the gesture that still worked is kept: the story IS re-armed # and persisted. What stops is the resume this surface folds in behind it, # because the warning above proved the re-drive would read a spec it cannot diff --git a/tests/test_cli.py b/tests/test_cli.py index 5055d45c..09b09736 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2675,7 +2675,7 @@ def fake_rearm( project_root=None, ): seen.append(load_state(rd).code_root) - return key + return _rearm_outcome(key) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) @@ -2776,7 +2776,11 @@ def test_resolve_degrades_when_the_config_cannot_name_the_code_root(tmp_path, mo run_dir = _escalated_run(tmp_path, "r1") # no _bmad/bmm/config.yaml anywhere rearmed: list = [] - monkeypatch.setattr(runs, "rearm_escalation", lambda rd, key, **k: rearmed.append(key) or key) + monkeypatch.setattr( + runs, + "rearm_escalation", + lambda rd, key, **k: rearmed.append(key) or _rearm_outcome(key), + ) monkeypatch.setattr(cli, "_resume_paused_run", lambda proj, rd: 0) argv = ["resolve", "--project", str(tmp_path), "r1", "--no-interactive", "--resume"] @@ -2807,10 +2811,29 @@ def fake_rearm( project_root=None, ): journal = Journal(rd) - journal.append("stale-restore-excluded", story_key=key, patch="a.patch", files=["new.txt"]) - journal.append("stale-restore-unparseable", story_key=key, patch="b.patch", error="OSErr") - journal.append("stale-restore-commits", story_key=key, old_baseline="f" * 40, commits=["c"]) - return key + entries = ( + { + "kind": "stale-restore-excluded", + "story_key": key, + "patch": "a.patch", + "files": ["new.txt"], + }, + { + "kind": "stale-restore-unparseable", + "story_key": key, + "patch": "b.patch", + "error": "OSErr", + }, + { + "kind": "stale-restore-commits", + "story_key": key, + "old_baseline": "f" * 40, + "commits": ["c"], + }, + ) + for entry in entries: + journal.append(entry["kind"], **{k: v for k, v in entry.items() if k != "kind"}) + return _rearm_outcome(key, *entries) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) monkeypatch.setattr(cli, "_resume_paused_run", lambda proj, rd: 0) @@ -2819,9 +2842,15 @@ def fake_rearm( ) err = capsys.readouterr().err - assert "excluded the abandoned restore's new files from the re-drive baseline: new.txt" in err - assert "could not read the abandoned restore patch (b.patch)" in err - assert "1 commit(s) sit below the re-drive's new baseline (ffffffffffff..)" in err + ordered_messages = ( + "excluded the abandoned restore's new files from the re-drive baseline: new.txt", + "could not read the abandoned restore patch (b.patch)", + "1 commit(s) sit below the re-drive's new baseline (ffffffffffff..)", + ) + assert all(message in err for message in ordered_messages) + assert [err.index(message) for message in ordered_messages] == sorted( + err.index(message) for message in ordered_messages + ) assert "FROM-LAST-TIME.txt" not in err @@ -2855,22 +2884,24 @@ def fake_rearm( project_root=None, ): journal = Journal(rd) - journal.append( - "rearm-baseline-advance-failed", - story_key=key, - repo=str(tmp_path), - baseline="a" * 40, - error="GitError: not a git repository", - ) - journal.append( - "rearm-baseline-restamped", - story_key=key, - spec_file="spec.md", - overwritten="b" * 40, - baseline="c" * 40, - restore=False, - ) - return key + advance = { + "kind": "rearm-baseline-advance-failed", + "story_key": key, + "repo": str(tmp_path), + "baseline": "a" * 40, + "error": "GitError: not a git repository", + } + restamped = { + "kind": "rearm-baseline-restamped", + "story_key": key, + "spec_file": "spec.md", + "overwritten": "b" * 40, + "baseline": "c" * 40, + "restore": False, + } + for entry in (advance, restamped): + journal.append(entry["kind"], **{k: v for k, v in entry.items() if k != "kind"}) + return _rearm_outcome(key, advance, restamped) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) monkeypatch.setattr(cli, "_resume_paused_run", lambda proj, rd: 0) @@ -2921,7 +2952,7 @@ def fake_rearm( baseline="c" * 40, restore=restore, ) - return key + return _journal_rearm_outcome(rd, key) return fake_rearm @@ -2949,8 +2980,8 @@ def fake_rearm( @pytest.mark.parametrize("outcome", ["ok", "rearm-error"]) def test_resolve_survives_a_corrupt_journal(tmp_path, monkeypatch, capsys, outcome): - """An undecodable byte in journal.jsonl costs the echo, never the gesture — and - never the exit code. + """An undecodable journal cannot suppress an authoritative successful hold, and + cannot replace the original error on an aborted call. The counterpart to `test_escalation_rearm_survives_a_corrupt_journal` in the TUI, which had no CLI twin: the TUI's reads were guarded while `cmd_resolve`'s two were @@ -2977,9 +3008,20 @@ def fake_rearm( ): if outcome == "rearm-error": raise runs.RearmError("cannot re-open story spec /x/spec.md") - return key + return runs.RearmOutcome( + key, + ( + runs.RearmNotice( + "warning", + "authoritative hold from the successful re-arm", + "Commit the corrected spec before resuming", + ), + ), + True, + ) - monkeypatch.setattr(cli, "_resume_paused_run", lambda proj, rd: 0) + resumed: list[str] = [] + monkeypatch.setattr(cli, "_resume_paused_run", lambda proj, rd: resumed.append(rd.name) or 0) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) run_dir = _escalated_run(tmp_path, "r1") (run_dir / JOURNAL_FILE).write_bytes( @@ -2995,6 +3037,8 @@ def fake_rearm( assert "cannot re-open story spec" in err else: assert rc == 0 + assert resumed == [] + assert "authoritative hold from the successful re-arm" in err def test_resolve_echoes_a_skipped_restamp(tmp_path, monkeypatch, capsys): @@ -3025,7 +3069,7 @@ def fake_rearm( spec_file="wt/specs/s1.md", baseline="c" * 40, ) - return key + return _journal_rearm_outcome(rd, key) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) monkeypatch.setattr(cli, "_resume_paused_run", lambda proj, rd: 0) @@ -3169,7 +3213,7 @@ def fake_rearm( project_root=None, ): Journal(rd).append(kind, story_key=key, **fields) - return key + return _journal_rearm_outcome(rd, key) return fake_rearm @@ -3212,6 +3256,19 @@ def fake_rearm( assert "NOT resuming in this gesture" not in out assert resumed == ["r2"] # ...and it still resumes: an advisory is not a proof + _escalated_run(tmp_path, "r3") + monkeypatch.setattr( + runs, + "rearm_escalation", + lambda rd, key, **kwargs: runs.RearmOutcome(key, (), True), + ) + assert ( + cli.main(["resolve", "--project", str(tmp_path), "r3", "--no-interactive", "--resume"]) == 0 + ) + out, _err = capsys.readouterr() + assert "NOT resuming in this gesture" in out + assert resumed == ["r2"] # a hold is authoritative even when it has no notice + def test_resolve_appends_the_next_step_imperative(tmp_path, monkeypatch, capsys): """This surface renders `severity: message; next_step`; the TUI renders `message`. @@ -3254,7 +3311,7 @@ def fake_rearm( journal.append( # table row whose next_step is "" "stale-restore-commits", story_key=key, old_baseline="f" * 40, commits=["c1"] ) - return key + return _journal_rearm_outcome(rd, key) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) monkeypatch.setattr(cli, "_resume_paused_run", lambda proj, rd: 0) @@ -3307,7 +3364,7 @@ def fake_rearm( error=f"GitError: git rev-list {baseline}..HEAD failed in /code: " "not a git repository", ) - return key + return _journal_rearm_outcome(rd, key) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) monkeypatch.setattr(cli, "_resume_paused_run", lambda proj, rd: 0) @@ -3416,7 +3473,7 @@ def fake_context(*args, **kwargs): "run_session", lambda adapter, project, *a, **k: seen.setdefault("cwd", project) or True, ) - monkeypatch.setattr(runs, "rearm_escalation", lambda rd, key, **kwargs: key) + monkeypatch.setattr(runs, "rearm_escalation", lambda rd, key, **kwargs: _rearm_outcome(key)) assert cli.main(["resolve", "--project", str(tmp_path), "r1", "--no-resume"]) == 0 @@ -3448,7 +3505,7 @@ def fake_session(*args, **kwargs): return True monkeypatch.setattr(resolve, "run_session", fake_session) - monkeypatch.setattr(runs, "rearm_escalation", lambda rd, key, **kwargs: key) + monkeypatch.setattr(runs, "rearm_escalation", lambda rd, key, **kwargs: _rearm_outcome(key)) argv = ["resolve", "--project", str(project.project), "r1", "--no-resume"] assert cli.main(argv) == 0 @@ -3695,6 +3752,23 @@ def _escalated_trail_run(tmp_path, run_id="r1", *, details=("first cycle",)): return run_dir +def _rearm_outcome(key: str, *entries: dict) -> runs.RearmOutcome: + notices = tuple( + runs.RearmNotice(*notice) + for entry in entries + if (notice := runs.rearm_event_notice(entry)) is not None + ) + return runs.RearmOutcome( + key, notices, any(runs.rearm_holds_the_resume(entry) for entry in entries) + ) + + +def _journal_rearm_outcome(run_dir: Path, key: str) -> runs.RearmOutcome: + from bmad_loop.journal import Journal + + return _rearm_outcome(key, *Journal(run_dir).entries()) + + def _redrive_escalates(run_dir, detail): """What a re-driven session that escalated again leaves behind, re-escalated so a second `bmad-loop resolve` is legal on it.""" @@ -4448,7 +4522,7 @@ def recording_rearm( project_root=None, ): seen.append(isolated_redrive) - return key + return _rearm_outcome(key) monkeypatch.setattr(cli, "_make_adapters", lambda *a, **k: {"dev": object()}) monkeypatch.setattr(resolve, "build_context", lambda *a, **k: (None, 0, 0)) diff --git a/tests/test_engine_worktree.py b/tests/test_engine_worktree.py index a5347af4..8387f346 100644 --- a/tests/test_engine_worktree.py +++ b/tests/test_engine_worktree.py @@ -6452,7 +6452,7 @@ def commit_fails(*_a, **_k): assert ( runs.rearm_escalation( engine.run_dir, "1-1-a", isolated_redrive=True, resolution_recorded=True - ) + ).story_key == "1-1-a" ) diff --git a/tests/test_resolve.py b/tests/test_resolve.py index 1706b476..c40e31bd 100644 --- a/tests/test_resolve.py +++ b/tests/test_resolve.py @@ -11,7 +11,7 @@ from bmad_loop import devcontract, platform_util, resolve, runs, verify from bmad_loop.engine import _session_task_id -from bmad_loop.journal import TASK_CYCLE_ARTIFACTS, load_state, save_state +from bmad_loop.journal import JOURNAL_FILE, TASK_CYCLE_ARTIFACTS, load_state, save_state from bmad_loop.model import ( PAUSE_ESCALATION, Phase, @@ -833,10 +833,10 @@ def test_rearm_flips_phase_and_spec_status(tmp_path): spec = tmp_path / "spec.md" spec.write_text(SPEC, encoding="utf-8") run_dir, _, _ = _escalated_run(tmp_path, spec_file=str(spec)) - key = runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) - assert key == "6-4-cli-list-command" + outcome = runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) + assert outcome.story_key == "6-4-cli-list-command" state = load_state(run_dir) - task = state.tasks[key] + task = state.tasks[outcome.story_key] assert task.phase == Phase.PENDING assert task.attempt == 0 assert task.review_cycle == 0 @@ -900,7 +900,7 @@ def test_rearm_warns_when_an_isolated_tasks_spec_writes_cannot_reach_the_redrive tmp_path, spec_file=str(spec), worktree_path=str(tmp_path / "wt" / "u1") ) - runs.rearm_escalation(run_dir, isolated_redrive=True, resolution_recorded=True) + outcome = runs.rearm_escalation(run_dir, isolated_redrive=True, resolution_recorded=True) (rec,) = [e for e in _kinds(run_dir) if e["kind"] == "rearm-spec-write-unreachable"] assert rec["story_key"] == "6-4-cli-list-command" @@ -912,6 +912,29 @@ def test_rearm_warns_when_an_isolated_tasks_spec_writes_cannot_reach_the_redrive assert severity == "warning" assert "commit the corrected spec" in message assert next_step + assert runs.RearmNotice(severity, message, next_step) in outcome.notices + assert outcome.hold_resume is True + + +def test_rearm_real_hold_survives_an_unreadable_journal(tmp_path): + """Successful control flow comes from the outcome, never a journal re-read.""" + _resolve_repo(tmp_path) + spec = tmp_path / "spec.md" + spec.write_text(SPEC, encoding="utf-8") + run_dir, _, _ = _escalated_run( + tmp_path, spec_file=str(spec), worktree_path=str(tmp_path / "wt" / "u1") + ) + journal = run_dir / JOURNAL_FILE + journal.write_bytes(b"\xff\xfe pre-existing non-UTF-8 journal\n") + with pytest.raises(UnicodeDecodeError): + journal.read_text(encoding="utf-8") + + outcome = runs.rearm_escalation(run_dir, isolated_redrive=True, resolution_recorded=True) + + assert outcome.hold_resume is True + assert len(outcome.notices) == 1 + assert "land in a tree it discards" in outcome.notices[0].message + assert "Commit the corrected spec" in outcome.notices[0].next_step def test_rearm_completes_on_an_unreachable_spec_it_could_not_capture(tmp_path, monkeypatch): @@ -1534,8 +1557,8 @@ def test_rearm_clears_sentinel_preserving_a_copy(tmp_path): tmp_path, spec_file=str(sentinel), source="stories", sentinel_kind="unresolved" ) - returned = runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) - assert returned == key + outcome = runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) + assert outcome.story_key == key # sentinel deleted from disk, a copy preserved under the run dir assert not sentinel.exists() @@ -1704,7 +1727,7 @@ def test_rearm_rejects_restore_patch_for_a_worktree_executed_task(tmp_path): assert task.restore_patch is None # a from-scratch re-arm of the same task is unaffected — the guard is latch-only assert ( - runs.rearm_escalation(run_dir, isolated_redrive=True, resolution_recorded=True) + runs.rearm_escalation(run_dir, isolated_redrive=True, resolution_recorded=True).story_key == "6-4-cli-list-command" ) @@ -2395,7 +2418,8 @@ def test_rearm_tolerates_non_utf8_sentinel(tmp_path): ) assert ( - runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) == key + runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True).story_key + == key ) # must not raise assert not sentinel.exists() # cleared by deletion assert (run_dir / "sentinels" / f"{key}-unresolved.md").is_file() # copy preserved @@ -4235,12 +4259,14 @@ def test_rearm_holds_a_sentinel_until_the_upstream_correction_reaches_the_redriv ) monkeypatch.chdir(tmp_path) - runs.rearm_escalation(run_dir, isolated_redrive=isolated, resolution_recorded=True) + outcome = runs.rearm_escalation(run_dir, isolated_redrive=isolated, resolution_recorded=True) assert not sentinel.exists() # the sentinel really was cleared on every row records = _upstream_records(run_dir) assert bool(records) is warns + assert outcome.hold_resume is warns if not warns: + assert outcome.notices == () return (rec,) = records # the FOLDER the correction lands in — the main checkout's, not `task_stories_root`'s @@ -4253,6 +4279,28 @@ def test_rearm_holds_a_sentinel_until_the_upstream_correction_reaches_the_redriv assert severity == "warning" assert "SPEC.md" in message and "stories.yaml" in message assert next_step == "Commit the corrected SPEC.md / stories.yaml on `main` before resuming" + assert outcome.notices == (runs.RearmNotice(severity, message, next_step),) + + +def test_rearm_hold_is_independent_of_notice_rendering(tmp_path, monkeypatch): + """A hold record remains authoritative even when it has no renderable notice.""" + run_dir, _, _ = _sentinel_run( + tmp_path, committed_intent=WEDGED_INTENT, working_intent=CORRECTED_INTENT + ) + monkeypatch.chdir(tmp_path) + real_notice = runs.rearm_event_notice + monkeypatch.setattr( + runs, + "rearm_event_notice", + lambda entry: ( + None if entry.get("kind") == "rearm-upstream-write-unreachable" else real_notice(entry) + ), + ) + + outcome = runs.rearm_escalation(run_dir, isolated_redrive=True, resolution_recorded=True) + + assert outcome.hold_resume is True + assert outcome.notices == () @pytest.mark.parametrize( @@ -4401,13 +4449,19 @@ def test_rearm_of_a_sentinel_survives_a_project_that_is_not_a_repository(tmp_pat ) monkeypatch.chdir(tmp_path) - assert ( - runs.rearm_escalation(run_dir, isolated_redrive=True, resolution_recorded=True) == key + outcome = runs.rearm_escalation( + run_dir, isolated_redrive=True, resolution_recorded=True ) # no GitError + assert outcome.story_key == key assert not sentinel.exists() # the destructive half still completed (rec,) = _upstream_records(run_dir) assert runs.rearm_holds_the_resume(rec) is True + assert outcome.hold_resume is True + # The upstream hold is appended before the baseline diagnostics and must remain + # first in the immutable outcome. + assert "sentinel was cleared" in outcome.notices[0].message + assert "could not advance the re-drive baseline" in outcome.notices[1].message def test_rearm_records_the_in_place_remedy_when_isolation_was_turned_off(tmp_path, monkeypatch): diff --git a/tests/test_runs.py b/tests/test_runs.py index 949fbe61..40b04062 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -3252,6 +3252,12 @@ def _kinds(run_dir, prefix="stale-restore-"): return [e for e in Journal(run_dir).entries() if e["kind"].startswith(prefix)] +def _rendered_rearm_notice(entry): + rendered = runs.rearm_event_notice(entry) + assert rendered is not None + return runs.RearmNotice(*rendered) + + def test_rearm_excludes_stale_restore_residue_from_baseline_snapshot(tmp_path): """The abandoned attempt's applied new files must NOT be blessed as pre-existing, or finalize_commit's `add -A` sweeps them into the corrected @@ -3269,7 +3275,7 @@ def test_rearm_excludes_stale_restore_residue_from_baseline_snapshot(tmp_path): """ run_dir, _spec, patch = _stale_restore_tree(tmp_path) - runs.rearm_escalation( + outcome = runs.rearm_escalation( run_dir, isolated_redrive=False, resolution_recorded=True ) # from-scratch re-arm replaces the latch @@ -3281,6 +3287,7 @@ def test_rearm_excludes_stale_restore_residue_from_baseline_snapshot(tmp_path): assert len(excluded) == 1 assert excluded[0]["files"] == ["newfile.txt"] assert excluded[0]["patch"] == str(patch) + assert outcome.notices == (_rendered_rearm_notice(excluded[0]),) # the probe ran and answered "none" — neither commit record may appear assert not _kinds(run_dir, "stale-restore-commits") assert not _kinds(run_dir, "rearm-commits-probe-failed") @@ -3314,7 +3321,7 @@ def test_rearm_missing_stale_patch_degrades_loudly_without_raising(tmp_path): git(tmp_path, "add", "committed.txt") git(tmp_path, "commit", "-q", "-m", "attempt commit") - runs.rearm_escalation( + outcome = runs.rearm_escalation( run_dir, isolated_redrive=False, resolution_recorded=True ) # must not raise RearmError @@ -3325,7 +3332,11 @@ def test_rearm_missing_stale_patch_degrades_loudly_without_raising(tmp_path): assert "FileNotFoundError" in unparseable[0]["error"] assert not _kinds(run_dir, "stale-restore-excluded") # the unreadable patch must not also cost the human the commits warning - assert _kinds(run_dir, "stale-restore-commits") + (commits,) = _kinds(run_dir, "stale-restore-commits") + assert outcome.notices == ( + _rendered_rearm_notice(unparseable[0]), + _rendered_rearm_notice(commits), + ) def test_rearm_without_a_stale_latch_journals_no_stale_restore_events(tmp_path): @@ -3353,7 +3364,7 @@ def test_rearm_warns_about_commits_below_the_refreshed_baseline(tmp_path): git(tmp_path, "commit", "-q", "-m", "attempt commit") old_baseline = load_state(run_dir).tasks["1-1-a"].baseline_commit - runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) + outcome = runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) task = load_state(run_dir).tasks["1-1-a"] assert task.baseline_commit != old_baseline # baseline advanced past the commit @@ -3361,6 +3372,11 @@ def test_rearm_warns_about_commits_below_the_refreshed_baseline(tmp_path): assert len(warned) == 1 assert warned[0]["old_baseline"] == old_baseline assert warned[0]["commits"] == [git(tmp_path, "rev-parse", "HEAD")] + (excluded,) = _kinds(run_dir, "stale-restore-excluded") + assert outcome.notices == ( + _rendered_rearm_notice(excluded), + _rendered_rearm_notice(warned[0]), + ) def test_rearm_survives_a_git_fault_reading_commits_above_the_old_baseline(tmp_path): @@ -3388,7 +3404,7 @@ def test_rearm_survives_a_git_fault_reading_commits_above_the_old_baseline(tmp_p task.baseline_commit = "0" * 39 + "1" # sha-shaped, but names no object save_state(run_dir, state) - runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) + outcome = runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) task = load_state(run_dir).tasks["1-1-a"] assert task.phase == Phase.PENDING @@ -3408,6 +3424,10 @@ def test_rearm_survives_a_git_fault_reading_commits_above_the_old_baseline(tmp_p excluded = _kinds(run_dir, "stale-restore-excluded") assert len(excluded) == 1 assert excluded[0]["files"] == ["newfile.txt"] + assert outcome.notices == ( + _rendered_rearm_notice(excluded[0]), + _rendered_rearm_notice(probe[0]), + ) def test_rearm_survives_a_non_repo_code_tree_when_reading_commits(tmp_path): diff --git a/tests/test_tui_app.py b/tests/test_tui_app.py index 37a4d41a..c8ad2620 100644 --- a/tests/test_tui_app.py +++ b/tests/test_tui_app.py @@ -41,6 +41,7 @@ from bmad_loop import bmadconfig, documents from bmad_loop import policy as policy_mod +from bmad_loop import runs as runs_mod from bmad_loop import verify from bmad_loop.adapters.multiplexer import MultiplexerError from bmad_loop.journal import Journal, save_state @@ -90,6 +91,21 @@ ) +def _rearm_outcome(key: str, *entries: dict) -> runs_mod.RearmOutcome: + notices = tuple( + runs_mod.RearmNotice(*notice) + for entry in entries + if (notice := runs_mod.rearm_event_notice(entry)) is not None + ) + return runs_mod.RearmOutcome( + key, notices, any(runs_mod.rearm_holds_the_resume(entry) for entry in entries) + ) + + +def _journal_rearm_outcome(run_dir: Path, key: str) -> runs_mod.RearmOutcome: + return _rearm_outcome(key, *Journal(run_dir).entries()) + + def make_run( root: Path, run_id: str, @@ -4778,7 +4794,7 @@ async def test_escalation_rearm_resumes_when_resolution_ready(project, monkeypat monkeypatch.setattr( runs, "rearm_escalation", - lambda rd, sk, **_k: rearms.append(sk) or "ready-for-dev", + lambda rd, sk, **_k: rearms.append(sk) or _rearm_outcome(sk), ) run_dir, _spec = _stories_paused_run( project.project, @@ -4892,7 +4908,7 @@ async def test_escalation_rearm_hands_the_rearm_the_live_isolation_mode(project, def fake_rearm(rd, sk, *, isolated_redrive, resolution_recorded, project_root=None): seen.append(isolated_redrive) roots.append(project_root) - return "ready-for-dev" + return _rearm_outcome(sk) monkeypatch.setattr(launch, "mux_available", lambda: True) monkeypatch.setattr(launch, "resume_detached", lambda proj, rid: None) @@ -4956,7 +4972,7 @@ async def test_escalation_rearm_refuses_when_the_policy_cannot_be_read(project, monkeypatch.setattr(launch, "resume_detached", lambda proj, rid: None) monkeypatch.setattr(data, "liveness", lambda run_dir: "dead") monkeypatch.setattr( - runs, "rearm_escalation", lambda rd, sk, **_k: rearms.append(sk) or "ready-for-dev" + runs, "rearm_escalation", lambda rd, sk, **_k: rearms.append(sk) or _rearm_outcome(sk) ) orig_notify = BmadLoopApp.notify monkeypatch.setattr( @@ -5017,7 +5033,7 @@ async def test_escalation_rearm_warns_when_restore_recorded(project, monkeypatch monkeypatch.setattr( runs, "rearm_escalation", - lambda rd, sk, **_k: rearms.append(sk) or "ready-for-dev", + lambda rd, sk, **_k: rearms.append(sk) or _rearm_outcome(sk), ) orig_notify = BmadLoopApp.notify monkeypatch.setattr( @@ -5075,7 +5091,7 @@ def fake_rearm(rd, sk, *, isolated_redrive=False, resolution_recorded=False, pro baseline="a" * 40, error="GitError: not a git repository", ) - return "ready-for-dev" + return _journal_rearm_outcome(rd, sk) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) orig_notify = BmadLoopApp.notify @@ -5129,7 +5145,7 @@ async def test_escalation_rearm_aims_the_code_root_before_it_rearms(project, mon def fake_rearm(rd, sk, *, isolated_redrive=False, resolution_recorded=False, project_root=None): seen.append(load_state(rd).code_root) - return "ready-for-dev" + return _rearm_outcome(sk) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) orig_notify = BmadLoopApp.notify @@ -5321,7 +5337,7 @@ def fake_rearm(rd, sk, *, isolated_redrive=False, resolution_recorded=False, pro error="OSError: [Errno 28] No space left on device", rollback="failed", ) - return "ready-for-dev" + return _journal_rearm_outcome(rd, sk) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) orig_notify = BmadLoopApp.notify @@ -5381,6 +5397,19 @@ def severity_of(fragment: str) -> str: # the CLI's trailing imperative is omitted here: the resume is already queued assert not any("before resuming" in n[0] for n in notes), notes assert any("re-armed 1" in n[0] for n in notes) # the ordinary notice still fires + ordered_messages = ( + "2 commit(s) sit below the re-drive's new baseline", + "excluded the abandoned restore's new files", + "could not list the commits above the abandoned attempt's baseline", + "is not a readable file from here", + "could not be re-opened to `ready-for-dev`", + "may be left part-written", + ) + positions = [ + next(i for i, note in enumerate(notes) if message in note[0]) + for message in ordered_messages + ] + assert positions == sorted(positions) async def test_escalation_rearm_holds_the_resume_it_folds_in(project, monkeypatch): @@ -5424,7 +5453,7 @@ def fake_rearm(rd, sk, *, isolated_redrive=False, resolution_recorded=False, pro spec_file="wt/specs/s1.md", baseline="c" * 40, ) - return "ready-for-dev" + return _journal_rearm_outcome(rd, sk) monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) orig_notify = BmadLoopApp.notify @@ -5457,6 +5486,46 @@ def fake_rearm(rd, sk, *, isolated_redrive=False, resolution_recorded=False, pro assert any("is not a readable file from here" in n for n in notes) +async def test_escalation_rearm_holds_without_a_renderable_notice(project, monkeypatch): + """The authoritative hold is independent of whether there is a toast to render.""" + from bmad_loop import resolve, runs + + calls: list[str] = [] + notes: list[str] = [] + monkeypatch.setattr(launch, "mux_available", lambda: True) + monkeypatch.setattr(launch, "resume_detached", lambda proj, rid: calls.append(rid)) + monkeypatch.setattr(data, "liveness", lambda run_dir: "dead") + monkeypatch.setattr( + runs, + "rearm_escalation", + lambda rd, sk, **kwargs: runs.RearmOutcome(sk, (), True), + ) + orig_notify = BmadLoopApp.notify + monkeypatch.setattr( + BmadLoopApp, + "notify", + lambda self, msg, **kw: notes.append(str(msg)) or orig_notify(self, msg, **kw), + ) + run_dir, _spec = _stories_paused_run( + project.project, + stage="escalation", + spec_status="blocked", + spec_checkpoint=False, + blocked_result="Blocked: needs a human decision on the auth scheme.", + ) + marker = resolve.resolution_path(run_dir, "1") + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("{}", encoding="utf-8") + app = BmadLoopApp(project.project) + async with app.run_test() as pilot: + await _open_review(app, pilot, EscalationModal) + await pilot.click(await ready(pilot, "#act-rearm")) + await until(pilot, lambda: any("not resuming" in note for note in notes)) + + assert calls == [] + assert any("re-armed 1" in note for note in notes) + + async def test_escalation_rearm_echoes_residue_when_the_rearm_aborts(project, monkeypatch): """An aborted re-arm still surfaces what it already journalled — the CLI parity gap. @@ -5519,7 +5588,7 @@ def fake_rearm(rd, sk, *, isolated_redrive=False, resolution_recorded=False, pro async def test_escalation_rearm_survives_a_corrupt_journal(project, monkeypatch): - """An undecodable byte in journal.jsonl costs the echo, never the gesture. + """An undecodable journal cannot suppress a successful authoritative hold. `_do_rearm` reads the journal twice to diff what the re-arm appended, and before that echo existed it read it not at all — so `Journal.entries()`' strict UTF-8 @@ -5536,7 +5605,7 @@ async def test_escalation_rearm_survives_a_corrupt_journal(project, monkeypatch) `re-armed 1` notice ever fires. """ from bmad_loop import resolve, runs - from bmad_loop.journal import JOURNAL_FILE, Journal + from bmad_loop.journal import JOURNAL_FILE calls: list[str] = [] notes: list[str] = [] @@ -5545,10 +5614,15 @@ async def test_escalation_rearm_survives_a_corrupt_journal(project, monkeypatch) monkeypatch.setattr(data, "liveness", lambda run_dir: "dead") def fake_rearm(rd, sk, *, isolated_redrive=False, resolution_recorded=False, project_root=None): - Journal(rd).append( - "stale-restore-commits", story_key=sk, old_baseline="f" * 40, commits=["c1"] + return runs.RearmOutcome( + sk, + ( + runs.RearmNotice( + "warning", "authoritative hold from the successful re-arm", "ignored" + ), + ), + True, ) - return "ready-for-dev" monkeypatch.setattr(runs, "rearm_escalation", fake_rearm) orig_notify = BmadLoopApp.notify @@ -5575,11 +5649,12 @@ def fake_rearm(rd, sk, *, isolated_redrive=False, resolution_recorded=False, pro async with app.run_test() as pilot: await _open_review(app, pilot, EscalationModal) await pilot.click(await ready(pilot, "#act-rearm")) - await until(pilot, lambda: calls == ["20260611-100000-aaaa"]) - # the re-arm ran and the run resumed: the corruption cost only the echo + await until(pilot, lambda: any("not resuming" in n for n in notes)) + # the re-arm ran, its outcome rendered, and the authoritative hold stopped resume assert any("re-armed 1" in n for n in notes) assert not any("re-arm failed" in n for n in notes), notes - assert not any("commit(s) sit below" in n for n in notes), notes + assert any("authoritative hold from the successful re-arm" in n for n in notes), notes + assert calls == [] async def test_escalation_rearm_disabled_without_resolution(project, monkeypatch): From b7814aa7ecaf1723b3bb8d4baff46617406d6079 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 12:34:05 -0700 Subject: [PATCH 02/18] sweep dw3-verify-command-fault-contract: DW-53, DW-54 via bmad-loop --- CHANGELOG.md | 4 +++ src/bmad_loop/engine.py | 12 ++++----- src/bmad_loop/verify.py | 54 ++++++++++++++++++++++++----------------- tests/test_engine.py | 21 ++++++++++++++++ tests/test_verify.py | 52 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 115 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aae706db..383863f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -263,6 +263,10 @@ breaking changes may land in a minor release. unrecoverably, on a gitignored ledger. The mark and append legs now re-anchor only when the preimage they wrote over is still the bytes the run last claimed; the restore skips and journals `ledger-restore-skipped-diverged`. +- **Treat embedded-NUL verify commands and working directories as typed environment + faults** (DW-53, DW-54), while documenting that stream retention degrades but + journal record writes remain fail-loud. + - Make successful escalation re-arms return authoritative ordered notices and a resume-hold verdict, so a corrupt journal cannot hide a persisted hold from the CLI or TUI gesture. - **An accepted spec reached through a link out of the unit worktree no longer counts as diff --git a/src/bmad_loop/engine.py b/src/bmad_loop/engine.py index ac5898e4..1450e0b8 100644 --- a/src/bmad_loop/engine.py +++ b/src/bmad_loop/engine.py @@ -4914,12 +4914,12 @@ def _journal_verify_command_results( still lands, still carrying the full byte count, because "nothing was retained" and "the command was silent" are different facts. - This is observation, so it degrades and never raises (AGENTS.md). An - ``OSError`` from the write — ENOSPC, a read-only run dir, ENAMETOOLONG on - a path this composition did not shorten enough — is journalled as - ``capture_error`` beside a null pointer and the verification continues. - The alternative is a lost log killing a dev pass whose commands passed, - which trades a diagnostic for the run it was there to diagnose. + Stream retention is best-effort observation. An ``OSError`` from the + stream write — ENOSPC, a read-only run dir, ENAMETOOLONG on a path this + composition did not shorten enough — is journalled as ``capture_error`` + beside a null pointer and verification continues. The journal record + itself is durable run state, not degradable capture: ``Journal.append`` + remains unguarded and any failure propagates fail-loud. No results means no records, and therefore no sequence: the ordinal is allocated only when at least one record lands, so it never runs ahead of diff --git a/src/bmad_loop/verify.py b/src/bmad_loop/verify.py index debb5d1f..ac07aac9 100644 --- a/src/bmad_loop/verify.py +++ b/src/bmad_loop/verify.py @@ -4241,12 +4241,13 @@ class CommandResult: code at all: the child was never started. The typical cause is the ``cwd`` it was to run in — missing, not a directory, or unsearchable — and the message names that directory as context, but the fault is caught as any - spawn-time ``OSError`` and the set is not closed: a missing shell, EMFILE - or ENOMEM reach the same field, and the wrapped exception is what says - which. ``None`` on every result that came from a process that actually ran — - including a timeout, which ran and hung. It is LAST and defaulted because the - construction sites pass three to seven POSITIONAL arguments; a field inserted - anywhere else would silently re-bind them. + spawn-time ``OSError`` or ``ValueError`` and the set is not closed: a missing + shell, EMFILE, ENOMEM, or an embedded NUL reach the same field, and the + wrapped exception is what says which. ``None`` on every result that came from + a process that actually ran — including a timeout, which ran and hung. It is + LAST and defaulted because the construction sites pass three to seven + POSITIONAL arguments; a field inserted anywhere else would silently re-bind + them. """ command: str @@ -4505,17 +4506,6 @@ def run_verify_commands(policy: Policy, cwd: Path) -> list[CommandResult]: errors="replace", timeout=COMMAND_TIMEOUT_S, ) - stdout, stdout_full = byte_tail(proc.stdout, MAX_STREAM_MEMORY_BYTES) - stderr, stderr_full = byte_tail(proc.stderr, MAX_STREAM_MEMORY_BYTES) - # merged from the ceilinged streams, not the raw pair: 2000 chars sits - # far below the ceiling, so the tail is identical while the full - # concatenation — a transient copy of both whole streams — is not built. - output = (stdout + stderr)[-2000:] - results.append( - CommandResult( - command, proc.returncode, output, stdout, stderr, stdout_full, stderr_full - ) - ) except subprocess.TimeoutExpired as exc: # the timeout leg is bounded too: a command killed at COMMAND_TIMEOUT_S # is exactly the one that may have been spewing output when it died. @@ -4524,15 +4514,18 @@ def run_verify_commands(policy: Policy, cwd: Path) -> list[CommandResult]: results.append( CommandResult(command, -1, "timed out", t_out, t_err, t_out_full, t_err_full) ) - except OSError as exc: + continue + except (OSError, ValueError) as exc: # The child was never started, so no exit status exists to classify: # `subprocess.run` raises out of the fork/exec (or CreateProcess) # itself when `cwd` is unusable — FileNotFoundError (missing), # NotADirectoryError (a regular file, or a path beneath one), - # PermissionError (a directory without +x). `except OSError` rather - # than the three names because they are the reachable shapes TODAY, - # not a closed set: the base class is what the platform actually - # guarantees, and one uncaught sibling here crashes the whole run. + # PermissionError (a directory without +x) — or raises ValueError + # before spawn when the command or cwd contains an embedded NUL. + # The OSError arm uses the base class rather than the three names + # because they are the reachable OS shapes TODAY, not a closed set: + # the base class is what the platform actually guarantees, and one + # uncaught sibling here crashes the whole run. # # Translated instead of raised, the same doctrine `_run_git` follows # for the faults that land before a return code exists (#343): left @@ -4560,6 +4553,23 @@ def run_verify_commands(policy: Policy, cwd: Path) -> list[CommandResult]: spawn_error=(f"child not started; cwd was {cwd}; {type(exc).__name__}: {exc}"), ) ) + continue + + # Keep result processing outside the spawn-fault handler. A ValueError + # here is a programmer defect, not rejected process configuration, and + # must remain fail-loud rather than being mislabeled as an environment + # fault. + stdout, stdout_full = byte_tail(proc.stdout, MAX_STREAM_MEMORY_BYTES) + stderr, stderr_full = byte_tail(proc.stderr, MAX_STREAM_MEMORY_BYTES) + # merged from the ceilinged streams, not the raw pair: 2000 chars sits + # far below the ceiling, so the tail is identical while the full + # concatenation — a transient copy of both whole streams — is not built. + output = (stdout + stderr)[-2000:] + results.append( + CommandResult( + command, proc.returncode, output, stdout, stderr, stdout_full, stderr_full + ) + ) return results diff --git a/tests/test_engine.py b/tests/test_engine.py index 4ff66c60..5a028f92 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -499,6 +499,27 @@ def test_verify_stream_capture_disabled_writes_no_files_and_still_journals(proje assert entry["output_tail"] == "tail" +def test_verify_result_journal_append_failure_propagates(project, monkeypatch): + """The durable record is fail-loud even though stream retention can degrade. + + Ablation: remove the append or catch its OSError and the expected exception + is not raised. + """ + engine = _capture_engine(project, 0) + + def failing_append(*_args, **_kwargs): + raise OSError("journal.jsonl is not writable") + + monkeypatch.setattr(engine.journal, "append", failing_append) + + with pytest.raises(OSError, match="journal.jsonl is not writable"): + engine._journal_verify_command_results( + StoryTask(story_key="1-1-a", epic=1), + "dev", + (verify.CommandResult("pytest -q", 0, "tail"),), + ) + + def test_verify_stream_capture_oserror_degrades_instead_of_killing_the_run(project, monkeypatch): """A failed retain is an observation loss, never a lost run (AGENTS.md). diff --git a/tests/test_verify.py b/tests/test_verify.py index f6d0ac92..8f0b989a 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -2229,6 +2229,58 @@ def test_unusable_cwd_yields_one_result_per_command(tmp_path): assert "second-check" not in outcome.reason and "third-check" not in outcome.reason +def test_embedded_nul_command_is_an_environment_fault_and_later_command_runs(tmp_path): + """A pre-spawn ValueError is typed without shortening the result list. + + The valid second command proves the loop continues after rejecting only the + first command. Ablation: remove ``ValueError`` from the spawn handler and the + raw exception escapes before the second command runs. + """ + invalid = f"{_OK}\x00ignored" + policy = Policy(verify=VerifyPolicy(commands=(invalid, _OK))) + + results = verify.run_verify_commands(policy, tmp_path) + + assert [result.command for result in results] == [invalid, _OK] + rejected, completed = results + assert rejected.returncode == verify.SPAWN_FAULT_RC + assert rejected.spawn_error is not None and "ValueError" in rejected.spawn_error + assert "ValueError" in rejected.output_tail + assert completed.returncode == 0 and completed.spawn_error is None + outcome = verify.verify_command_results_outcome(results, tmp_path) + assert not outcome.ok and outcome.env_fault + assert not outcome.retryable and not outcome.fixable + + +def test_embedded_nul_cwd_yields_one_spawn_fault_per_command(tmp_path): + """An invalid cwd rejects every spawn but still yields one typed result each.""" + cwd = Path(f"{tmp_path}\x00invalid") + commands = (_OK, _OK) + policy = Policy(verify=VerifyPolicy(commands=commands)) + + results = verify.run_verify_commands(policy, cwd) + + assert [result.command for result in results] == list(commands) + assert all(result.returncode == verify.SPAWN_FAULT_RC for result in results) + assert all(result.spawn_error and "ValueError" in result.spawn_error for result in results) + outcome = verify.verify_command_results_outcome(results, cwd) + assert not outcome.ok and outcome.env_fault + assert not outcome.retryable and not outcome.fixable + + +def test_value_error_after_process_creation_remains_fail_loud(tmp_path, monkeypatch): + """Only subprocess creation ValueErrors belong to the spawn-fault taxonomy.""" + + def broken_result_processing(_text, _max_bytes): + raise ValueError("result processing defect") + + monkeypatch.setattr(verify, "byte_tail", broken_result_processing) + policy = Policy(verify=VerifyPolicy(commands=(_OK,))) + + with pytest.raises(ValueError, match="result processing defect"): + verify.run_verify_commands(policy, tmp_path) + + def test_a_spawn_fault_unrelated_to_the_cwd_translates_too(tmp_path, monkeypatch): """The handler is `except OSError`, not three named cwd classes — and the record must not describe every one of them as a directory problem. From 049558908e801a30f48bf723df8b7cbc0a77a7c2 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 13:16:51 -0700 Subject: [PATCH 03/18] sweep dw3-root-divergence-fixture-hardening: DW-56, DW-57, DW-58, DW-59, DW-60, DW-61, DW-62, DW-63 via bmad-loop --- docs/testing.md | 29 +++++++++ src/bmad_loop/verify.py | 20 +++--- tests/conftest.py | 72 ++++++++++++++++------ tests/test_conftest.py | 113 ++++++++++++++++++++++++++++++++++ tests/test_engine.py | 45 ++++++++++---- tests/test_engine_worktree.py | 65 ++++++++++++++++++- tests/test_hook_bus.py | 7 ++- tests/test_verify.py | 26 ++++++++ 8 files changed, 336 insertions(+), 41 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index e116298b..511828ed 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -115,6 +115,35 @@ test: review sessions do and the orchestrator re-verifies; the bundle twins write none, because bundles have no sprint-status entry.) +Divergent-root tests choose a topology for the distinction they need to prove: + +- **Default** (`project`) keeps `project == repo_root`. Use it for ordinary sandbox behavior, + never for an assertion that claims to distinguish those roots. +- **Sibling** places `repo_root` beside or otherwise outside the BMAD project. Lower-level seam + rows may hand-build this shape (`test_verify.py::_repo_root_override`); caller/config coverage + writes it with `write_repo_root_override` and reloads it through `load_paths`. Use it to prove + that code commands run in the configured root while artifact reads stay in the project. + Artifact-derived excludes correctly collapse to `()` from the disjoint code tree; a non-empty + project-relative spelling may still match nothing there, so this shape cannot grade pathspec + selection by outcome alone. +- **Nested monorepo** (`nested_repo_root_paths`) puts the BMAD project at `/app`, writes + and commits `_bmad/bmm/config.yaml`, and returns `load_paths(app)`. Use it when both right- + and wrong-root pathspecs must be non-empty and separable: the correct value carries `app/`, + while the wrong value can select a plausible outer-tree decoy. The helper accepts alias + spellings but returns canonical paths and commits every seed/config file, so fixture residue + cannot masquerade as session work. +- **Isolated worktree** adds a third live root. Use a real relative command against a marker + created only in the mounted unit worktree; assert the main checkout lacks it, the command + record passed, and classification received the same mounted cwd. Keep the marker under a + gitignored generated-state path so it cannot merge back as proof of work. + +Every divergent-root row guards its premise before its outcome: compare resolved roots, assert +the nested parent relation when nesting matters, and for cwd tests plant/probe both directions +(`plant_root_markers`, `REPO_ROOT_MARKER_CMD`, `PROJECT_MARKER_CMD`). A positive marker identifies +the intended root; the opposite-root marker rules out the tempting alternative. When a wrong +root could still name a real path, create that decoy and assert the selected value rather than +asserting only that some path is absent or a gate refused. + **`MockAdapter` is production code** — `src/bmad_loop/adapters/mock.py`, shipped in the wheel, scripted with a list of `SessionResult`s or `callable(spec) -> SessionResult` effects. It is not reachable from configuration (no `mock` profile exists; `runsetup.make_adapters` builds diff --git a/src/bmad_loop/verify.py b/src/bmad_loop/verify.py index ac07aac9..304fae02 100644 --- a/src/bmad_loop/verify.py +++ b/src/bmad_loop/verify.py @@ -3404,11 +3404,13 @@ def verify_dev_exclude_relpaths( describes. The requirement buys a caller who must think about the root, not a checker that knows the right answer. - A relpath computed against the wrong root does not raise: it simply - matches nothing on git's side, so the exclusion silently disappears and a bare - status flip starts counting as real work. The latched `restore_patch` is - anchored on the SAME root for the same reason (a relative latch names a path - in the tree it will be applied to).""" + The wrong-root symptom depends on topology. With disjoint sibling project and + code roots, a code-root relative artifact path collapses to ``()`` and a + project-root spelling is non-empty but still matches nothing in the code tree. + In a nested monorepo both spellings are non-empty: omitting the project prefix + can select a plausible outer-tree file instead of the nested artifact. The + latched `restore_patch` is anchored on the SAME root for the same reason (a + relative latch names a path in the tree it will be applied to).""" candidates: list[Path] = [paths.sprint_status, spec_path] if restore_patch: candidates.append(resolve_restore_path(restore_patch, root)) @@ -4170,9 +4172,11 @@ def _stories_relpaths(root: Path, spec_folder: Path) -> tuple[str, ...]: Empty when the spec folder is outside that tree (nothing to exclude there). ``root`` is the tree git is invoked against — `paths.repo_root` at the one - production call site, which under the `repo_root` override is NOT - `paths.project` (the spec folder then sits outside the code tree and this - correctly returns ``()``).""" + production call site. Under a disjoint sibling `repo_root` override the spec + folder sits outside the code tree and this correctly returns ``()``. Under a + nested-monorepo override it remains inside that tree and returns non-empty + paths carrying the project prefix; dropping that prefix would instead name a + plausible outer-tree location.""" from .stories import STORIES_FILENAME, STORIES_SUBDIR try: diff --git a/tests/conftest.py b/tests/conftest.py index dbfb7600..590e28af 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,6 @@ from __future__ import annotations -import dataclasses import io import json import shutil @@ -17,7 +16,7 @@ from bmad_loop import cli, documents, envvars, platform_util, runs from bmad_loop.adapters.base import SessionResult, SessionSpec -from bmad_loop.bmadconfig import ProjectPaths +from bmad_loop.bmadconfig import ProjectPaths, load_paths from bmad_loop.checks import ValidationReport from bmad_loop.journal import save_state from bmad_loop.model import PAUSE_ESCALATION, Phase, RunState, SessionRecord, StoryTask @@ -216,6 +215,25 @@ def _file_exists_cmd(path) -> str: return f'test -f "{path}"' +def scripted_verify_runner(expected_root: Path, next_results): + """Return a scripted ``run_verify_commands`` double that pins its cwd. + + ``next_results`` is a zero-argument callable so callers can return one fixed + result list, advance an iterator, or provide an iterator fallback without + this helper changing the scenario's existing script semantics. + """ + expected = expected_root.resolve() + + def run(_policy, cwd: Path): + actual = cwd.resolve() + assert ( + actual == expected + ), f"scripted verify runner called in the wrong root: expected {expected}, got {actual}" + return next_results() + + return run + + def passes_once(marker) -> str: """Return a host-shell command that succeeds once, then fails. @@ -661,8 +679,8 @@ def nested_repo_root_paths(paths: ProjectPaths) -> ProjectPaths: """The MONOREPO shape of the override: `repo_root` an ANCESTOR of `project`. The BMAD project lives at ``/app`` inside a checkout whose root is the - git root — `repo_root` stays `paths.project` (the sandbox repo) while - `project` and all three artifact dirs move under ``app/``. + git root. The helper writes the same config shape production loads, then + returns :func:`load_paths`' canonical snapshot rather than hand-building one. Why a second shape at all. `tests/test_verify.py::_repo_root_override` builds the SIBLING shape, where the artifact tree is disjoint from the code tree — so @@ -681,10 +699,11 @@ def nested_repo_root_paths(paths: ProjectPaths) -> ProjectPaths: for the reason `plant_root_markers` gives: a session's edit to a TRACKED file is proof of work the attempt's baseline snapshot cannot absorb. - Also writes ``app/.gitignore`` with the `bmad-loop init` run-state entry. + Also writes ``app/.gitignore`` with all four entries `bmad-loop init` owns. Init writes that file next to the project it initializes, and the sandbox - template's own root-anchored ``.bmad-loop/runs/`` does not match a nested one — - so without it a nested engine run's journal would show up as untracked work. + template's root-anchored entries do not match nested state — so without the + nested file run state, caches, policy, and renderer output can show up as + untracked work. The subdirectory is FIXED at `NESTED_SUBDIR` rather than a parameter because every consumer's assertions spell the ``app/`` prefix literally. A parameter @@ -697,20 +716,21 @@ def nested_repo_root_paths(paths: ProjectPaths) -> ProjectPaths: naming neither the helper nor the precondition. Both guards below fail with the precondition instead. """ - assert paths.project == paths.repo_root, ( + assert paths.project.resolve() == paths.repo_root.resolve(), ( "nested_repo_root_paths builds the divergence; it cannot be applied to paths " "that already have one. Pass the plain `project` fixture." ) - staged = git(paths.project, "diff", "--cached", "--name-only") + repo_root = paths.project.resolve() + staged = git(repo_root, "diff", "--cached", "--name-only") assert not staged, ( "nested_repo_root_paths commits its seed files and requires an empty index; " f"already staged: {staged}" ) - project = paths.project / NESTED_SUBDIR - assert not project.exists(), ( - f"{NESTED_SUBDIR}/ already exists under {paths.project}: this helper seeds and " - "COMMITS it, so a second call (or a caller that pre-created it) would reach " - "`git commit` with nothing staged." + project = repo_root / NESTED_SUBDIR + assert not project.exists() and not project.is_symlink(), ( + f"{NESTED_SUBDIR}/ already exists or is a symlink under {repo_root}: this helper " + "seeds and COMMITS it, so a second call (or a caller that pre-created it) would " + "reach setup/commit with an opaque filesystem or git error." ) output_folder = project / "_bmad-output" impl = output_folder / "implementation-artifacts" @@ -718,17 +738,29 @@ def nested_repo_root_paths(paths: ProjectPaths) -> ProjectPaths: impl.mkdir(parents=True, exist_ok=True) plan.mkdir(parents=True, exist_ok=True) (project / "src.txt").write_text("original\n", encoding="utf-8") - (project / ".gitignore").write_text(".bmad-loop/runs/\n", encoding="utf-8") - git(paths.project, "add", f"{NESTED_SUBDIR}/src.txt", f"{NESTED_SUBDIR}/.gitignore") - git(paths.project, "commit", "-q", "-m", f"seed the {NESTED_SUBDIR}/ project") - return dataclasses.replace( - paths, + (project / ".gitignore").write_text( + "\n".join( + ( + ".bmad-loop/runs/", + ".bmad-loop/cache/", + ".bmad-loop/policy.toml", + "_bmad/render/", + ) + ) + + "\n", + encoding="utf-8", + ) + configured = ProjectPaths( project=project, implementation_artifacts=impl, planning_artifacts=plan, output_folder=output_folder, - repo_root=paths.project, + repo_root=repo_root, ) + write_repo_root_override(configured, repo_root) + git(repo_root, "add", NESTED_SUBDIR) + git(repo_root, "commit", "-q", "-m", f"seed the {NESTED_SUBDIR}/ project") + return load_paths(project) UNRESOLVABLE = "stubbed: the provider is registered but not serving" diff --git a/tests/test_conftest.py b/tests/test_conftest.py index 447c9a10..747c077c 100644 --- a/tests/test_conftest.py +++ b/tests/test_conftest.py @@ -117,6 +117,100 @@ def test_write_repo_root_override_refuses_a_relative_code_root(project): assert not (project.project / conftest.BMAD_CONFIG_REL).exists() +def test_scripted_verify_runner_refuses_the_wrong_canonical_cwd(tmp_path): + """A cwd-discarding command double cannot hide a caller-root regression. + + Ablation: delete the cwd equality assertion in `scripted_verify_runner` and + this row fails because the wrong-root call no longer raises. + """ + expected = tmp_path / "expected" + wrong = tmp_path / "wrong" + expected.mkdir() + wrong.mkdir() + runner = conftest.scripted_verify_runner(expected, lambda: ["scripted"]) + + with pytest.raises(AssertionError, match="wrong root"): + runner(None, wrong) + + assert runner(None, expected / ".." / expected.name) == ["scripted"] + + +def test_nested_repo_root_paths_round_trips_committed_config_and_conflict(project): + """The nested fixture is a production-loadable config, not a hand-built snapshot. + + Ablation: short-circuit `worktree_isolation_conflict` for the worktree mode + and this row fails because the divergent loaded config is no longer refused. + """ + paths = conftest.nested_repo_root_paths(project) + + assert bmadconfig.load_paths(paths.project) == paths + assert paths.project == paths.project.resolve() + assert paths.repo_root == paths.repo_root.resolve() + assert paths.project.parent == paths.repo_root + config_rel = (paths.project / conftest.BMAD_CONFIG_REL).relative_to(paths.repo_root) + assert conftest.git(paths.repo_root, "ls-files", "--error-unmatch", config_rel.as_posix()) + assert bmadconfig.worktree_isolation_conflict(paths, "none") is None + conflict = bmadconfig.worktree_isolation_conflict(paths, "worktree") + assert conflict is not None and "not supported" in conflict + + +def test_nested_repo_root_paths_canonicalizes_a_dotdot_input(project): + """A portable alias spelling cannot collapse pathspecs through mixed roots.""" + alias = project.project / ".." / project.project.name + assert alias != alias.resolve() + assert alias.resolve() == project.project.resolve() + aliased = replace( + project, + project=alias, + implementation_artifacts=alias / "_bmad-output" / "implementation-artifacts", + planning_artifacts=alias / "_bmad-output" / "planning-artifacts", + output_folder=alias / "_bmad-output", + repo_root=alias, + ) + + paths = conftest.nested_repo_root_paths(aliased) + + assert all( + path == path.resolve() + for path in ( + paths.project, + paths.implementation_artifacts, + paths.planning_artifacts, + paths.output_folder, + paths.repo_root, + ) + ) + assert paths.project.parent == paths.repo_root == project.project.resolve() + spec = paths.implementation_artifacts / "spec-1-1-a.md" + assert verify.verify_dev_exclude_relpaths(paths, spec, root=paths.repo_root) + assert verify._stories_relpaths(paths.repo_root, paths.planning_artifacts / "epic-a") + + +def test_nested_repo_root_paths_seeds_the_complete_init_ignore_shape(project): + """Nested generated state cannot become proof-of-work residue.""" + paths = conftest.nested_repo_root_paths(project) + expected = [ + ".bmad-loop/runs/", + ".bmad-loop/cache/", + ".bmad-loop/policy.toml", + "_bmad/render/", + ] + assert (paths.project / ".gitignore").read_text(encoding="utf-8").splitlines() == expected + + candidates = [ + ".bmad-loop/runs/run/state.json", + ".bmad-loop/cache/plugin/cache.bin", + ".bmad-loop/policy.toml", + "_bmad/render/skill/workflow.md", + ] + for rel in candidates: + path = paths.project / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("generated\n", encoding="utf-8") + ignored = conftest.git(paths.repo_root, "check-ignore", *[f"app/{rel}" for rel in candidates]) + assert ignored.splitlines() == [f"app/{rel}" for rel in candidates] + + def test_nested_repo_root_paths_refuses_a_nonempty_index(project): """Its seed commit must never absorb setup another fixture already staged.""" staged = project.project / "staged.txt" @@ -154,6 +248,25 @@ def test_nested_repo_root_paths_refuses_an_existing_nested_project(project): assert not (nested / ".gitignore").exists() +def test_nested_repo_root_paths_refuses_a_dangling_nested_symlink(project): + """A dangling `app` alias hits the helper precondition before any writes.""" + nested = project.project / conftest.NESTED_SUBDIR + missing = project.project / "missing-app-target" + try: + nested.symlink_to(missing, target_is_directory=True) + except OSError as exc: + pytest.skip(f"directory symlinks unavailable: {exc}") + assert nested.is_symlink() and not nested.exists() + status_before = conftest.git(project.project, "status", "--porcelain") + + with pytest.raises(AssertionError, match="already exists or is a symlink"): + conftest.nested_repo_root_paths(project) + + assert nested.is_symlink() and not nested.exists() + assert not missing.exists() + assert conftest.git(project.project, "status", "--porcelain") == status_before + + def test_template_leaves_no_detached_git_maintenance_writing_into_the_copies(project, tmp_path): """No background git process may outlive a commit into the sandbox. diff --git a/tests/test_engine.py b/tests/test_engine.py index 5a028f92..d0cef3ad 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -34,6 +34,7 @@ plant_root_markers, refuse_to_resolve, review_effect, + scripted_verify_runner, set_sprint, spec_path, write_gated_ledger, @@ -184,7 +185,11 @@ def test_post_dev_verify_exposes_journaled_command_results(project, monkeypatch) capture = _PostDevVerifyCaptureBus() engine._bus = capture result = verify.CommandResult("pytest -q", 0, "out\nerr\n", "out\n", "err\n") - monkeypatch.setattr(verify, "run_verify_commands", lambda policy, cwd: [result]) + monkeypatch.setattr( + verify, + "run_verify_commands", + scripted_verify_runner(project.repo_root, lambda: [result]), + ) summary = engine.run() @@ -238,7 +243,11 @@ def test_review_gate_verify_commands_are_journalled_under_the_review_stage(proje ), ) result = verify.CommandResult("pytest -q", 0, "out\nerr\n", "out\n", "err\n") - monkeypatch.setattr(verify, "run_verify_commands", lambda policy, cwd: [result]) + monkeypatch.setattr( + verify, + "run_verify_commands", + scripted_verify_runner(project.repo_root, lambda: [result]), + ) summary = engine.run() @@ -544,7 +553,10 @@ def test_verify_stream_capture_oserror_degrades_instead_of_killing_the_run(proje monkeypatch.setattr( verify, "run_verify_commands", - lambda policy, cwd: [verify.CommandResult("pytest -q", 0, "tail", "out\n", "err\n")], + scripted_verify_runner( + project.repo_root, + lambda: [verify.CommandResult("pytest -q", 0, "tail", "out\n", "err\n")], + ), ) def _enospc(*_args, **_kwargs): @@ -773,7 +785,11 @@ def _dev_then_fix_run(project, monkeypatch, capture): [verify.CommandResult("pytest -q", 0, "final", "final-out", "")], ] ) - monkeypatch.setattr(verify, "run_verify_commands", lambda policy, cwd: next(calls)) + monkeypatch.setattr( + verify, + "run_verify_commands", + scripted_verify_runner(project.repo_root, lambda: next(calls)), + ) return engine, engine.run() @@ -881,7 +897,11 @@ def test_a_critical_session_emits_post_dev_verify_on_both_legs(project, monkeypa [verify.CommandResult("pytest -q", 0, "fix", "fix-out", "")], ] ) - monkeypatch.setattr(verify, "run_verify_commands", lambda policy, cwd: next(calls)) + monkeypatch.setattr( + verify, + "run_verify_commands", + scripted_verify_runner(project.repo_root, lambda: next(calls)), + ) summary = engine.run() @@ -2438,7 +2458,10 @@ def test_dev_retry_notice_collapses_a_multiline_reason(project, monkeypatch): monkeypatch.setattr( verify, "run_verify_commands", - lambda policy, cwd: next(failing, [verify.CommandResult("pytest -q", 0, "ok")]), + scripted_verify_runner( + project.repo_root, + lambda: next(failing, [verify.CommandResult("pytest -q", 0, "ok")]), + ), ) assert engine.run().done == 1 @@ -2553,11 +2576,11 @@ def resolve_fault(self, *args, **kwargs): # # `Engine._verify_commands_with_results` runs the commands in `self.workspace.root` # — which `Workspace.default` sets to `paths.repo_root`, the CODE tree. Both of its -# stages (`dev` and `fix`) were unpinned: every other engine row mocks -# `verify.run_verify_commands` with a `lambda policy, cwd:` that DISCARDS the cwd, -# so moving the root back to `paths.project` left the whole suite green. These two -# rows therefore run the real commands, and use `conftest.nested_repo_root_paths` -# so the two roots are genuinely different directories. +# stages (`dev` and `fix`) once had only cwd-discarding scripted doubles, so moving +# the root back to `paths.project` left the whole suite green. The shared scripted +# helper now pins the default-root scenarios too; these two rows still run real +# commands under `conftest.nested_repo_root_paths` to grade the divergent behavior +# an operator sees. @pytest.mark.parametrize("marker_root", ["repo_root", "project"]) diff --git a/tests/test_engine_worktree.py b/tests/test_engine_worktree.py index 8387f346..c522dbcd 100644 --- a/tests/test_engine_worktree.py +++ b/tests/test_engine_worktree.py @@ -10,12 +10,14 @@ import shutil import sys +from dataclasses import replace from pathlib import Path import pytest from conftest import ( _OK, _exists_run, + _file_exists_cmd, _seeded_then_touch, _spec_baseline, _touch_run, @@ -45,7 +47,14 @@ ) from bmad_loop.journal import Journal, load_state from bmad_loop.model import Phase, RunState, SessionRecord, StoryTask, TokenUsage -from bmad_loop.policy import GatesPolicy, LimitsPolicy, NotifyPolicy, Policy, ScmPolicy +from bmad_loop.policy import ( + GatesPolicy, + LimitsPolicy, + NotifyPolicy, + Policy, + ScmPolicy, + VerifyPolicy, +) from bmad_loop.verify import ( branch_exists, current_branch, @@ -237,6 +246,60 @@ def test_worktree_happy_path_merges_to_target(project): assert "worktree-teardown-degraded" not in kinds +def test_isolated_verify_commands_execute_and_classify_in_the_unit_worktree(project, monkeypatch): + """A relative dev command is rooted on the live isolated checkout. + + The marker exists only under the mounted unit worktree and is gitignored, so + its successful real command pins execution without merging test residue back + to the main checkout. The classifier spy delegates to production and pins the + second cwd hop independently. + + Ablation: hand `project.repo_root` to either verifier call in + `Engine._verify_commands_with_results`; execution then records rc 1, while a + classifier-only change leaves the command green but reddens the cwd assertion. + """ + commit_sprint(project, {"1-1-a": "ready-for-dev"}) + marker = Path(".bmad-loop") / "runs" / "unit-only-verify.marker" + assert not (project.repo_root / marker).exists() + mounted: dict[str, Path] = {} + base_effect = wt_dev_effect(project, "1-1-a", followup_review=False) + + def dev_with_marker(spec): + mounted["root"] = spec.cwd.resolve() + marker_path = spec.cwd / marker + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text("unit only\n", encoding="utf-8") + return base_effect(spec) + + classified: list[Path] = [] + real_classify = verify.verify_command_results_outcome + + def spy_classify(results, cwd): + classified.append(cwd.resolve()) + return real_classify(results, cwd) + + monkeypatch.setattr(verify, "verify_command_results_outcome", spy_classify) + policy = replace( + wt_policy(), + verify=VerifyPolicy(commands=(_file_exists_cmd(marker.as_posix()),)), + ) + engine, _ = make_engine(project, [dev_with_marker], policy=policy) + + summary = engine.run() + + assert summary.done == 1 and not summary.paused + unit_root = mounted["root"] + assert unit_root != project.repo_root.resolve() + assert not (project.repo_root / marker).exists() + (dev_record,) = [ + entry + for entry in engine.journal.entries() + if entry["kind"] == "verify-command-result" and entry["verification_stage"] == "dev" + ] + assert dev_record["returncode"] == 0 + assert classified and all(cwd == unit_root for cwd in classified) + + def test_local_absolute_ignored_accepted_spec_is_seeded_and_bound_in_mount(project): """Relativizing an accepted spec also delivers it to a tracked-only checkout.""" rel = "_bmad-output/implementation-artifacts/accepted-untracked.md" diff --git a/tests/test_hook_bus.py b/tests/test_hook_bus.py index 2440940d..4a976967 100644 --- a/tests/test_hook_bus.py +++ b/tests/test_hook_bus.py @@ -23,6 +23,7 @@ dev_effect, needs_strict_codec, review_effect, + scripted_verify_runner, write_sprint, ) @@ -446,7 +447,11 @@ def on_post_dev_verify(self, c): seen.append((c.verification_stage, c.verification_sequence, c.command_results)) result = verify.CommandResult("pytest -q", 0, "tail", "out", "err") - monkeypatch.setattr(verify, "run_verify_commands", lambda policy, cwd: [result]) + monkeypatch.setattr( + verify, + "run_verify_commands", + scripted_verify_runner(project.repo_root, lambda: [result]), + ) engine, _ = make_engine(project, one_story(project), registry_of(py_plugin(P, "verifyobs"))) summary = engine.run() diff --git a/tests/test_verify.py b/tests/test_verify.py index 8f0b989a..d6945016 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -6300,6 +6300,32 @@ def test_verify_dev_exclude_relpaths_separates_the_two_roots_in_a_monorepo(proje assert (paths.repo_root / from_project[0]) != paths.sprint_status +def test_verify_dev_exclude_relpaths_roots_restore_patch_on_the_outer_repo(project): + """A relative restore latch selects the code-root file, not a nested decoy. + + Both candidates exist so the assertion grades root selection by value; an + absence-only assertion would pass if setup simply forgot to create the decoy. + + Ablation: resolve `restore_patch` against `paths.project` instead of `root` + and the final equality selects `app/restore.patch`, reddening this row. + """ + paths = nested_repo_root_paths(project) + assert paths.project.parent == paths.repo_root + outer = paths.repo_root / "restore.patch" + decoy = paths.project / "restore.patch" + outer.write_text("outer\n", encoding="utf-8") + decoy.write_text("nested decoy\n", encoding="utf-8") + assert outer.is_file() and decoy.is_file() and outer.resolve() != decoy.resolve() + sp = paths.implementation_artifacts / "spec-1-1-a.md" + + relpaths = verify.verify_dev_exclude_relpaths(paths, sp, "restore.patch", root=paths.repo_root) + + restore_rel = relpaths[-1] + assert restore_rel == "restore.patch" + assert (paths.repo_root / restore_rel).resolve() == outer.resolve() + assert (paths.repo_root / restore_rel).resolve() != decoy.resolve() + + def test_stories_relpaths_separates_the_two_roots_in_a_monorepo(project): """Same rule, same shape, for the stories-mode exclude. From 44b2da9a6906976959f561f13bc6608a9f8e89b1 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 13:29:04 -0700 Subject: [PATCH 04/18] sweep dw3-session-artifact-contract-docs: DW-67, DW-75 via bmad-loop --- docs/FEATURES.md | 2 +- docs/adapter-authoring-guide.md | 12 +++++++++++- tests/test_portability_guard.py | 27 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/FEATURES.md b/docs/FEATURES.md index e07fbd4b..51ccfe83 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -212,7 +212,7 @@ Result` section. Every other spec keeps warn-and-continue, and the record says w - Every run is a resumable on-disk state machine: `bmad-loop resume ` continues from a gate, escalation, or interruption. - A graceful stop (`stop --graceful` / TUI `S`) is resumable too: unlike a hard stop killed mid-item, it lets the in-flight item finish through commit and finalizes cleanly, ending as a `stopped` run that `resume` picks up at the next item. -- All run state in `.bmad-loop/runs//` (gitignored): `state.json` (which records `repo_root`, the git root code work happens in, so an out-of-process re-arm reads back the tree the run measured, #716 — resume re-stamps it from the config.yaml it just re-read, since that is the tree it arms the engine against, and warns when the root moved: the baselines, preserve refs and branches already recorded name objects in the previous one. `resolve` and the TUI's re-arm re-stamp it themselves, before they re-arm: both re-arm and then resume in one gesture, so resume's own re-stamp lands after the re-arm has already advanced the baseline in whichever tree the mirror still named); `journal.jsonl` (every decision, including the `session-synthesized-from-frontmatter` catch and its `spec-marker-repaired` repair, #276, plus one `verify-command-result` per verifier command — emitted on the dev, repair and review legs alike, carrying `verification_stage` and a per-story `verification_sequence` that orders the passes across all three; the two passes that leave no record are `bmad-loop confirm --reverify`, which runs after the run is over, and any pass with no `[verify] commands` configured, which records nothing because nothing ran — each entry also carrying `spawn_error`, set when the verify command's child could not be started at all — typically because its working directory is missing, is not a directory, or cannot be searched, though any spawn-time `OSError` (a missing shell, EMFILE, ENOMEM) reaches the same field and the wrapped exception is what names the cause — which is an environment fault that pauses the run rather than a command that failed — whose stream pointers name the `verify/` directory below, and one `park-proof-of-work-skipped` per attempt that cleared the dev artifact gate on an `awaiting-operator` park with proof-of-work waived — not per park that committed, since the stages after that gate can still reject the attempt — carrying `zero_diff`: `true` when the waived gate found no non-excluded changes (its exclusions include the spec, board, any restore-patch artifact, and an orchestrator-authored deferred-work ledger append), `false` when it found changes, and `null` when the probe could not answer (a git fault, a git refusal such as an unresolvable baseline, or an attempt with no recorded baseline to measure from) and the gate was waived anyway, so the waiver itself is recorded whatever the probe managed to say. `false` is a statement about the tree, not about who wrote what: the gate this stands in for cannot attribute residue to a session in a shared checkout, and the record inherits that limit rather than improving on it); `tasks//` (per-session prompt + `result.json` + breadcrumbs — `session-lifecycle.jsonl` records timeout fires, budget-guard trips (`budget-tripped` / `over-budget-fired`), transport-failure classification (`env-fault-classified`, #194), a mux session lost under the run (`session-vanished`, #489) and the #276 forensics (`spec-status-transition-observed`, `frontmatter-unmodified-refused`, `contract-nudge-sent`); `heartbeat.json` is the wait loop's proof-of-life; `resultless-stops.jsonl` records give-up Stops with a verdict — `no-artifact`, `ambiguous-frontmatter`, `unmodified-since-launch`, `terminal-frontmatter-pending`); `logs/`; `verify/` (verifier command stdout/stderr, one file per stream, pointed at by the journal's `verify-command-result` records — its own store, because every name in `logs/` is a session task id the TUI resolves as a pane log; each stream is retained tail-first up to `[verify] stream_capture_kb` (256 KiB, `0` = keep nothing) and the record carries the full byte count, a truncation flag, and a `capture_error` when the write itself failed); `deferred/`; `resolve/`; `ATTENTION`; `ctl-window` (the control-session window id the last TUI launch minted, so attach/stop follow the live window, #482). +- All run state in `.bmad-loop/runs//` (gitignored): `state.json` (which records `repo_root`, the git root code work happens in, so an out-of-process re-arm reads back the tree the run measured, #716 — resume re-stamps it from the config.yaml it just re-read, since that is the tree it arms the engine against, and warns when the root moved: the baselines, preserve refs and branches already recorded name objects in the previous one. `resolve` and the TUI's re-arm re-stamp it themselves, before they re-arm: both re-arm and then resume in one gesture, so resume's own re-stamp lands after the re-arm has already advanced the baseline in whichever tree the mirror still named); `journal.jsonl` (every decision, including the `session-synthesized-from-frontmatter` catch and its `spec-marker-repaired` repair, #276, plus one `verify-command-result` per verifier command — emitted on the dev, repair and review legs alike, carrying `verification_stage` and a per-story `verification_sequence` that orders the passes across all three; the two passes that leave no record are `bmad-loop confirm --reverify`, which runs after the run is over, and any pass with no `[verify] commands` configured, which records nothing because nothing ran — each entry also carrying `spawn_error`, set when the verify command's child could not be started at all — typically because its working directory is missing, is not a directory, or cannot be searched, though any spawn-time `OSError` (a missing shell, EMFILE, ENOMEM) reaches the same field and the wrapped exception is what names the cause — which is an environment fault that pauses the run rather than a command that failed — whose stream pointers name the `verify/` directory below, and one `park-proof-of-work-skipped` per attempt that cleared the dev artifact gate on an `awaiting-operator` park with proof-of-work waived — not per park that committed, since the stages after that gate can still reject the attempt — carrying `zero_diff`: `true` when the waived gate found no non-excluded changes (its exclusions include the spec, board, any restore-patch artifact, and an orchestrator-authored deferred-work ledger append), `false` when it found changes, and `null` when the probe could not answer (a git fault, a git refusal such as an unresolvable baseline, or an attempt with no recorded baseline to measure from) and the gate was waived anyway, so the waiver itself is recorded whatever the probe managed to say. `false` is a statement about the tree, not about who wrote what: the gate this stands in for cannot attribute residue to a session in a shared checkout, and the record inherits that limit rather than improving on it); `tasks//` (per-session prompt + shared artifacts: [`result.json`, `escalation.json`] — respectively the per-session result and escalation outputs — plus adapter-specific breadcrumbs: `session-lifecycle.jsonl` records timeout fires, budget-guard trips (`budget-tripped` / `over-budget-fired`), transport-failure classification (`env-fault-classified`, #194), a mux session lost under the run (`session-vanished`, #489) and the #276 forensics (`spec-status-transition-observed`, `frontmatter-unmodified-refused`, `contract-nudge-sent`); `heartbeat.json` is the wait loop's proof-of-life; `resultless-stops.jsonl` records give-up Stops with a verdict — `no-artifact`, `ambiguous-frontmatter`, `unmodified-since-launch`, `terminal-frontmatter-pending`); `logs/`; `verify/` (verifier command stdout/stderr, one file per stream, pointed at by the journal's `verify-command-result` records — its own store, because every name in `logs/` is a session task id the TUI resolves as a pane log; each stream is retained tail-first up to `[verify] stream_capture_kb` (256 KiB, `0` = keep nothing) and the record carries the full byte count, a truncation flag, and a `capture_error` when the write itself failed); `deferred/`; `resolve/`; `ATTENTION`; `ctl-window` (the control-session window id the last TUI launch minted, so attach/stop follow the live window, #482). - One piece deliberately lives **outside** that directory: the hook-event channel (#494) is at `///events/` under the user-scoped state root (`BMAD_LOOP_STATE_DIR`, see the [transport section](#hook-based-transport-no-pane-scraping) below and the README's env-var table), not `/events/`. The orchestrator still polls the legacy in-tree location, so a project whose installed relay predates the move keeps completing its sessions. `delete`, `archive` and `clean` remove the out-of-tree counterpart along with the run dir, and `clean` sweeps counterparts whose run dir is already gone; an **archived** run's tarball therefore no longer contains `events/` — those files are transient completion signals, consumed while the run was live, and everything an archive is read for later is in the run dir. - `journal.jsonl` records `session-end` for every session unconditionally — even a teardown that throws still lands one (status `aborted` when the outcome is unknowable). A timed-out session's entry carries `fired_at` (wall time the deadline was declared), `teardown_s` (wall seconds from that fire to this entry — the teardown gap), and `expired_clock` (`monotonic` / `wall` / `both` — `wall` alone fingerprints a host suspend that froze the monotonic clock). Every entry whose usage was read carries `tokens` (raw) and `tokens_weighted` (cache reads at `limits.cache_read_weight`), keeping per-session spend reconstructible; both are `null` when the usage read failed, and both are absent on an `aborted` end. `tokens_weighted` is the end-of-session total — distinct from a tripped session's `budget_weighted`, the guard's mid-session sample at trip time. diff --git a/docs/adapter-authoring-guide.md b/docs/adapter-authoring-guide.md index 883f585f..3cbae97e 100644 --- a/docs/adapter-authoring-guide.md +++ b/docs/adapter-authoring-guide.md @@ -581,7 +581,17 @@ Three frozen dataclasses cross the seam: Required (abstract): -- `start_session(spec) -> SessionHandle` — launch the session. +- `start_session(spec) -> SessionHandle` — launch the session. An adapter that + persists the standard `tasks//` directory must reset its shared cycle + artifacts when an id is reused: after creating the task directory and before + launching the session, remove every file named by + `journal.TASK_CYCLE_ARTIFACTS` (shared artifacts: [`result.json`, + `escalation.json`]). + Use missing-safe deletion; a missing artifact is a normal no-op and must not + make startup fail. This tuple covers only artifacts shared across adapters and + readers. Adapter-private breadcrumbs such as `heartbeat.json`, + `resultless-stops.jsonl`, `session-lifecycle.jsonl`, and `messages.json` remain + outside the shared cleanup contract and are managed by their owning adapter. - `wait_for_completion(handle, spec) -> SessionResult` — block until the session ends (or stalls/times out), then report status. Poll `runs.read_stop_request_mode(run_dir) == "hard"` on both sides of the loop's diff --git a/tests/test_portability_guard.py b/tests/test_portability_guard.py index c6398ced..eb3745a0 100644 --- a/tests/test_portability_guard.py +++ b/tests/test_portability_guard.py @@ -2138,6 +2138,33 @@ def test_task_cycle_artifacts_named_only_through_the_constant(): ) +def test_task_cycle_artifact_docs_track_the_canonical_tuple(): + """The run inventory and extension boundary keep pace with the shared list.""" + project_root = Path(__file__).resolve().parents[1] + features = (project_root / "docs/FEATURES.md").read_text(encoding="utf-8") + inventory = features.split("- All run state in", 1)[1].split("\n- ", 1)[0] + guide = (project_root / "docs/adapter-authoring-guide.md").read_text(encoding="utf-8") + start_session_contract = guide.split("- `start_session", 1)[1].split( + "- `wait_for_completion", 1 + )[0] + + def shared_artifacts(contract: str) -> set[str]: + listing = contract.split("shared artifacts: [", 1)[1].split("]", 1)[0] + return set(listing.split("`")[1::2]) + + canonical = set(TASK_CYCLE_ARTIFACTS) + assert shared_artifacts(inventory) == canonical + assert shared_artifacts(start_session_contract) == canonical + + assert "`journal.TASK_CYCLE_ARTIFACTS`" in start_session_contract + assert ( + "after creating the task directory and before\n launching the session" + in start_session_contract + ) + assert "a missing artifact is a normal no-op" in start_session_contract + assert "Adapter-private breadcrumbs" in start_session_contract + + def _session_task_id_offenders(findings) -> list[tuple[str, int, str]]: """The chokepoint invariant as a filter: a composed task id is sanctioned only in a ``SESSION_TASK_ID_CHOKEPOINT`` file AND only inside that file's one listed From deb97e5ca0f08607a65066d479f9f90c3672469c Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 14:14:49 -0700 Subject: [PATCH 05/18] sweep dw3-adapter-task-dir-confinement: DW-74 via bmad-loop --- CHANGELOG.md | 4 + src/bmad_loop/adapters/base.py | 92 ++++++++ src/bmad_loop/adapters/generic.py | 30 ++- src/bmad_loop/adapters/opencode_http.py | 25 +- tests/test_generic_tmux.py | 288 +++++++++++++++++++++++- tests/test_opencode_http.py | 255 ++++++++++++++++++++- 6 files changed, 686 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 383863f8..4e3c5f52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -263,6 +263,10 @@ breaking changes may land in a minor release. unrecoverably, on a gitignored ledger. The mark and append legs now re-anchor only when the preimage they wrote over is still the bytes the run last claimed; the restore skips and journals `ledger-restore-skipped-diverged`. +- **Confine built-in adapter task directories** (DW-74), refusing unsafe task ids and + symlink- or junction-redirected task directories before prompt, artifact, log, or + transport side effects. + - **Treat embedded-NUL verify commands and working directories as typed environment faults** (DW-53, DW-54), while documenting that stream retention degrades but journal record writes remain fail-loud. diff --git a/src/bmad_loop/adapters/base.py b/src/bmad_loop/adapters/base.py index 7c1daef6..f021ff5e 100644 --- a/src/bmad_loop/adapters/base.py +++ b/src/bmad_loop/adapters/base.py @@ -14,12 +14,104 @@ from __future__ import annotations +import stat from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path from typing import Any from ..model import TokenUsage +from ..platform_util import is_link_like, safe_segment + + +class AdapterTaskDirectoryError(ValueError): + """A built-in adapter refused an unsafe or redirected task directory.""" + + +def validated_task_directory(tasks_dir: Path, task_id: str) -> Path: + """Return ``tasks/`` only when its authored name is confined. + + Validation is deliberately identity-based rather than sanitizing: callers use + ``task_id`` for handles, environment, logs, and artifacts, so rewriting it here + would split one session across multiple identities. The link-like check covers + both symlinks and Windows directory junctions through ``platform_util``. + + This is a pre-write boundary, not descriptor-anchored I/O; callers must invoke + it before any operation derived from the task id. + """ + if safe_segment(task_id) != task_id: + raise AdapterTaskDirectoryError( + f"unsafe adapter task id {task_id!r}: expected one clean path segment" + ) + + if is_link_like(tasks_dir): + raise AdapterTaskDirectoryError( + f"adapter tasks directory is a symlink or junction: {tasks_dir}" + ) + + task_dir = tasks_dir / task_id + if is_link_like(task_dir): + raise AdapterTaskDirectoryError( + f"adapter task directory is a symlink or junction: {task_dir}" + ) + return task_dir + + +def validate_adapter_artifact_paths(root_dir: Path, paths: tuple[Path, ...]) -> None: + """Refuse redirecting or special standing entries before adapter writes. + + A regular file with one link is the only existing leaf an adapter may open in + place. Symlinks, junctions, hardlinks, FIFOs, and devices can redirect or + block a later write; callers provide every leaf they will write during the + session and invoke this boundary before mutating any task or log artifact. + """ + if is_link_like(root_dir): + raise AdapterTaskDirectoryError( + f"adapter artifact directory is a symlink or junction: {root_dir}" + ) + + for path in paths: + if is_link_like(path): + raise AdapterTaskDirectoryError(f"adapter artifact is a symlink or junction: {path}") + try: + entry = path.lstat() + except FileNotFoundError: + continue + except OSError as exc: + raise AdapterTaskDirectoryError( + f"cannot inspect adapter artifact before writing: {path}" + ) from exc + if not stat.S_ISREG(entry.st_mode) or entry.st_nlink != 1: + raise AdapterTaskDirectoryError( + f"adapter artifact is special or multiply linked: {path}" + ) + + +def reset_task_prompt(task_dir: Path, prompt: str) -> None: + """Write ``prompt.txt`` without following a redirecting filesystem entry. + + A normal single-link file is truncated in place so its inode and metadata keep + the clean-session behavior. A symlink, hardlink, FIFO, or device is unlinked + first so the replacement is an ordinary file and no outside target is touched. + """ + prompt_path = task_dir / "prompt.txt" + try: + entry = prompt_path.lstat() + except FileNotFoundError: + pass + except OSError as exc: + raise AdapterTaskDirectoryError( + f"cannot inspect adapter prompt before writing: {prompt_path}" + ) from exc + else: + if not stat.S_ISREG(entry.st_mode) or entry.st_nlink != 1: + try: + prompt_path.unlink() + except OSError as exc: + raise AdapterTaskDirectoryError( + f"cannot replace unsafe adapter prompt: {prompt_path}" + ) from exc + prompt_path.write_text(prompt + "\n", encoding="utf-8") @dataclass(frozen=True) diff --git a/src/bmad_loop/adapters/generic.py b/src/bmad_loop/adapters/generic.py index dc8017fd..fa226e41 100644 --- a/src/bmad_loop/adapters/generic.py +++ b/src/bmad_loop/adapters/generic.py @@ -39,7 +39,16 @@ from ..signals import SignalWatcher from ..tokens import read_usage as tally_usage from ..verify import read_frontmatter, status_of -from .base import CodingCLIAdapter, SessionHandle, SessionResult, SessionSpec, SpecSnapshot +from .base import ( + CodingCLIAdapter, + SessionHandle, + SessionResult, + SessionSpec, + SpecSnapshot, + reset_task_prompt, + validate_adapter_artifact_paths, + validated_task_directory, +) # Re-exported for importers that predate the env_fault module split (#194 landed # these names on this module); the definitions now live in .env_fault. The @@ -538,9 +547,24 @@ def build_command(self, spec: SessionSpec) -> str: # --------------------------------------------------------------- adapter def start_session(self, spec: SessionSpec) -> SessionHandle: - task_dir = self.tasks_dir / spec.task_id + task_dir = validated_task_directory(self.tasks_dir, spec.task_id) + validate_adapter_artifact_paths( + task_dir, + tuple( + task_dir / name + for name in ( + "heartbeat.json", + "resultless-stops.jsonl", + "session-lifecycle.jsonl", + ) + ), + ) + validate_adapter_artifact_paths( + self.logs_dir, + (self.logs_dir / f"{spec.task_id}.log",), + ) task_dir.mkdir(parents=True, exist_ok=True) - (task_dir / "prompt.txt").write_text(spec.prompt + "\n", encoding="utf-8") + reset_task_prompt(task_dir, spec.prompt) # Task ids are supplied by the caller, so defensively reset cycle-scoped # outputs if one is reused. A silent session must not inherit a stale result. # The list is `journal.TASK_CYCLE_ARTIFACTS` rather than two literals here: diff --git a/src/bmad_loop/adapters/opencode_http.py b/src/bmad_loop/adapters/opencode_http.py index 032c4f81..bba082ad 100644 --- a/src/bmad_loop/adapters/opencode_http.py +++ b/src/bmad_loop/adapters/opencode_http.py @@ -150,7 +150,15 @@ from ..model import TokenUsage from ..policy import Policy from ..process_host import ProcessHostError, get_process_host -from .base import CodingCLIAdapter, SessionHandle, SessionResult, SessionSpec +from .base import ( + CodingCLIAdapter, + SessionHandle, + SessionResult, + SessionSpec, + reset_task_prompt, + validate_adapter_artifact_paths, + validated_task_directory, +) from .env_fault import EnvFaultMixin from .generic import ( BUDGET_NUDGE_TEXT, @@ -619,9 +627,20 @@ def _await_healthy(self, sess: _ServerSession) -> bool: # -------------------------------------------------------------- adapter def start_session(self, spec: SessionSpec) -> SessionHandle: - task_dir = self.tasks_dir / spec.task_id + task_dir = validated_task_directory(self.tasks_dir, spec.task_id) + validate_adapter_artifact_paths( + task_dir, + (task_dir / "messages.json",), + ) + log_paths = [ + self.logs_dir / f"{spec.task_id}.log", + self.logs_dir / f"{spec.task_id}.server.out", + ] + if self.sse_trace: + log_paths.append(self.logs_dir / f"{spec.task_id}.sse.jsonl") + validate_adapter_artifact_paths(self.logs_dir, tuple(log_paths)) task_dir.mkdir(parents=True, exist_ok=True) - (task_dir / "prompt.txt").write_text(spec.prompt + "\n", encoding="utf-8") + reset_task_prompt(task_dir, spec.prompt) # Task ids are supplied by the caller, so defensively reset cycle-scoped # outputs if one is reused. A silent session must not inherit a stale result. # Iterating `journal.TASK_CYCLE_ARTIFACTS` is what makes the parity with diff --git a/tests/test_generic_tmux.py b/tests/test_generic_tmux.py index 8124bee4..dc95261b 100644 --- a/tests/test_generic_tmux.py +++ b/tests/test_generic_tmux.py @@ -23,8 +23,15 @@ import regex from bmad_loop import devcontract, runs +from bmad_loop.adapters import base as adapter_base from bmad_loop.adapters import env_fault, generic, tmux_base -from bmad_loop.adapters.base import SessionHandle, SessionResult, SessionSpec, SpecSnapshot +from bmad_loop.adapters.base import ( + AdapterTaskDirectoryError, + SessionHandle, + SessionResult, + SessionSpec, + SpecSnapshot, +) from bmad_loop.adapters.generic import GenericDevAdapter, GenericTmuxAdapter from bmad_loop.adapters.multiplexer import MultiplexerError from bmad_loop.adapters.profile import get_profile @@ -3102,8 +3109,10 @@ class _StartSessionMux: def __init__(self): self.piped: list[tuple[str, Path]] = [] + self.windows: list[tuple[str, str, Path]] = [] def new_window(self, session_name, window_name, cwd, env, cmd): + self.windows.append((session_name, window_name, Path(cwd))) return "@1" def pipe_pane(self, window_id, log_file): @@ -3115,6 +3124,283 @@ def has_session(self, name): return True +@pytest.mark.parametrize( + "task_id_kind", ["absolute", "parent-traversal", "empty", "windows-reserved"] +) +def test_start_session_refuses_unconfined_task_id_without_side_effects(tmp_path, task_id_kind): + mux = _StartSessionMux() + adapter = make_adapter(tmp_path, mux=mux) + ensure_calls = [] + adapter._ensure_session = lambda cwd: ensure_calls.append(Path(cwd)) + outside = tmp_path / "outside" + outside.mkdir() + (outside / "prompt.txt").write_text("theirs", encoding="utf-8") + for artifact in TASK_CYCLE_ARTIFACTS: + (outside / artifact).write_text("theirs", encoding="utf-8") + before = {path.name: path.read_bytes() for path in outside.iterdir()} + task_ids = { + "absolute": str(outside), + "parent-traversal": str(Path("..") / ".." / "outside"), + "empty": "", + "windows-reserved": "CON", + } + task_id = task_ids[task_id_kind] + escaped_log = adapter.logs_dir / f"{task_id}.log" + + with pytest.raises(AdapterTaskDirectoryError, match="expected one clean path segment"): + adapter.start_session(make_spec(tmp_path, task_id=task_id)) + + assert {path.name: path.read_bytes() for path in outside.iterdir()} == before + assert list(adapter.tasks_dir.iterdir()) == [] + assert list(adapter.logs_dir.iterdir()) == [] + assert not escaped_log.exists() + assert ensure_calls == [] + assert mux.windows == [] + assert mux.piped == [] + + +def test_start_session_refuses_symlinked_task_directory_without_side_effects(tmp_path): + mux = _StartSessionMux() + adapter = make_adapter(tmp_path, mux=mux) + ensure_calls = [] + adapter._ensure_session = lambda cwd: ensure_calls.append(Path(cwd)) + outside = tmp_path / "outside" + outside.mkdir() + (outside / "prompt.txt").write_text("theirs", encoding="utf-8") + for artifact in TASK_CYCLE_ARTIFACTS: + (outside / artifact).write_text("theirs", encoding="utf-8") + before = {path.name: path.read_bytes() for path in outside.iterdir()} + task_id = "clean-task" + task_dir = adapter.tasks_dir / task_id + try: + task_dir.symlink_to(outside, target_is_directory=True) + except OSError as exc: + pytest.skip(f"directory symlinks unavailable: {exc}") + + with pytest.raises(AdapterTaskDirectoryError, match="symlink or junction"): + adapter.start_session(make_spec(tmp_path, task_id=task_id)) + + assert task_dir.is_symlink() + assert {path.name: path.read_bytes() for path in outside.iterdir()} == before + assert list(adapter.logs_dir.iterdir()) == [] + assert ensure_calls == [] + assert mux.windows == [] + assert mux.piped == [] + + +def test_start_session_refuses_symlinked_tasks_root_without_side_effects(tmp_path): + mux = _StartSessionMux() + adapter = make_adapter(tmp_path, mux=mux) + ensure_calls = [] + adapter._ensure_session = lambda cwd: ensure_calls.append(Path(cwd)) + outside = tmp_path / "outside" + outside.mkdir() + (outside / "theirs.txt").write_text("theirs", encoding="utf-8") + before = {path.name: path.read_bytes() for path in outside.iterdir()} + adapter.tasks_dir.rmdir() + try: + adapter.tasks_dir.symlink_to(outside, target_is_directory=True) + except OSError as exc: + pytest.skip(f"directory symlinks unavailable: {exc}") + + with pytest.raises(AdapterTaskDirectoryError, match="tasks directory is a symlink"): + adapter.start_session(make_spec(tmp_path, task_id="clean-task")) + + assert adapter.tasks_dir.is_symlink() + assert {path.name: path.read_bytes() for path in outside.iterdir()} == before + assert list(adapter.logs_dir.iterdir()) == [] + assert ensure_calls == [] + assert mux.windows == [] + assert mux.piped == [] + + +def test_start_session_refuses_junction_like_tasks_root_before_mutation(tmp_path, monkeypatch): + mux = _StartSessionMux() + adapter = make_adapter(tmp_path, mux=mux) + ensure_calls = [] + adapter._ensure_session = lambda cwd: ensure_calls.append(Path(cwd)) + monkeypatch.setattr(adapter_base, "is_link_like", lambda path: Path(path) == adapter.tasks_dir) + + with pytest.raises(AdapterTaskDirectoryError, match="tasks directory is a symlink"): + adapter.start_session(make_spec(tmp_path, task_id="clean-task")) + + assert list(adapter.tasks_dir.iterdir()) == [] + assert list(adapter.logs_dir.iterdir()) == [] + assert ensure_calls == [] + assert mux.windows == [] + assert mux.piped == [] + + +def test_start_session_replaces_prompt_symlink_without_following_it(tmp_path): + mux = _StartSessionMux() + adapter = make_adapter(tmp_path, mux=mux) + adapter._ensure_session = lambda cwd: None + task_id = "clean-task" + task_dir = adapter.tasks_dir / task_id + task_dir.mkdir() + outside_prompt = tmp_path / "outside-prompt.txt" + outside_prompt.write_text("theirs", encoding="utf-8") + prompt_path = task_dir / "prompt.txt" + try: + prompt_path.symlink_to(outside_prompt) + except OSError as exc: + pytest.skip(f"file symlinks unavailable: {exc}") + spec = make_spec(tmp_path, task_id=task_id) + + adapter.start_session(spec) + + assert outside_prompt.read_text(encoding="utf-8") == "theirs" + assert not prompt_path.is_symlink() + assert prompt_path.read_text(encoding="utf-8") == spec.prompt + "\n" + assert len(mux.windows) == 1 + assert len(mux.piped) == 1 + + +def test_start_session_replaces_prompt_hardlink_without_following_it(tmp_path): + mux = _StartSessionMux() + adapter = make_adapter(tmp_path, mux=mux) + adapter._ensure_session = lambda cwd: None + task_id = "clean-task" + task_dir = adapter.tasks_dir / task_id + task_dir.mkdir() + outside_prompt = tmp_path / "outside-prompt.txt" + outside_prompt.write_text("theirs", encoding="utf-8") + prompt_path = task_dir / "prompt.txt" + try: + os.link(outside_prompt, prompt_path) + except OSError as exc: + pytest.skip(f"hardlinks unavailable: {exc}") + spec = make_spec(tmp_path, task_id=task_id) + + adapter.start_session(spec) + + assert outside_prompt.read_text(encoding="utf-8") == "theirs" + assert prompt_path.stat().st_ino != outside_prompt.stat().st_ino + assert prompt_path.read_text(encoding="utf-8") == spec.prompt + "\n" + + +def test_start_session_preserves_existing_regular_prompt_inode(tmp_path): + mux = _StartSessionMux() + adapter = make_adapter(tmp_path, mux=mux) + adapter._ensure_session = lambda cwd: None + task_id = "clean-task" + task_dir = adapter.tasks_dir / task_id + task_dir.mkdir() + prompt_path = task_dir / "prompt.txt" + prompt_path.write_text("old", encoding="utf-8") + inode = prompt_path.stat().st_ino + spec = make_spec(tmp_path, task_id=task_id) + + adapter.start_session(spec) + + assert prompt_path.stat().st_ino == inode + assert prompt_path.read_text(encoding="utf-8") == spec.prompt + "\n" + + +@pytest.mark.parametrize( + "artifact_name", + ["heartbeat.json", "resultless-stops.jsonl", "session-lifecycle.jsonl"], +) +def test_start_session_refuses_redirected_task_artifact_before_mutation(tmp_path, artifact_name): + mux = _StartSessionMux() + adapter = make_adapter(tmp_path, mux=mux) + ensure_calls = [] + adapter._ensure_session = lambda cwd: ensure_calls.append(Path(cwd)) + task_dir = adapter.tasks_dir / "clean-task" + task_dir.mkdir() + outside = tmp_path / "outside.txt" + outside.write_text("theirs", encoding="utf-8") + try: + (task_dir / artifact_name).symlink_to(outside) + except OSError as exc: + pytest.skip(f"file symlinks unavailable: {exc}") + + with pytest.raises(AdapterTaskDirectoryError, match="artifact is a symlink"): + adapter.start_session(make_spec(tmp_path, task_id="clean-task")) + + assert outside.read_text(encoding="utf-8") == "theirs" + assert not (task_dir / "prompt.txt").exists() + assert list(adapter.logs_dir.iterdir()) == [] + assert ensure_calls == [] + assert mux.windows == [] + assert mux.piped == [] + + +@pytest.mark.parametrize("redirect_kind", ["root", "junction-like-root", "log-file"]) +def test_start_session_refuses_redirected_log_path_before_mutation( + tmp_path, redirect_kind, monkeypatch +): + mux = _StartSessionMux() + adapter = make_adapter(tmp_path, mux=mux) + ensure_calls = [] + adapter._ensure_session = lambda cwd: ensure_calls.append(Path(cwd)) + outside = tmp_path / "outside" + outside.mkdir() + outside_file = outside / "theirs.log" + outside_file.write_text("theirs", encoding="utf-8") + try: + if redirect_kind == "root": + adapter.logs_dir.rmdir() + adapter.logs_dir.symlink_to(outside, target_is_directory=True) + elif redirect_kind == "junction-like-root": + monkeypatch.setattr( + adapter_base, + "is_link_like", + lambda path: Path(path) == adapter.logs_dir, + ) + else: + (adapter.logs_dir / "clean-task.log").symlink_to(outside_file) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + with pytest.raises(AdapterTaskDirectoryError, match="artifact (directory )?is a symlink"): + adapter.start_session(make_spec(tmp_path, task_id="clean-task")) + + assert outside_file.read_text(encoding="utf-8") == "theirs" + assert not (adapter.tasks_dir / "clean-task").exists() + assert ensure_calls == [] + assert mux.windows == [] + assert mux.piped == [] + + +def test_start_session_refuses_junction_like_task_directory_before_mutation(tmp_path, monkeypatch): + """The adapter consumes the shared predicate's Windows-junction verdict. + + platform_util's reparse-tag tests own junction detection itself; an ordinary + directory standing in here makes this composition arm run on every platform. + """ + mux = _StartSessionMux() + adapter = make_adapter(tmp_path, mux=mux) + ensure_calls = [] + adapter._ensure_session = lambda cwd: ensure_calls.append(Path(cwd)) + task_id = "clean-task" + task_dir = adapter.tasks_dir / task_id + task_dir.mkdir() + (task_dir / "prompt.txt").write_text("theirs", encoding="utf-8") + for artifact in TASK_CYCLE_ARTIFACTS: + (task_dir / artifact).write_text("theirs", encoding="utf-8") + before = {path.name: path.read_bytes() for path in task_dir.iterdir()} + monkeypatch.setattr(adapter_base, "is_link_like", lambda path: Path(path) == task_dir) + + with pytest.raises(AdapterTaskDirectoryError, match="symlink or junction"): + adapter.start_session(make_spec(tmp_path, task_id=task_id)) + + assert {path.name: path.read_bytes() for path in task_dir.iterdir()} == before + assert list(adapter.logs_dir.iterdir()) == [] + assert ensure_calls == [] + assert mux.windows == [] + assert mux.piped == [] + + +def test_validated_task_directory_refuses_direct_junction_verdict(tmp_path, monkeypatch): + tasks_dir = tmp_path / "tasks" + task_dir = tasks_dir / "clean-task" + monkeypatch.setattr(adapter_base, "is_link_like", lambda path: Path(path) == task_dir) + + with pytest.raises(AdapterTaskDirectoryError, match="task directory is a symlink"): + adapter_base.validated_task_directory(tasks_dir, "clean-task") + + def test_start_session_resets_reused_task_log(tmp_path): """A re-armed run reuses task_ids and both mux backends APPEND to logs/.log, so a prior cycle's transport-failure line would linger in the diff --git a/tests/test_opencode_http.py b/tests/test_opencode_http.py index 669348c7..169068c6 100644 --- a/tests/test_opencode_http.py +++ b/tests/test_opencode_http.py @@ -25,8 +25,14 @@ from conftest import write_script_launcher from bmad_loop import runs +from bmad_loop.adapters import base as adapter_base from bmad_loop.adapters import generic, opencode_http -from bmad_loop.adapters.base import SessionHandle, SessionResult, SessionSpec +from bmad_loop.adapters.base import ( + AdapterTaskDirectoryError, + SessionHandle, + SessionResult, + SessionSpec, +) from bmad_loop.adapters.generic import BUDGET_NUDGE_TEXT, NUDGE_TEXT, STALL_NUDGE_TEXT from bmad_loop.adapters.opencode_http import ( _RESET, @@ -1291,6 +1297,253 @@ def test_missing_binary_is_a_clean_error(tmp_path): adapter.start_session(spec) +@pytest.mark.parametrize( + "task_id_kind", ["absolute", "parent-traversal", "empty", "windows-reserved"] +) +def test_start_session_refuses_unconfined_task_id_without_side_effects(tmp_path, task_id_kind): + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + spawn_calls = [] + adapter._spawn_server = lambda spec: spawn_calls.append(spec) + outside = tmp_path / "outside" + outside.mkdir() + (outside / "prompt.txt").write_text("theirs", encoding="utf-8") + for artifact in TASK_CYCLE_ARTIFACTS: + (outside / artifact).write_text("theirs", encoding="utf-8") + before = {path.name: path.read_bytes() for path in outside.iterdir()} + task_ids = { + "absolute": str(outside), + "parent-traversal": str(Path("..") / ".." / "outside"), + "empty": "", + "windows-reserved": "CON", + } + task_id = task_ids[task_id_kind] + escaped_log = adapter.logs_dir / f"{task_id}.server.out" + spec = SessionSpec(task_id=task_id, role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(AdapterTaskDirectoryError, match="expected one clean path segment"): + adapter.start_session(spec) + + assert {path.name: path.read_bytes() for path in outside.iterdir()} == before + assert list(adapter.tasks_dir.iterdir()) == [] + assert list(adapter.logs_dir.iterdir()) == [] + assert not escaped_log.exists() + assert spawn_calls == [] + assert adapter._sessions == {} + + +def test_start_session_refuses_symlinked_task_directory_without_side_effects(tmp_path): + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + spawn_calls = [] + adapter._spawn_server = lambda spec: spawn_calls.append(spec) + outside = tmp_path / "outside" + outside.mkdir() + (outside / "prompt.txt").write_text("theirs", encoding="utf-8") + for artifact in TASK_CYCLE_ARTIFACTS: + (outside / artifact).write_text("theirs", encoding="utf-8") + before = {path.name: path.read_bytes() for path in outside.iterdir()} + task_id = "clean-task" + task_dir = adapter.tasks_dir / task_id + try: + task_dir.symlink_to(outside, target_is_directory=True) + except OSError as exc: + pytest.skip(f"directory symlinks unavailable: {exc}") + spec = SessionSpec(task_id=task_id, role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(AdapterTaskDirectoryError, match="symlink or junction"): + adapter.start_session(spec) + + assert task_dir.is_symlink() + assert {path.name: path.read_bytes() for path in outside.iterdir()} == before + assert list(adapter.logs_dir.iterdir()) == [] + assert spawn_calls == [] + assert adapter._sessions == {} + + +def test_start_session_refuses_symlinked_tasks_root_without_side_effects(tmp_path): + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + spawn_calls = [] + adapter._spawn_server = lambda spec: spawn_calls.append(spec) + outside = tmp_path / "outside" + outside.mkdir() + (outside / "theirs.txt").write_text("theirs", encoding="utf-8") + before = {path.name: path.read_bytes() for path in outside.iterdir()} + adapter.tasks_dir.rmdir() + try: + adapter.tasks_dir.symlink_to(outside, target_is_directory=True) + except OSError as exc: + pytest.skip(f"directory symlinks unavailable: {exc}") + spec = SessionSpec(task_id="clean-task", role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(AdapterTaskDirectoryError, match="tasks directory is a symlink"): + adapter.start_session(spec) + + assert adapter.tasks_dir.is_symlink() + assert {path.name: path.read_bytes() for path in outside.iterdir()} == before + assert list(adapter.logs_dir.iterdir()) == [] + assert spawn_calls == [] + assert adapter._sessions == {} + + +def test_start_session_refuses_junction_like_tasks_root_before_mutation(tmp_path, monkeypatch): + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + spawn_calls = [] + adapter._spawn_server = lambda spec: spawn_calls.append(spec) + monkeypatch.setattr(adapter_base, "is_link_like", lambda path: Path(path) == adapter.tasks_dir) + spec = SessionSpec(task_id="clean-task", role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(AdapterTaskDirectoryError, match="tasks directory is a symlink"): + adapter.start_session(spec) + + assert list(adapter.tasks_dir.iterdir()) == [] + assert list(adapter.logs_dir.iterdir()) == [] + assert spawn_calls == [] + assert adapter._sessions == {} + + +def test_start_session_replaces_prompt_symlink_without_following_it(tmp_path): + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + task_id = "clean-task" + task_dir = adapter.tasks_dir / task_id + task_dir.mkdir() + outside_prompt = tmp_path / "outside-prompt.txt" + outside_prompt.write_text("theirs", encoding="utf-8") + prompt_path = task_dir / "prompt.txt" + try: + prompt_path.symlink_to(outside_prompt) + except OSError as exc: + pytest.skip(f"file symlinks unavailable: {exc}") + spec = SessionSpec(task_id=task_id, role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(OpencodeServerError, match="not found on PATH"): + adapter.start_session(spec) + + assert outside_prompt.read_text(encoding="utf-8") == "theirs" + assert not prompt_path.is_symlink() + assert prompt_path.read_text(encoding="utf-8") == "p\n" + assert adapter._sessions == {} + + +def test_start_session_replaces_prompt_hardlink_without_following_it(tmp_path): + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + task_id = "clean-task" + task_dir = adapter.tasks_dir / task_id + task_dir.mkdir() + outside_prompt = tmp_path / "outside-prompt.txt" + outside_prompt.write_text("theirs", encoding="utf-8") + prompt_path = task_dir / "prompt.txt" + try: + os.link(outside_prompt, prompt_path) + except OSError as exc: + pytest.skip(f"hardlinks unavailable: {exc}") + spec = SessionSpec(task_id=task_id, role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(OpencodeServerError, match="not found on PATH"): + adapter.start_session(spec) + + assert outside_prompt.read_text(encoding="utf-8") == "theirs" + assert prompt_path.stat().st_ino != outside_prompt.stat().st_ino + assert prompt_path.read_text(encoding="utf-8") == "p\n" + assert adapter._sessions == {} + + +def test_start_session_refuses_redirected_messages_before_mutation(tmp_path): + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + spawn_calls = [] + adapter._spawn_server = lambda spec: spawn_calls.append(spec) + task_dir = adapter.tasks_dir / "clean-task" + task_dir.mkdir() + outside = tmp_path / "outside-messages.json" + outside.write_text("theirs", encoding="utf-8") + try: + (task_dir / "messages.json").symlink_to(outside) + except OSError as exc: + pytest.skip(f"file symlinks unavailable: {exc}") + spec = SessionSpec(task_id="clean-task", role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(AdapterTaskDirectoryError, match="artifact is a symlink"): + adapter.start_session(spec) + + assert outside.read_text(encoding="utf-8") == "theirs" + assert not (task_dir / "prompt.txt").exists() + assert list(adapter.logs_dir.iterdir()) == [] + assert spawn_calls == [] + assert adapter._sessions == {} + + +@pytest.mark.parametrize("suffix", [".log", ".server.out", ".sse.jsonl"]) +def test_start_session_refuses_redirected_log_path_before_mutation(tmp_path, suffix): + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + spawn_calls = [] + adapter._spawn_server = lambda spec: spawn_calls.append(spec) + outside = tmp_path / "outside.log" + outside.write_text("theirs", encoding="utf-8") + try: + (adapter.logs_dir / f"clean-task{suffix}").symlink_to(outside) + except OSError as exc: + pytest.skip(f"file symlinks unavailable: {exc}") + spec = SessionSpec(task_id="clean-task", role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(AdapterTaskDirectoryError, match="artifact is a symlink"): + adapter.start_session(spec) + + assert outside.read_text(encoding="utf-8") == "theirs" + assert not (adapter.tasks_dir / "clean-task").exists() + assert spawn_calls == [] + assert adapter._sessions == {} + + +def test_start_session_refuses_redirected_logs_root_before_mutation(tmp_path): + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + spawn_calls = [] + adapter._spawn_server = lambda spec: spawn_calls.append(spec) + outside = tmp_path / "outside" + outside.mkdir() + outside_file = outside / "clean-task.server.out" + outside_file.write_text("theirs", encoding="utf-8") + adapter.logs_dir.rmdir() + try: + adapter.logs_dir.symlink_to(outside, target_is_directory=True) + except OSError as exc: + pytest.skip(f"directory symlinks unavailable: {exc}") + spec = SessionSpec(task_id="clean-task", role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(AdapterTaskDirectoryError, match="artifact directory is a symlink"): + adapter.start_session(spec) + + assert outside_file.read_text(encoding="utf-8") == "theirs" + assert not (adapter.tasks_dir / "clean-task").exists() + assert spawn_calls == [] + assert adapter._sessions == {} + + +def test_start_session_refuses_junction_like_task_directory_before_mutation(tmp_path, monkeypatch): + """The adapter consumes the shared predicate's Windows-junction verdict. + + platform_util's reparse-tag tests own junction detection itself; an ordinary + directory standing in here makes this composition arm run on every platform. + """ + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + spawn_calls = [] + adapter._spawn_server = lambda spec: spawn_calls.append(spec) + task_id = "clean-task" + task_dir = adapter.tasks_dir / task_id + task_dir.mkdir() + (task_dir / "prompt.txt").write_text("theirs", encoding="utf-8") + for artifact in TASK_CYCLE_ARTIFACTS: + (task_dir / artifact).write_text("theirs", encoding="utf-8") + before = {path.name: path.read_bytes() for path in task_dir.iterdir()} + monkeypatch.setattr(adapter_base, "is_link_like", lambda path: Path(path) == task_dir) + spec = SessionSpec(task_id=task_id, role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(AdapterTaskDirectoryError, match="symlink or junction"): + adapter.start_session(spec) + + assert {path.name: path.read_bytes() for path in task_dir.iterdir()} == before + assert list(adapter.logs_dir.iterdir()) == [] + assert spawn_calls == [] + assert adapter._sessions == {} + + def test_start_session_drops_every_reused_task_cycle_artifact(tmp_path): """Parity with GenericAdapter: both adapters own a tasks// dir, so both must drop a prior cycle's artifacts before a re-armed run reusing the id lands there. From f92b8d6961b8a95de7b4eec00b9f2701b07173aa Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 15:02:25 -0700 Subject: [PATCH 06/18] sweep dw4-diagnostic-journal-sanitization: DW-76, DW-77, DW-80, DW-84 via bmad-loop --- CHANGELOG.md | 7 + src/bmad_loop/diagnostics.py | 108 ++++++++++-- src/bmad_loop/sanitize.py | 5 +- tests/test_cli.py | 4 +- tests/test_diagnostics.py | 299 +++++++++++++++++++++++++++++++- tests/test_portability_guard.py | 64 ++++--- 6 files changed, 441 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e3c5f52..df50b4ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,13 @@ breaking changes may land in a minor release. ### Changed +- **`bmad-loop diagnose --json` reports `schema_version: 4`.** Journal `path` values + become `path_present`; stale-restore and merge filename lists become counts. + +- **Sanitize remaining diagnostic journal identifiers.** Commit residue and sentinel + names are aliased explicitly, excluded and merge filenames are counted, overloaded + paths are presence-only, and derived fields win same-named raw-field collisions. + - **Remove the unused whole-artifact-folder exclusion helper** (DW-15). Proof-of-work exclusions remain file-granular and rollback protection keeps its workspace-rooted path derivation. diff --git a/src/bmad_loop/diagnostics.py b/src/bmad_loop/diagnostics.py index 99c72ee6..eff895eb 100644 --- a/src/bmad_loop/diagnostics.py +++ b/src/bmad_loop/diagnostics.py @@ -86,7 +86,11 @@ # serialized to `journal.jsonl` and read back, and JSON has no tuple type, so every # sequence arrives as a `list` and takes the same arm it always did. The new arm is # reachable only by a shape no round-tripped entry can hold. -SCHEMA_VERSION = 3 +# v4 removes the overloaded journal-entry `path` value in favour of `path_present` +# and replaces `stale-restore-excluded.files` with `files_count`. The same release +# explicitly aliases `stale-restore-commits.commits` and `sentinel-cleared.sentinel`; +# those two fields keep their names, but their values no longer carry source ids. +SCHEMA_VERSION = 4 DEFAULT_JOURNAL_CAP = 200 # Subdirectories whose mere existence/size is diagnostic but whose CONTENTS are @@ -202,8 +206,8 @@ # Kind-scoped routing, consulted BEFORE the by-name table above and losing to # `_JOURNAL_DROP_FIELDS`, which is stricter than any alias. # -# It exists for ONE field, and the by-name rule genuinely cannot express it: `target` -# carries the target BRANCH on the three merge kinds below and a sprint STATUS on the +# `target` is the field for which a by-name rule genuinely cannot work: it carries +# the target BRANCH on the three merge kinds below and a sprint STATUS on the # `board-advance-*` family (`board-advance-carried`, `-carry-failed`, # `-carry-foreign-dirt`, `-carry-uncommitted`). Aliasing it by NAME would pseudonymize # statuses as branches, turning a legible `"target": "done"` into `branch-3f2a` and @@ -223,13 +227,16 @@ # correlate, silently skipping the carry replay so a resumed sweep re-triages work that # already landed. The scrub is what is wrong, so the scrub is where the fix belongs. # -# A closed set, not a growing one: any NEW producer should pick a name the by-name table -# already routes (`branch`, or `target_branch` — see `runs.rearm_escalation`) rather than -# add a row here. +# Any new branch producer should pick a name the by-name table already routes +# (`branch`, or `target_branch` — see `runs.rearm_escalation`) rather than add a target +# row here. `sentinel` is scoped for a different reason: its sole producer carries a +# spec basename, so that known shape is aliased without making the same claim about a +# future kind that reuses the generic name. _JOURNAL_KIND_ALIAS_FIELDS: dict[str, dict[str, str]] = { "unit-merge-started": {"target": "branch"}, "unit-merged": {"target": "branch"}, "resume-unit-merge": {"target": "branch"}, + "sentinel-cleared": {"sentinel": "spec"}, } # Namespaces whose journalled value arrives in more than one shape and must be # reduced to its basename before it is aliased. `spec` is one: engine.py's @@ -335,6 +342,11 @@ # and spec filename. Drop rather than create a second spec correlation; # the fallback redacts it only by virtue of its current separators. "stashed_to", + # An overloaded path spelling used for a generated sweep intent and for + # isolated worktrees. None of those host/customer paths adds useful + # correlation beyond the record's story key, so every kind gets the same + # presence-only treatment. + "path", } ) # Journal fields whose value is a LIST of identifiers, aliased element-wise rather @@ -347,6 +359,21 @@ # beside it in the neighbouring record was aliased. _JOURNAL_KEYLIST_FIELDS = frozenset({"keys", "dw_ids", "story_keys"}) +# Kind-scoped container policies for names whose other producers carry a different +# shape. `commits` is a SHA list on the stale-restore record but an integer count on +# `rollback-manual-required`, so a by-name list rule would destroy that useful count. +# Filename lists are reduced to counts rather than aliases: filename correlation +# adds no diagnostic value and would put the proprietary names into the legend. +_JOURNAL_KIND_KEYLIST_FIELDS: dict[str, dict[str, str]] = { + "stale-restore-commits": {"commits": "commit"}, +} +_JOURNAL_KIND_COUNTLIST_FIELDS: dict[str, frozenset[str]] = { + "merge-preflight-refused": frozenset({"tolerated"}), + "merge-target-cleaned": frozenset({"paths"}), + "merge-target-tolerated": frozenset({"paths"}), + "stale-restore-excluded": frozenset({"files"}), +} + # ``kind -> the field names that kind's record is DECLARED to carry``. On a kind # listed here the usual ``scrub_json`` fallback is replaced by a fail-closed one: # any key outside its declared set renders as ``_present`` rather than as a @@ -837,6 +864,45 @@ def _alias_input(value: Any, ns: str) -> Any: return _PATH_SEP_RE.split(value)[-1] or value +def _reserved_output_names( + entry: dict, + kind_aliases: dict[str, str], + kind_keylists: dict[str, str], + kind_countlists: frozenset[str], + declared: frozenset[str] | None, +) -> frozenset[str]: + """Names synthesized from this complete raw entry. + + Computing them before the entry is traversed makes a derived presence marker or + count authoritative when the source also contains that name, independent of JSON + key order. The branches deliberately mirror `_scrub_entry`'s routing precedence: + a routed alias on a declared-schema kind does not also generate a presence name. + """ + # ``ts_offset`` is synthesized before the raw fields are traversed. It is never + # a meaningful producer field, so reserve it even for a malformed entry whose + # timestamp cannot produce the derived value; a caller-supplied value must not + # replace the truthful offset or leak through the generic scrubber. + generated: set[str] = {"ts_offset"} + for key, value in entry.items(): + if key in ("ts", "kind"): + continue + if key in _JOURNAL_DROP_FIELDS: + generated.add(f"{key}_present") + elif key in _JOURNAL_KEYLIST_FIELDS: + if not isinstance(value, list): + generated.add(f"{key}_present") + elif key in kind_keylists: + if not isinstance(value, list): + generated.add(f"{key}_present") + elif key in kind_countlists: + generated.add(f"{key}_{'count' if isinstance(value, list) else 'present'}") + elif key in kind_aliases or key in _JOURNAL_ALIAS_FIELDS: + continue + elif declared is not None and key not in declared and key not in SELF_MINTED_FIELDS: + generated.add(f"{key}_present") + return frozenset(generated) + + def _scrub_entry( entry: dict, pseudo: sanitize.Pseudonymizer, @@ -847,11 +913,10 @@ def _scrub_entry( verbatim, identifier fields aliased, free-text fields collapsed to a presence boolean, and every remaining/unknown field scrub_json'd. - Two kinds of field never reach that last fallback, because for them - ``scrub_json`` fails closed only by accident of a value's shape. A name in - ``_JOURNAL_KEYLIST_FIELDS`` carrying something other than a list collapses to a - presence key, and on a kind with a declared schema - (``_JOURNAL_KIND_SCHEMAS``) so does every key the schema does not name.""" + Several field classes never reach that last fallback, because for them + ``scrub_json`` fails closed only by accident of a value's shape. Identifier-list + and count-list policies validate their containers, while a kind with a declared + schema (``_JOURNAL_KIND_SCHEMAS``) collapses every unnamed key to presence.""" out: dict[str, Any] = {} ts = entry.get("ts") if isinstance(ts, (int, float)) and first_ts is not None: @@ -862,10 +927,20 @@ def _scrub_entry( # `looks_like_identifier` is not one of the three below anyway, and keying on the # placeholder would silently unroute every entry in a dump that had one. by_kind = _JOURNAL_KIND_ALIAS_FIELDS.get(kind, {}) + kind_keylists = _JOURNAL_KIND_KEYLIST_FIELDS.get(kind, {}) + kind_countlists = _JOURNAL_KIND_COUNTLIST_FIELDS.get(kind, frozenset()) declared = _JOURNAL_KIND_SCHEMAS.get(kind) + reserved_outputs = _reserved_output_names( + entry, by_kind, kind_keylists, kind_countlists, declared + ) for k, v in entry.items(): if k in ("ts", "kind"): continue + if k in reserved_outputs: + # A raw field cannot overwrite a value derived from another field in this + # entry. The reservation is computed above from the complete shape, so + # this is deterministic in both source-key orders. + continue kind_ns = by_kind.get(k) if k in _JOURNAL_DROP_FIELDS: out[f"{k}_present"] = v is not None and v != "" @@ -889,6 +964,17 @@ def _scrub_entry( # genuinely unknown: a dict or an int has no sensible alias, and the # one thing worth reporting is that the field was set. out[f"{k}_present"] = v is not None and v != "" + elif k in kind_keylists: + if isinstance(v, list): + ns = kind_keylists[k] + out[k] = [pseudo.alias(x, ns=ns) for x in v] + else: + out[f"{k}_present"] = v is not None and v != "" + elif k in kind_countlists: + if isinstance(v, list): + out[f"{k}_count"] = len(v) + else: + out[f"{k}_present"] = v is not None and v != "" elif kind_ns is not None or k in _JOURNAL_ALIAS_FIELDS: ns = kind_ns or _JOURNAL_ALIAS_FIELDS[k] v = _alias_input(v, ns) diff --git a/src/bmad_loop/sanitize.py b/src/bmad_loop/sanitize.py index 63b30461..a0a4c676 100644 --- a/src/bmad_loop/sanitize.py +++ b/src/bmad_loop/sanitize.py @@ -413,7 +413,10 @@ def alias(self, value: Any, *, ns: str = "id", epic: int | None = None) -> Any: # collision re-hash with a counter until the alias is free. counter = 0 while True: - material = self._salt + value.encode("utf-8") + # Journal strings can carry surrogateescape code points when a POSIX + # filename contains undecodable bytes. Hash those values losslessly + # instead of letting one malformed identifier make its run unreadable. + material = self._salt + value.encode("utf-8", errors="surrogatepass") if counter: material += counter.to_bytes(4, "big") alias = f"{prefix}-{hashlib.blake2s(material, digest_size=6).hexdigest()}" diff --git a/tests/test_cli.py b/tests/test_cli.py index 09b09736..995407de 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5901,7 +5901,7 @@ def test_diagnose_json_emits_pure_document(project, capsys): _seed_run(project.project) doc = machine_json(["diagnose", "--project", str(project.project), "--json"], capsys) - assert doc["schema_version"] == diagnostics.SCHEMA_VERSION == 3 + assert doc["schema_version"] == diagnostics.SCHEMA_VERSION == 4 assert doc["runs"], "the document carries the run it resolved" for canary in CANARIES: assert canary not in json.dumps(doc), f"LEAK via CLI: {canary!r}" @@ -5921,7 +5921,7 @@ def test_diagnose_json_out_writes_document_and_keeps_stdout_empty(project, tmp_p assert "written to" in err # the confirmation moved to stderr written = out_file.read_text() doc = json.loads(written) - assert doc["schema_version"] == diagnostics.SCHEMA_VERSION == 3 + assert doc["schema_version"] == diagnostics.SCHEMA_VERSION == 4 assert "```" not in written # no fences in a file written in JSON mode for canary in CANARIES: assert canary not in written, f"LEAK via CLI: {canary!r}" diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index 01be5c00..51b62ecc 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -681,9 +681,9 @@ def test_the_two_commit_probe_records_alias_one_baseline_to_one_name(): grade: depending on its entropy, the fallback may redact a sha as a secret rather than preserving the correlatable alias this table promises. - `commits` remains deliberately outside this test and outside DW-81's routing - change. It is a list on `stale-restore-commits` but an integer count on - `rollback-manual-required`; routing it requires a separate, kind-scoped policy. + `commits` is now routed by kind because it is a list here but an integer count on + `rollback-manual-required`; this row therefore also sees the residue SHA enter + the same commit namespace without changing the baseline's alias. """ pseudo = sanitize.Pseudonymizer(salt=b"fixed") probe_failed = diagnostics._scrub_entry( @@ -714,8 +714,9 @@ def test_the_two_commit_probe_records_alias_one_baseline_to_one_name(): alias = next(a for ns, orig, a in pseudo.entries() if ns == "commit" and orig == SHA) # aliased, not dropped — the key stays and only the VALUE is replaced assert probe_failed["old_baseline"] == commits["old_baseline"] == alias != SHA - # one legend entry for the shared baseline, not one per record spelling - assert {orig for ns, orig, _a in pseudo.entries() if ns == "commit"} == {SHA} + # one legend entry for the shared baseline, not one per record spelling, plus + # the independently aliased residue commit + assert {orig for ns, orig, _a in pseudo.entries() if ns == "commit"} == {SHA, "c" * 40} # the free-text sibling on the probe record quotes both the sha and a host path # back, and is reached by the drop set rather than aliased assert "error" not in probe_failed and probe_failed["error_present"] is True @@ -725,6 +726,199 @@ def test_the_two_commit_probe_records_alias_one_baseline_to_one_name(): assert canary not in rendered, f"LEAK: {canary!r}" +def test_remaining_journal_shapes_route_by_kind_and_preserve_safe_structure(): + """The overloaded names are handled according to the producer shape, not by + their generic scalar fallback.""" + pseudo = sanitize.Pseudonymizer(salt=b"fixed") + commit_values = [SHA, "0f" * 20] + commits = diagnostics._scrub_entry( + {"kind": "stale-restore-commits", "commits": commit_values}, pseudo, {}, None + ) + files = diagnostics._scrub_entry( + { + "kind": "stale-restore-excluded", + "files": ["AcmePayrollExport.py", "AcmeMergerPlan.md"], + }, + pseudo, + {}, + None, + ) + bare_sentinel = diagnostics._scrub_entry( + {"kind": "sentinel-cleared", "sentinel": SPEC_NAME}, pseudo, {}, None + ) + qualified_sentinel = diagnostics._scrub_entry( + {"kind": "sentinel-cleared", "sentinel": SPEC_ABS}, pseudo, {}, None + ) + manual_count = diagnostics._scrub_entry( + {"kind": "rollback-manual-required", "commits": 2}, pseudo, {}, None + ) + + assert commits["commits"] != commit_values + assert len(commits["commits"]) == 2 + assert all(value.startswith("commit-") for value in commits["commits"]) + expected_commit_aliases = [ + next( + alias + for ns, original, alias in pseudo.entries() + if ns == "commit" and original == value + ) + for value in commit_values + ] + assert commits["commits"] == expected_commit_aliases + assert len(set(commits["commits"])) == len(commit_values) + assert files == {"kind": "stale-restore-excluded", "files_count": 2} + assert bare_sentinel["sentinel"] == qualified_sentinel["sentinel"] + assert bare_sentinel["sentinel"].startswith("spec-") + assert manual_count["commits"] == 2 + + legend = pseudo.legend() + assert SPEC_ABS not in legend.values() + assert "AcmePayrollExport.py" not in legend.values() + assert "AcmeMergerPlan.md" not in legend.values() + + +def test_non_string_sentinel_is_safely_pseudonymized(): + """A malformed sentinel still takes the explicit alias route. + + Ablation: remove the `sentinel-cleared` kind route and `scrub_json` preserves + the identifier-shaped nested value instead of returning one opaque spec alias. + """ + raw = {"customer_spec": "AcmeVaultRotation"} + pseudo = sanitize.Pseudonymizer(salt=b"fixed") + scrubbed = diagnostics._scrub_entry( + {"kind": "sentinel-cleared", "sentinel": raw}, pseudo, {}, None + ) + + assert isinstance(scrubbed["sentinel"], str) + assert scrubbed["sentinel"].startswith("spec-") + assert raw["customer_spec"] not in json.dumps(scrubbed) + assert pseudo.entries() == [("spec", str(raw), scrubbed["sentinel"])] + + +def test_journal_alias_routes_accept_lone_unicode_surrogates(project): + run_dir = _seed_run(project.project) + sentinel_value = chr(0xDC80) + commit_values = [chr(0xDC81), chr(0xDC82)] + journal = Journal(run_dir) + journal.append("sentinel-cleared", sentinel=sentinel_value) + journal.append("stale-restore-commits", commits=commit_values) + + pseudo = sanitize.Pseudonymizer(salt=b"fixed") + diag = diagnostics.collect([run_dir], pseudo=pseudo, project=project.project) + rendered = diagnostics.render_json(diag, pseudo=pseudo) + entries = json.loads(rendered)["runs"][0]["journal"]["entries"] + sentinel = next(entry for entry in entries if entry["kind"] == "sentinel-cleared") + commits = next(entry for entry in entries if entry["kind"] == "stale-restore-commits") + + assert sentinel["sentinel"].startswith("spec-") + assert all(value.startswith("commit-") for value in commits["commits"]) + assert len(set(commits["commits"])) == len(commit_values) + assert sentinel_value not in rendered + assert all(value not in rendered for value in commit_values) + + +@pytest.mark.parametrize( + ("kind", "field", "value"), + [ + ("stale-restore-commits", "commits", "AcmeCommitResidue"), + ("stale-restore-excluded", "files", "AcmePayrollExport.py"), + ], +) +def test_kind_scoped_container_routes_fail_closed_on_malformed_shapes(kind, field, value): + scrubbed = diagnostics._scrub_entry( + {"kind": kind, field: value}, sanitize.Pseudonymizer(salt=b"fixed"), {}, None + ) + + assert field not in scrubbed + assert scrubbed[f"{field}_present"] is True + assert f"{field}_count" not in scrubbed + assert value not in json.dumps(scrubbed) + + +@pytest.mark.parametrize("raw_first", [True, False], ids=["raw-first", "raw-last"]) +def test_derived_files_count_wins_raw_count_collision_in_both_orders(raw_first): + fields = [("files_count", 999), ("files", ["one.py", "two.py"])] + if not raw_first: + fields.reverse() + scrubbed = diagnostics._scrub_entry( + {"kind": "stale-restore-excluded", **dict(fields)}, + sanitize.Pseudonymizer(salt=b"fixed"), + {}, + None, + ) + + assert scrubbed["files_count"] == 2 + assert "files" not in scrubbed + + +@pytest.mark.parametrize( + ("kind", "field", "malformed"), + [ + ("run-start", "story_keys", "AcmeStoryKey"), + ("stale-restore-commits", "commits", "AcmeCommitResidue"), + ("stale-restore-excluded", "files", "AcmePayrollExport.py"), + ], + ids=["global-keylist", "kind-keylist", "kind-countlist"], +) +@pytest.mark.parametrize("raw_first", [True, False], ids=["raw-first", "raw-last"]) +def test_derived_malformed_presence_wins_raw_collision_in_both_orders( + kind, field, malformed, raw_first +): + presence = f"{field}_present" + raw_value = f"AcmeRaw{field.title()}Presence" + fields = [(presence, raw_value), (field, malformed)] + if not raw_first: + fields.reverse() + scrubbed = diagnostics._scrub_entry( + {"kind": kind, **dict(fields)}, sanitize.Pseudonymizer(salt=b"fixed"), {}, None + ) + + assert scrubbed[presence] is True + assert field not in scrubbed + assert raw_value not in json.dumps(scrubbed) + + +def test_declared_schema_reservation_respects_routed_field_precedence(): + raw_presence = "AcmeRawStoryPresence" + pseudo = sanitize.Pseudonymizer(salt=b"fixed") + scrubbed = diagnostics._scrub_entry( + { + "kind": "preference-escalation", + "story_key": STORY_KEY, + "story_key_present": raw_presence, + }, + pseudo, + {}, + None, + ) + + assert scrubbed["story_key"].startswith("story-") + assert "story_key_present" not in scrubbed + assert scrubbed["story_key_present_present"] is True + assert raw_presence not in json.dumps(scrubbed) + + +@pytest.mark.parametrize("value", [None, ""]) +def test_empty_path_becomes_a_false_presence_flag(value): + scrubbed = diagnostics._scrub_entry( + {"kind": "worktree-opened", "path": value}, + sanitize.Pseudonymizer(salt=b"fixed"), + {}, + None, + ) + assert scrubbed == {"kind": "worktree-opened", "path_present": False} + + +def test_unrelated_raw_presence_field_is_not_suppressed(): + scrubbed = diagnostics._scrub_entry( + {"kind": "attempt-restored", "patch_present": False}, + sanitize.Pseudonymizer(salt=b"fixed"), + {}, + None, + ) + assert scrubbed["patch_present"] is False + + def test_sentinel_upstream_record_drops_the_stories_root_it_names(): """`rearm-upstream-write-unreachable` carries an absolute host path naming the folder a sentinel's upstream correction has to land in. @@ -891,6 +1085,101 @@ def test_patch_and_stash_paths_are_absent_from_public_diagnostic_renders(project assert dropped not in legend_values +def test_remaining_journal_sanitization_contract_reaches_both_public_renders(project): + """Separator-free canaries grade the explicit routes; the decoded document + grades the retained aliases, counts, and authoritative presence booleans.""" + run_dir = _seed_run(project.project) + commit_value = "0f" * 20 + filename = "AcmePayrollExport.py" + sentinel_path = f"{HOME_PATH}/stories/{SPEC_NAME}" + sweep_path = "AcmeBundleIntent" + worktree_path = f"{HOME_PATH}/worktrees/AcmePrivateTree" + patch_value = "AcmePatchLatch" + raw_before = "AcmeRawPresenceBefore" + raw_after = "AcmeRawPresenceAfter" + raw_ts_offset = "AcmePrivateClock" + tolerated_filename = "AcmeToleratedScene.unity" + cleaned_filename = "AcmeCleanedPrefab.prefab" + refused_filename = "AcmeRefusedAsset.asset" + journal = Journal(run_dir) + journal.append("stale-restore-commits", commits=[commit_value]) + journal.append("stale-restore-excluded", files=[filename]) + journal.append("sentinel-cleared", sentinel=sentinel_path) + journal.append("sweep-intent-regenerated", path=sweep_path) + journal.append("worktree-opened", path=worktree_path, ts_offset=raw_ts_offset) + journal.append("merge-target-tolerated", paths=[tolerated_filename]) + journal.append("merge-target-cleaned", paths=[cleaned_filename]) + journal.append("merge-preflight-refused", tolerated=[refused_filename]) + journal.append( + "attempt-restored", + case="before", + **{"patch_present": raw_before, "patch": patch_value}, + ) + journal.append( + "attempt-restored", + case="after", + **{"patch": patch_value, "patch_present": raw_after}, + ) + + pseudo = sanitize.Pseudonymizer(salt=b"fixed") + diag = diagnostics.collect([run_dir], pseudo=pseudo, project=project.project) + markdown = diagnostics.render_markdown(diag, pseudo=pseudo) + json_text = diagnostics.render_json(diag, pseudo=pseudo) + document = json.loads(json_text) + entries = document["runs"][0]["journal"]["entries"] + + commits = next(e for e in entries if e["kind"] == "stale-restore-commits") + excluded = next(e for e in entries if e["kind"] == "stale-restore-excluded") + sentinel = next(e for e in entries if e["kind"] == "sentinel-cleared") + sweep = next(e for e in entries if e["kind"] == "sweep-intent-regenerated") + worktree = next(e for e in entries if e["kind"] == "worktree-opened") + merge_tolerated = next(e for e in entries if e["kind"] == "merge-target-tolerated") + merge_cleaned = next(e for e in entries if e["kind"] == "merge-target-cleaned") + merge_refused = next(e for e in entries if e["kind"] == "merge-preflight-refused") + collisions = [e for e in entries if e["kind"] == "attempt-restored"] + + assert commits["commits"][0].startswith("commit-") + assert excluded["files_count"] == 1 and "files" not in excluded + assert sentinel["sentinel"].startswith("spec-") + assert sweep["path_present"] is True and "path" not in sweep + assert worktree["path_present"] is True and "path" not in worktree + assert isinstance(worktree["ts_offset"], (int, float)) + assert merge_tolerated["paths_count"] == 1 and "paths" not in merge_tolerated + assert merge_cleaned["paths_count"] == 1 and "paths" not in merge_cleaned + assert merge_refused["tolerated_count"] == 1 and "tolerated" not in merge_refused + assert {entry["case"] for entry in collisions} == {"before", "after"} + assert all(entry["patch_present"] is True for entry in collisions) + + rendered = markdown + json_text + for canary in ( + commit_value, + filename, + sentinel_path, + sweep_path, + worktree_path, + patch_value, + raw_before, + raw_after, + raw_ts_offset, + tolerated_filename, + cleaned_filename, + refused_filename, + ): + assert canary not in rendered, f"LEAK: {canary!r}" + legend_values = set(pseudo.legend().values()) + for canary in ( + filename, + sentinel_path, + sweep_path, + worktree_path, + patch_value, + tolerated_filename, + cleaned_filename, + refused_filename, + ): + assert canary not in legend_values, f"LEAK via legend: {canary!r}" + + def test_target_field_routes_by_kind_because_it_carries_two_kinds_of_value(): """`target` is a BRANCH on the merge kinds and a sprint STATUS on `board-advance-*`. diff --git a/tests/test_portability_guard.py b/tests/test_portability_guard.py index eb3745a0..fd360b58 100644 --- a/tests/test_portability_guard.py +++ b/tests/test_portability_guard.py @@ -242,14 +242,22 @@ ) # ``kind -> the field names routed on THAT kind only``, read off the same module so -# the guard still cannot drift from it. A name here is routed on its own kinds and -# unrouted everywhere else, which is the distinction the flattened union destroyed. -JOURNAL_KIND_ROUTED_FIELDS = { - kind: frozenset(row) for kind, row in diagnostics._JOURNAL_KIND_ALIAS_FIELDS.items() -} +# the guard still cannot drift from it. Alias, identifier-list, and count-list rules +# share this inventory because all three claim the same `(kind, field)` boundary. +JOURNAL_KIND_ROUTING_TABLES = ( + diagnostics._JOURNAL_KIND_ALIAS_FIELDS, + diagnostics._JOURNAL_KIND_KEYLIST_FIELDS, + diagnostics._JOURNAL_KIND_COUNTLIST_FIELDS, +) +JOURNAL_KIND_ROUTED_FIELDS: dict[str, frozenset[str]] = {} +for _routing_table in JOURNAL_KIND_ROUTING_TABLES: + for _kind, _row in _routing_table.items(): + JOURNAL_KIND_ROUTED_FIELDS[_kind] = JOURNAL_KIND_ROUTED_FIELDS.get( + _kind, frozenset() + ) | frozenset(_row) # ``kind -> field names declared benign on that kind alone`` — the kind-scoped twin of -# ``JOURNAL_BENIGN_FIELDS``, and it exists for the same field the routing table does. +# ``JOURNAL_BENIGN_FIELDS`` for overloaded names whose other shapes are routed. # ``engine``'s board-advance carry paths journal ``target`` carrying a sprint STATUS # ("done"), not a branch; ``diagnostics``' ``_JOURNAL_KIND_ALIAS_FIELDS`` comment is # explicit that aliasing those would destroy the field a maintainer reads the record @@ -261,6 +269,9 @@ "board-advance-carry-failed": frozenset({"target"}), "board-advance-carry-foreign-dirt": frozenset({"target"}), "board-advance-carry-uncommitted": frozenset({"target"}), + # The stale-restore record carries SHA strings under this name and is routed; + # this recovery notice carries only the already-derived integer count. + "rollback-manual-required": frozenset({"commits"}), } # Every OTHER field name journalled today: a declared inventory, not a per-name @@ -306,7 +317,6 @@ "checkpoint", "code_root_changed", "command_index", - "commits", "condition", "contradiction", "converted", @@ -328,7 +338,6 @@ "expired_clock", "failed", "field", - "files", "finished", "fired_at", "flat_remainder", @@ -369,8 +378,6 @@ "open_now", "original", "owed_after_implement", - "path", - "paths", "phase", "platform", "plugin", @@ -408,7 +415,6 @@ "run_type", "security_config_changed", "seen_again", - "sentinel", "sentinel_kind", "session_status", "session_vanished", @@ -431,7 +437,6 @@ "to", "tokens", "tokens_weighted", - "tolerated", "total", "trigger", "verification_sequence", @@ -2281,11 +2286,10 @@ def _journal_field_offenders(findings) -> list[tuple[str, int, str, str]]: declared itself a hole. Routing is checked BY NAME first and then BY KIND, mirroring ``_scrub_entry``'s - own order rather than a flattened union of the two. A kind-scoped name — today - only ``target`` — is routed on its own kinds, declared benign on the - ``board-advance-*`` family that carries a sprint status under the same name, and - an offender everywhere else, INCLUDING at a call whose kind the scan could not - resolve. That is the case a by-name union got wrong in the dangerous direction: + own order rather than a flattened union of the two. A kind-scoped name is routed + only on its declared shapes and is an offender everywhere else unless that other + shape is explicitly benign. That includes a call whose kind the scan could not + resolve. `target` is the dangerous example: flattening it made ``journal.append("unit-merge-failed", target=branch)`` read as routed.""" offenders: list[tuple[str, int, str, str]] = [] for _, rel, ln, txt, (field, fn, kind) in findings: @@ -4148,19 +4152,25 @@ def test_journal_routing_tables_are_read_from_diagnostics(): diagnostics._JOURNAL_KEYLIST_FIELDS, ): assert set(table) <= JOURNAL_ROUTED_FIELDS - assert JOURNAL_KIND_ROUTED_FIELDS == { - kind: frozenset(row) for kind, row in diagnostics._JOURNAL_KIND_ALIAS_FIELDS.items() - } + expected_kind_routing: dict[str, frozenset[str]] = {} + for table in JOURNAL_KIND_ROUTING_TABLES: + for kind, row in table.items(): + expected_kind_routing[kind] = expected_kind_routing.get(kind, frozenset()) | frozenset( + row + ) + assert JOURNAL_KIND_ROUTED_FIELDS == expected_kind_routing # …and the kind-scoped names are deliberately NOT in the by-name union. This is # the assertion that would have caught the flattening: `target` routed by name # says the board-advance family is covered when `_scrub_entry` does not cover it. - for row in diagnostics._JOURNAL_KIND_ALIAS_FIELDS.values(): - assert not set(row) & JOURNAL_ROUTED_FIELDS, ( - "a kind-scoped field name leaked into the by-name routed union; " - "`_scrub_entry` consults `_JOURNAL_KIND_ALIAS_FIELDS` per kind, so a " - "by-name claim about it is false on every other kind" - ) - # `_JOURNAL_KIND_SCHEMAS` is the FOURTH table `_scrub_entry` consults, and it was + for table in JOURNAL_KIND_ROUTING_TABLES: + for row in table.values(): + assert not set(row) & JOURNAL_ROUTED_FIELDS, ( + "a kind-scoped field name leaked into the by-name routed union; " + "`_scrub_entry` consults its kind tables per kind, so a by-name " + "claim about it is false on every other kind" + ) + # `_JOURNAL_KIND_SCHEMAS` is the fail-closed schema table `_scrub_entry` consults, + # and it was # coupled to this guard by prose alone: deleting its `preference-escalation` row # left every assertion here green while the fail-closed arm stopped running and # `customer="AcmeVault"` went back to shipping verbatim (measured). Read it here From 72d3e865eed0bf68b6fa50da9ad0d80065a028fa Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 16:23:10 -0700 Subject: [PATCH 07/18] sweep dw4-session-artifact-json-hardening: DW-86, DW-89 via bmad-loop --- CHANGELOG.md | 3 ++ src/bmad_loop/adapters/generic.py | 19 +++++-- src/bmad_loop/escalation.py | 13 +++-- src/bmad_loop/resolve.py | 50 ++++++++++++++---- tests/test_escalation.py | 23 +++++++++ tests/test_generic_tmux.py | 86 +++++++++++++++++++++++++++++++ tests/test_resolve.py | 77 +++++++++++++++++++++++---- 7 files changed, 243 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df50b4ed..959993e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -270,6 +270,9 @@ breaking changes may land in a minor release. unrecoverably, on a gitignored ledger. The mark and append legs now re-anchor only when the preimage they wrote over is still the bytes the run last claimed; the restore skips and journals `ledger-restore-skipped-diverged`. +- Reject malformed session escalation/result artifacts and non-finite resolve JSON + (DW-86, DW-89). + - **Confine built-in adapter task directories** (DW-74), refusing unsafe task ids and symlink- or junction-redirected task directories before prompt, artifact, log, or transport side effects. diff --git a/src/bmad_loop/adapters/generic.py b/src/bmad_loop/adapters/generic.py index fa226e41..04fa4958 100644 --- a/src/bmad_loop/adapters/generic.py +++ b/src/bmad_loop/adapters/generic.py @@ -21,6 +21,7 @@ from __future__ import annotations +import copy import enum import hashlib import json @@ -420,13 +421,23 @@ def _write_heartbeat(self, task_id: str, payload: dict) -> None: def _read_result(self, task_id: str) -> dict | None: path = self._result_path(task_id) - if not path.is_file(): - return None try: + if not path.is_file(): + return None data = json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): + if not isinstance(data, dict): + return None + # Plugin HookContext makes this same defensive copy before exposing + # result data, so reject a shape that would recurse there while the + # artifact is still inside the shared observation boundary. + copy.deepcopy(data) + # JSON accepts escaped lone surrogates, but the default ATTENTION + # sink writes reasons as UTF-8. Validate every parsed string without + # imposing stricter numeric semantics on completed session results. + json.dumps(data, ensure_ascii=False).encode("utf-8") + except (OSError, ValueError, RecursionError): return None - return data if isinstance(data, dict) else None + return data def _await_result(self, task_id: str, grace_s: float = RESULT_GRACE_S) -> dict | None: deadline = time.monotonic() + grace_s diff --git a/src/bmad_loop/escalation.py b/src/bmad_loop/escalation.py index 5baaeb9b..cc9524fa 100644 --- a/src/bmad_loop/escalation.py +++ b/src/bmad_loop/escalation.py @@ -44,22 +44,25 @@ class Decision: reason: str = "" -def critical_escalations(result_json: dict[str, Any] | None) -> list[dict[str, Any]]: +def _escalation_list(result_json: dict[str, Any] | None) -> list[Any]: if not result_json: return [] + escalations = result_json.get("escalations", []) + return escalations if isinstance(escalations, list) else [] + + +def critical_escalations(result_json: dict[str, Any] | None) -> list[dict[str, Any]]: return [ e - for e in result_json.get("escalations", []) + for e in _escalation_list(result_json) if isinstance(e, dict) and str(e.get("severity", "")).upper() == SEVERITY_CRITICAL ] def preference_escalations(result_json: dict[str, Any] | None) -> list[dict[str, Any]]: - if not result_json: - return [] return [ e - for e in result_json.get("escalations", []) + for e in _escalation_list(result_json) if isinstance(e, dict) and str(e.get("severity", "")).upper() != SEVERITY_CRITICAL ] diff --git a/src/bmad_loop/resolve.py b/src/bmad_loop/resolve.py index ce50c4c4..bb90a4bf 100644 --- a/src/bmad_loop/resolve.py +++ b/src/bmad_loop/resolve.py @@ -15,6 +15,7 @@ from __future__ import annotations import json +import math import os import subprocess from pathlib import Path @@ -39,6 +40,17 @@ RESOLVE_DIR = "resolve" +def _reject_json_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant: {value}") + + +def _parse_finite_json_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError(f"non-finite JSON float: {value}") + return parsed + + def _story_dir(run_dir: Path, story_key: str) -> Path: return run_dir / RESOLVE_DIR / safe_segment(story_key) @@ -142,7 +154,7 @@ def _gather_escalations( global across the pass, not per directory; it removes only exact repeats, so a directory holding CRITICAL A in one file and A + B in the other still yields both. - * the ``except`` tuple and the ``list`` check — ``build_context`` is an + * the ``except`` tuple and shared list guard — ``build_context`` is an OBSERVATION path: a malformed artifact must cost its own contents and nothing more, never raise out to the interactive resolve command. ``UnicodeDecodeError`` is a ``ValueError``, not an ``OSError`` (the same @@ -150,11 +162,13 @@ def _gather_escalations( also raise a plain ``ValueError`` when an integer exceeds Python's configured digit limit. Deeply nested input can raise ``RecursionError`` while either parsing the document or canonicalizing an entry, so both operations live - under the same artifact-level guard. Meanwhile, - ``critical_escalations`` iterates ``escalations`` with no list guard of its - own, so a ``{"escalations": null}`` artifact would raise ``TypeError`` - here. The guard belongs in this caller; the shared predicate stays the - single definition of CRITICAL. + under the same artifact-level guard. ``critical_escalations`` owns the + list-only shape guard shared by every control-loop caller, so malformed + ``escalations`` values contribute nothing here just as they do elsewhere. + This reader re-checks the shape only AFTER that selector has run, and not + to avoid a crash: the recheck decides whether the artifact counts as READ, + so a wrong-shaped one reaches ``skipped`` instead of passing for an empty + one — see that parameter's note below. * the ``stat`` classification and its ``S_ISREG`` check — deciding ABSENT from UNREADABLE cannot go through ``Path.is_file()``, whose error behavior splits by interpreter. Through 3.13 it re-raises anything outside @@ -240,7 +254,11 @@ def _gather_escalations( if not S_ISREG(st.st_mode): continue try: - doc = json.loads(fpath.read_text(encoding="utf-8")) + doc = json.loads( + fpath.read_text(encoding="utf-8"), + parse_constant=_reject_json_constant, + parse_float=_parse_finite_json_float, + ) if not isinstance(doc, dict): raise ValueError("artifact is not a JSON object") if "escalations" not in doc: @@ -249,11 +267,23 @@ def _gather_escalations( # skip — counting it as one would withhold coverage from every # resolve cycle, permanently. continue - if not isinstance(doc["escalations"], list): - raise ValueError("'escalations' is not a list") artifact_entries: dict[str, dict[str, Any]] = {} for esc in critical_escalations(doc): artifact_entries.setdefault(json.dumps(esc, sort_keys=True), esc) + if not isinstance(doc["escalations"], list): + # Deliberately AFTER the selector, and no longer a crash guard: + # ``escalation._escalation_list`` absorbs a non-list at the shared + # predicate, which is the single definition of what CONTRIBUTES and + # must run for this artifact too, so this reader keeps no private + # shape guard that could drift from the engine and sweep callers. + # What the predicate cannot answer is whether the artifact was + # READ, and that is this walk's own question: an ``escalations`` key + # holding the wrong shape is, from here, indistinguishable from a + # file truncated mid-write. Raise so it reaches ``skipped`` by the + # same ``except`` the other read faults use, and the watermark + # cannot launder it into durable coverage. Nothing is discarded — + # the selector already answered a non-list with no entries. + raise ValueError("'escalations' is not a list") except (OSError, ValueError, RecursionError): if skipped is not None and target is found: skipped.add(str(fpath)) @@ -420,7 +450,7 @@ def build_context( context["stories"] = stories_ctx path = context_path(run_dir, story_key) path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(context, indent=2), encoding="utf-8") + path.write_text(json.dumps(context, indent=2, allow_nan=False), encoding="utf-8") return path, withheld, len(unreadable) diff --git a/tests/test_escalation.py b/tests/test_escalation.py index bb62cc7b..d79b4fb0 100644 --- a/tests/test_escalation.py +++ b/tests/test_escalation.py @@ -4,8 +4,10 @@ from bmad_loop.adapters.base import SessionResult from bmad_loop.escalation import ( Action, + critical_escalations, decide_dev, decide_review_session, + preference_escalations, review_retry_or_exhaust, ) from bmad_loop.model import StoryTask @@ -20,6 +22,27 @@ FAILING = VerifyOutcome.retry("spec status is 'in-progress', expected 'done'") +def test_escalation_selectors_preserve_valid_list_semantics(): + critical = {"severity": "critical", "detail": "stop"} + preferences = [ + {}, + {"severity": "PREFERENCE", "detail": "explicit"}, + {"detail": "implicit"}, + {"severity": 1, "detail": "non-critical"}, + ] + result = {"escalations": [None, "junk", critical, *preferences]} + + assert critical_escalations(result) == [critical] + assert preference_escalations(result) == preferences + + +def test_escalation_selectors_reject_every_non_list_shape(): + for value in (None, 1, "escalation", {"severity": "CRITICAL"}, ("tuple",)): + result = {"escalations": value} + assert critical_escalations(result) == [] + assert preference_escalations(result) == [] + + def _task(**kw) -> StoryTask: return StoryTask(story_key="9-0-x", epic=9, **kw) diff --git a/tests/test_generic_tmux.py b/tests/test_generic_tmux.py index dc95261b..dbca0488 100644 --- a/tests/test_generic_tmux.py +++ b/tests/test_generic_tmux.py @@ -7,6 +7,7 @@ propagation / hook-signal waiting / kill end-to-end for any profile. """ +import copy import dataclasses import hashlib import json @@ -198,6 +199,91 @@ def test_read_result_variants(tmp_path): assert adapter._read_result("t1") is None # malformed (task_dir / "result.json").write_text('["not a dict"]') assert adapter._read_result("t1") is None # wrong shape + (task_dir / "result.json").write_bytes(_BAD_UTF8) + assert adapter._read_result("t1") is None # invalid UTF-8 + (task_dir / "result.json").write_text('{"clean": true}') + assert adapter._read_result("t1") == {"clean": True} # valid rewrite + + +def test_read_result_degrades_a_plain_decoder_value_error(tmp_path, monkeypatch): + adapter = make_adapter(tmp_path) + task_dir = adapter.tasks_dir / "t1" + task_dir.mkdir(parents=True) + marker = '{"value":"decoder-value-error"}' + (task_dir / "result.json").write_text(marker) + real_loads = json.loads + + def loads_with_value_error(data, *args, **kwargs): + if data == marker: + raise ValueError("synthetic decoder value error") + return real_loads(data, *args, **kwargs) + + with monkeypatch.context() as mp: + mp.setattr(generic.json, "loads", loads_with_value_error) + assert adapter._read_result("t1") is None + + valid_unicode = {"escalations": [{"severity": "PREFERENCE", "detail": "café 🚀"}]} + (task_dir / "result.json").write_text( + json.dumps(valid_unicode, ensure_ascii=False), encoding="utf-8" + ) + assert adapter._read_result("t1") == valid_unicode + + +def test_read_result_degrades_a_decoder_recursion_error(tmp_path): + adapter = make_adapter(tmp_path) + task_dir = adapter.tasks_dir / "t1" + task_dir.mkdir(parents=True) + depth = sys.getrecursionlimit() * 20 + nested = '{"value":' + ("[" * depth) + "0" + ("]" * depth) + "}" + with pytest.raises(RecursionError): + json.loads(nested) + + (task_dir / "result.json").write_text(nested) + + assert adapter._read_result("t1") is None + + +def test_read_result_degrades_an_unreadable_existence_probe(tmp_path, monkeypatch): + adapter = make_adapter(tmp_path) + path = adapter._result_path("t1") + real_is_file = Path.is_file + + def is_file_with_permission_error(candidate): + if candidate == path: + raise PermissionError("task directory is not searchable") + return real_is_file(candidate) + + monkeypatch.setattr(Path, "is_file", is_file_with_permission_error) + + assert adapter._read_result("t1") is None + + +def test_read_result_rejects_data_that_plugin_deepcopy_cannot_handle(tmp_path): + adapter = make_adapter(tmp_path) + task_dir = adapter.tasks_dir / "t1" + task_dir.mkdir(parents=True) + depth = sys.getrecursionlimit() // 2 + nested = '{"value":' + ("[" * depth) + "0" + ("]" * depth) + "}" + parsed = json.loads(nested) + with pytest.raises(RecursionError): + copy.deepcopy(parsed) + (task_dir / "result.json").write_text(nested) + + assert adapter._read_result("t1") is None + + +def test_read_result_rejects_lone_surrogates_and_recovers_after_rewrite(tmp_path): + adapter = make_adapter(tmp_path) + task_dir = adapter.tasks_dir / "t1" + task_dir.mkdir(parents=True) + escaped_surrogate = '{"escalations":[{"severity":"CRITICAL","detail":"\\ud800"}]}' + parsed = json.loads(escaped_surrogate) + with pytest.raises(UnicodeEncodeError): + json.dumps(parsed, ensure_ascii=False).encode("utf-8") + (task_dir / "result.json").write_text(escaped_surrogate) + + assert adapter._read_result("t1") is None + (task_dir / "result.json").write_text('{"clean": true}') assert adapter._read_result("t1") == {"clean": True} diff --git a/tests/test_resolve.py b/tests/test_resolve.py index c40e31bd..dbe56916 100644 --- a/tests/test_resolve.py +++ b/tests/test_resolve.py @@ -2759,6 +2759,53 @@ def test_gather_escalations_skips_a_non_utf8_artifact(tmp_path): assert [e["detail"] for e in ctx["escalations"]] == ["still readable"] +def test_gather_escalations_skips_an_unreadable_existence_probe(tmp_path, monkeypatch): + run_dir, state, task = _escalated_run(tmp_path) + task_dir = _task_dir(run_dir, task) + unreadable = task_dir / "result.json" + sibling = {"severity": "CRITICAL", "detail": "sibling survives"} + (task_dir / "escalation.json").write_text( + json.dumps({"escalations": [sibling]}), encoding="utf-8" + ) + real_is_file = Path.is_file + + def is_file_with_permission_error(candidate): + if candidate == unreadable: + raise PermissionError("task directory is not searchable") + return real_is_file(candidate) + + with monkeypatch.context() as mp: + mp.setattr(Path, "is_file", is_file_with_permission_error) + path = _context(state, run_dir, "6-4-cli-list-command", isolation="") + + ctx = json.loads(path.read_text(encoding="utf-8")) + assert ctx["escalations"] == [sibling] + + +@pytest.mark.parametrize("constant", ["NaN", "Infinity", "-Infinity", "1e999", "-1e999"]) +def test_gather_escalations_skips_a_nonfinite_artifact(tmp_path, constant): + """A numeric spelling decoded as non-finite poisons only its own artifact; a valid + sibling still reaches context, whose output is accepted by a strict parser.""" + run_dir, state, task = _escalated_run(tmp_path) + task_dir = _task_dir(run_dir, task) + (task_dir / "result.json").write_text( + '{"escalations":[{"severity":"CRITICAL","detail":' + constant + "}]}", + encoding="utf-8", + ) + sibling = {"severity": "CRITICAL", "detail": "sibling survives", "score": 0.5} + (task_dir / "escalation.json").write_text( + json.dumps({"escalations": [sibling]}), encoding="utf-8" + ) + + path = _context(state, run_dir, "6-4-cli-list-command", isolation="") + + def reject_constant(value): + raise ValueError(f"non-finite JSON constant: {value}") + + ctx = json.loads(path.read_text(encoding="utf-8"), parse_constant=reject_constant) + assert ctx["escalations"] == [sibling] + + def test_gather_escalations_skips_a_plain_json_value_error(tmp_path, monkeypatch): """`json.loads` raises plain ValueError, not JSONDecodeError, when an integer exceeds Python's configured digit limit. That malformed file costs only its @@ -2841,14 +2888,11 @@ def dumps_with_recursion_error(value, *args, **kwargs): @pytest.mark.parametrize("bad", [None, 1, "x", {}]) def test_gather_escalations_skips_a_non_list_escalations_field(tmp_path, monkeypatch, bad): - """DW-70/73's other half. `escalation.critical_escalations` iterates - `escalations` with no list guard of its own, so `{"escalations": null}` raised - `TypeError` straight out of `build_context`. The guard sits in this caller; the - shared predicate stays the single definition of CRITICAL. - - Every parameter must fail when the list guard is ablated. ``None`` and ``1`` - raise without it; the call trace below distinguishes the iterable ``"x"`` and - ``{}`` shapes, which the shared filter would otherwise accept as empty.""" + """The shared selector owns the list guard, including for resolve artifacts. + + Every parameter fails when that shared guard is ablated. The call trace also + proves this reader delegates malformed shapes instead of retaining a private + guard that could drift from engine and sweep behavior.""" run_dir, state, task = _escalated_run(tmp_path) task_dir = _task_dir(run_dir, task) (task_dir / "result.json").write_text(json.dumps({"escalations": bad}), encoding="utf-8") @@ -2870,15 +2914,30 @@ def recording_critical_escalations(doc): ctx = json.loads(path.read_text(encoding="utf-8")) assert filtered == [ + {"escalations": bad}, { "escalations": [ {"severity": "CRITICAL", "detail": "sibling survives"}, ] - } + }, ] assert [e["detail"] for e in ctx["escalations"]] == ["sibling survives"] +@pytest.mark.parametrize( + "nonfinite", [float("nan"), float("inf"), float("-inf")], ids=["nan", "inf", "-inf"] +) +def test_build_context_refuses_nonfinite_in_memory_values(tmp_path, nonfinite): + run_dir, state, _task = _escalated_run(tmp_path) + state.paused_reason = nonfinite + path = resolve.context_path(run_dir, "6-4-cli-list-command") + + with pytest.raises(ValueError, match="Out of range float values"): + resolve.build_context(state, run_dir, "6-4-cli-list-command", isolation="") + + assert not path.exists() + + def test_gather_escalations_preference_only_yields_nothing(tmp_path): """The CRITICAL-only filter is unchanged by the de-duplication rewrite: a directory carrying only non-CRITICAL entries contributes nothing, and mirroring From fb1f1e15d2353d664afa80dd71c7f12cd35b16eb Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 16:32:59 -0700 Subject: [PATCH 08/18] sweep dw4-resolve-context-contract-docs: DW-87 via bmad-loop --- .../data/skills/bmad-loop-resolve/SKILL.md | 3 +++ tests/test_resolve_skill_contract.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/bmad_loop/data/skills/bmad-loop-resolve/SKILL.md b/src/bmad_loop/data/skills/bmad-loop-resolve/SKILL.md index 095b4d50..188237ad 100644 --- a/src/bmad_loop/data/skills/bmad-loop-resolve/SKILL.md +++ b/src/bmad_loop/data/skills/bmad-loop-resolve/SKILL.md @@ -49,6 +49,9 @@ These environment variables are set: } ``` +The `escalations` array is ordered newest-first. +Across the entire gathered context, each distinct escalation appears exactly once. + The interactive session's working directory is always `project_root`. That tree holds the BMAD artifacts and specs you inspect or clarify. `code_root` is the tree where the run's code and git work belong; it may be different. When the roots differ, do not diff --git a/tests/test_resolve_skill_contract.py b/tests/test_resolve_skill_contract.py index c17b0f82..918dad5b 100644 --- a/tests/test_resolve_skill_contract.py +++ b/tests/test_resolve_skill_contract.py @@ -84,6 +84,23 @@ def test_every_emitted_context_key_is_documented(skill_md): ) +def test_skill_documents_escalation_ordering_and_global_uniqueness(skill_md): + """The context consumer can rely on the reader's cross-session guarantees. + + These assertions are deliberately separate so removing either the ordering + promise or the global de-duplication promise fails the contract guard. + """ + after_schema = skill_md.split("}\n```\n\n", maxsplit=1)[1] + contract_lines = after_schema.split("\n\n", maxsplit=1)[0].splitlines() + + assert "The `escalations` array is ordered newest-first." in contract_lines + assert ( + "Across the entire gathered context, each distinct escalation appears exactly once." + in contract_lines + ) + assert len(contract_lines) == 2 + + def test_skill_routes_project_artifacts_and_code_work_to_their_distinct_roots(skill_md): """Mentioning both keys is inert unless the skill explains the operational split. From 554d39781c9f5ed83800be55f180e7aadca6864d Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 17:05:05 -0700 Subject: [PATCH 09/18] sweep dw4-decision-dw-91: DW-91 via bmad-loop --- CHANGELOG.md | 3 ++ docs/FEATURES.md | 2 +- .../data/skills/bmad-loop-resolve/SKILL.md | 22 +++++++-- tests/test_resolve_skill_contract.py | 45 +++++++++++++++++++ 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 959993e9..ebc911f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -270,6 +270,9 @@ breaking changes may land in a minor release. unrecoverably, on a gitignored ledger. The mark and append legs now re-anchor only when the preimage they wrote over is still the bytes the run last claimed; the restore skips and journals `ledger-restore-skipped-diverged`. +- Let interactive resolve present `paused_reason` when watermark filtering leaves no newer + recorded escalation detail, without recovering or inventing an escalation (DW-91). + - Reject malformed session escalation/result artifacts and non-finite resolve JSON (DW-86, DW-89). diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 51ccfe83..947bb044 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -69,7 +69,7 @@ See [README.md](../README.md) for the narrative overview and [setup-guide.md](se - Completing a park: `bmad-loop confirm ` walks the outstanding actions one at a time, writes the spec's `## Operator Confirmation` audit section, advances spec and board to `done`, and commits the pair together with the park record's deletion. Nothing is re-driven — the agent-doable work was committed at park time; `--reverify` re-runs your `[verify]` commands first and a failure blocks the confirmation. Each park is a **committed per-story file** under `.bmad-loop/operator/`, written inside the story's commit window so it rides the park's own commit through the merge-back to every clone — a teammate, a fresh clone or CI can confirm a story parked elsewhere (#356). `validate` warns on drift in every direction (`operator.registry-stale`, `operator.actions-malformed`, `operator.park-record-missing`), and `confirm` refuses a drifted record. A park written before #356 lives in the machine-local `.bmad-loop/operator-actions.json`, which `confirm` still reads and prunes but nothing writes anymore — so an in-flight park from an older version stays confirmable on the machine that wrote it. - A confirmation is resumable. Every write is checked — the spec is read back from disk, so a story is never declared done over a write that did not land — and the park record is dropped last, so a failure part-way leaves the story findable. Interrupted between the spec writes and the board write, what survives is a signed-off spec at `done` with the entry still pointing at it; re-running `confirm` **finishes** that rather than refusing it as stale, with no second prompt and no second audit section (the section on disk _is_ the acknowledgment, and the check is fence-aware). It resumes equally from a board a human fixed by hand, which is what the failure message asks for — advancing an already-`done` board is idempotent. `--list`, `--json` (`resumable`, `confirmation_recorded`) and `validate` (`operator.confirm-interrupted`) name that state rather than calling it stale. - Dispatched sessions are told the sprint board is orchestrator-owned (#437) — the sibling of the park contract above, injected into the prompt the same way. The board advances as soon as dev verifies, but the story's single commit lands only after the review loop, so a session dispatched in between opens on an uncommitted, unattributed change to `sprint-status.yaml` with nothing in the repo naming its author (one read it as a spec violation, reverted it, and tripped the sign-off-regression gate on a story both sessions agreed was finished). Story dev prompts and the review prompts of sprint and sweep runs carry the same prohibition: never write the board, never revert it, and a row at `done` or `awaiting-operator` is the orchestrator's own bookkeeping — not a defect to fix, and not proof that the work is verified, deliberately, since the row is written _before_ the deterministic dev verification runs and a repair session opens on a red tree under a `done` row. Only the **review** prompt adds where to go instead: a story that cannot be finished without a human decision is finalized to `status: blocked` with a reason — the one hand-back that both withholds the commit and reaches a human, where any other non-terminal status just burns the review budget onto a defer that rolls the work back. A dev prompt gets no such invitation, because `blocked` halts the whole run — the exact failure park exists to avoid — and a dev session that cannot finish already has park. A deferred-work bundle's dev prompt carries nothing (a bundle has no board row) while a bundle's _review_ prompt does, since a sweep runs inside a project whose board exists and is just as revertible; every injected plugin-workflow session carries the prohibition too — `post_dev_phase`, `post_review_result` and `pre_commit_gate` all fire inside that same window — as its own `## Sprint board` section appended _after_ the session-gate hooks, so a plugin prompt rewrite cannot strip it, and without the `blocked` redirect for the same reason a dev prompt has none; stories mode carries none of it, having no board at all. -- Typed escalations: `CRITICAL` pauses the run + notifies (desktop + `ATTENTION` file); `PREFERENCE` is journaled and continues. A story's escalation trail is append-only and deliberately survives a re-arm (it is the run-dir audit a later resolve cycle reads), so a second `bmad-loop resolve` used to re-present every CRITICAL the story ever raised, interleaved with the new ones and with nothing marking which was which — against a resolve skill whose contract is singular. An interactive resolve session that records a `resolution.json` now **watermarks** the trail at its current length, and every later cycle hands the agent only the escalations recorded since; how many earlier ones were withheld is printed to your terminal, never added to the agent's `context.json` (the agent-facing contract is unchanged). The watermark moves only on a gesture that actually accepted a resolution — a resolve session that exited without writing one, `resolve --no-interactive`, and the TUI's Re-arm button all leave it where it stands. Leaving a watermark is not clearing it: a watermark already standing still filters on those paths, which show everything recorded since the last accepted resolution rather than the whole trail. That is where the bias is deliberate, and it is a claim about which GESTURES move the watermark: one that accepted nothing never moves it. Within a cycle that DID accept a resolution the watermark covers everything that cycle PRESENTED — it is stamped at the trail's length, not at the entries individually answered — so answering one of five escalations shown together retires all five. A task's watermark is reported as the `esc-upto` column of `bmad-loop diagnose`'s markdown task table, and as `escalations_resolved_upto` under `--json` (that is the key to grep in a support bundle), which is what explains a short `context.json` on a bug report. +- Typed escalations: `CRITICAL` pauses the run + notifies (desktop + `ATTENTION` file); `PREFERENCE` is journaled and continues. A story's escalation trail is append-only and deliberately survives a re-arm (it is the run-dir audit a later resolve cycle reads), so a second `bmad-loop resolve` used to re-present every CRITICAL the story ever raised, interleaved with the new ones and with nothing marking which was which — against a resolve skill whose contract is singular. An interactive resolve session that records a `resolution.json` now **watermarks** the trail at its current length, and every later cycle hands the agent only the escalations recorded since; how many earlier ones were withheld is printed to your terminal, never added to the agent's `context.json`. When that filtered list contains entries, the resolve skill presents them newest-first under the existing globally de-duplicated contract. When a new pause precedes any newer recorded escalation and the filtered list is empty, the skill presents `paused_reason` as the available current-pause evidence and discloses that no newer recorded detail exists; it does not read below the watermark, recover an older artifact entry, or synthesize an escalation object. The watermark moves only on a gesture that actually accepted a resolution — a resolve session that exited without writing one, `resolve --no-interactive`, and the TUI's Re-arm button all leave it where it stands. Leaving a watermark is not clearing it: a watermark already standing still filters on those paths, which show everything recorded since the last accepted resolution rather than the whole trail. That is where the bias is deliberate, and it is a claim about which GESTURES move the watermark: one that accepted nothing never moves it. Within a cycle that DID accept a resolution the watermark covers everything that cycle PRESENTED — it is stamped at the trail's length, not at the entries individually answered — so answering one of five escalations shown together retires all five. A task's watermark is reported as the `esc-upto` column of `bmad-loop diagnose`'s markdown task table, and as `escalations_resolved_upto` under `--json` (that is the key to grep in a support bundle), which is what explains a short or empty `context.json` escalation list on a bug report. - A rejected dev attempt notifies too, with its reason (#640). RETRY was the only dev outcome that rejected an attempt silently, and it is the one that discards a completed implementation — the non-fixable leg resets the tree to baseline. The notice fires once per rejected attempt in an uninterrupted run (so ordinarily at most `max_dev_attempts` per story) and has no suppression knob of its own; it follows `[notify]` like every other notice. One attempt can raise it twice: the notice precedes the rollback, so a host that dies in between replays that verdict on resume and announces it again — treat the count as a floor on attempts rejected, not an exact tally. The reason is reduced to its first line and capped, with a `[…]` marker when it was trimmed, because a `Decision.reason` routinely carries a verify-output tail that would otherwise spill into `ATTENTION` and a desktop bubble; the untruncated reason stays in the `dev-decision` journal entry. It fires above the fixable/non-fixable split, so on a leg that goes on to pause for manual recovery the operator sees both notices. - Environment faults pause without burning budget (#194): a session whose coding CLI never reached the API — a verify command whose _environment_ is broken (`sh` reports rc `126`/`127`; on Windows a missing tool is caught by its `is not recognized` message or by resolving the command's leading token, and a command naming a file `cmd` cannot execute — a `.sh`, or any extension outside `PATHEXT`, which cmd hands to the file association and which exits `0` without running anything — is a fault rather than a silent rc `0` pass, #302; and on either OS a verify command whose child could not be started at all — most often because the directory it was to run in is missing, is a file, or cannot be searched, but any spawn-time `OSError` counts — is translated into the same fault instead of crashing the run, since no exit code exists to classify) **or** a session whose log matches the profile's `env_fault_patterns` (an `API Error … Connection refused`-class transport failure, or a provider quota/usage-limit refusal, that idled out the session clock) — pauses the run with the matched evidence instead of charging the attempt and deferring the story as if its code were broken. Re-arm restores the budget. Patterns are per-profile: `claude` seeds three, reproducing only complete error sentences its CLI was captured printing (connection loss, and the two captured provider 5xx refusals — statuses enumerated, never ranged, so an uncaptured `503` stays prose), so a story that merely writes _about_ a provider error cannot trip them (#507); `opencode` seeds a provider quota/rate-limit and connection pair (#323), matched against the `opencode serve` process's own stdout, which the model cannot write to; the other four profiles ship none. Each adapter matches them against the log named by its `ENV_FAULT_LOG_SUFFIX` — the tmux pane capture `logs/.log`, or `.server.out` (the `opencode serve` process's own stdout) for `opencode-http`, never that adapter's model-written transcript. A pattern is only sound against a log the model cannot write to; where that does not hold — the pane capture — the pattern has to reproduce a whole captured sentence, because an error token plus a cause on the same line is precisely the shape a story writing about the error emits, and that framing is what the guard now refuses (#507). A usage-limit / quota cause stays unseeded on the pane-capture profiles for the same evidentiary reason: no captured line exists for them (#323). Extend or disable them in a project profile overlay. - A session the multiplexer lost says so (#489). Sessions complete on a hook `Stop` or on window death, and a window is gone whether the CLI exited or something destroyed the whole mux session out from under the run — an external reaper, a concurrent prune or `bmad-loop stop`, an operator `kill-session`, a server crash, the host sleeping. Both are `crashed`, so the retry/defer reason an operator reads said only `dev session crashed` — pointing at the agent when the host was at fault. The crash verdict now asks whether the _session_ still exists and, when it does not, says so in the reason (`… session crashed: the multiplexer no longer reports the session, so the window's disappearance is not evidence the CLI exited`), as `session_vanished` on `dev-decision` and `fix-decision` either way, beside the routing each fed, on every role's `session-end` journal entry when it is true (the convention `env_fault` already uses there), and as a `session-vanished` breadcrumb in `session-lifecycle.jsonl`. The repair path carries it the same way: when fix attempts are exhausted the defer names the lost session instead of blaming the tree for repairs that never ran. The wording states what the evidence _withdraws_, not what it proves: `has_session` maps every nonzero backend result to False, so a negative lookup is "the backend did not confirm it" rather than proof the session is gone — enough to stop an operator reading window death as a CLI exit, not enough to name a destroyer. It composes with an environment-fault pause instead of being swallowed by it. A session reaped _after_ flushing its result still scores `completed` and is not diagnosed — it produced something. Diagnosis only — the routing is unchanged, and a retry re-creates the session. diff --git a/src/bmad_loop/data/skills/bmad-loop-resolve/SKILL.md b/src/bmad_loop/data/skills/bmad-loop-resolve/SKILL.md index 188237ad..407864f9 100644 --- a/src/bmad_loop/data/skills/bmad-loop-resolve/SKILL.md +++ b/src/bmad_loop/data/skills/bmad-loop-resolve/SKILL.md @@ -152,10 +152,24 @@ case below — omit it entirely for an ordinary resolution. especially its `` block (the intent the dev/review sessions treat as authoritative). The escalation is almost always that this block is silent on, or contradicts, a case the implementation hit. -2. **Present the escalation plainly** to the human: what is ambiguous or - contradictory, why it blocks safe implementation, and **2–4 concrete - resolution options** with a clear recommendation and its trade-offs. Keep it - tight — quote the relevant spec lines. +2. **Present the current pause evidence plainly** to the human: + - When the `escalations` array is non-empty, present its recorded entries in + their existing newest-first order. Do not replace recorded escalation + detail with `paused_reason`. + - When the `escalations` array is empty, first require `paused_reason` to be + text containing at least one non-whitespace character. If it is missing, + `null`, non-text, or blank after trimming, report a malformed resolve + context and do not write the resolution marker. Otherwise, present + `paused_reason` verbatim as the available evidence for the current pause + and disclose that no newer recorded escalation detail is available. Do not + read below the watermark, unfilter or recover an older artifact escalation, + or synthesize an escalation object from `paused_reason`. + + Using the selected evidence, explain what is ambiguous or contradictory, why + it blocks safe implementation, and offer **2–4 concrete resolution options** + with a clear recommendation and its trade-offs. Keep it tight — quote the + relevant spec lines. + 3. **Get the human's decision.** Ask follow-ups if the choice is unclear. Do not invent requirements; if the human is unsure, help them reason, don't guess. 4. **Update the frozen spec** to encode the decision unambiguously: amend the diff --git a/tests/test_resolve_skill_contract.py b/tests/test_resolve_skill_contract.py index 918dad5b..7052ac1d 100644 --- a/tests/test_resolve_skill_contract.py +++ b/tests/test_resolve_skill_contract.py @@ -101,6 +101,51 @@ def test_skill_documents_escalation_ordering_and_global_uniqueness(skill_md): assert len(contract_lines) == 2 +def test_skill_presents_paused_reason_when_no_newer_escalation_detail(skill_md): + """A watermarked pause can have no new session escalation to present. + + The positive non-empty assertion keeps the empty-path prohibitions from being + satisfied by deleting recorded-entry handling altogether. + """ + presentation_step = skill_md.split( + "2. **Present the current pause evidence plainly**", maxsplit=1 + )[1].split("\n3. **Get the human's decision.**", maxsplit=1)[0] + normalized = " ".join(presentation_step.split()) + + assert "When the `escalations` array is non-empty" in normalized + assert "present its recorded entries in their existing newest-first order" in normalized + assert "Do not replace recorded escalation detail with `paused_reason`." in normalized + assert "When the `escalations` array is empty" in normalized + assert ( + "present `paused_reason` verbatim as the available evidence for the current pause" + in normalized + ) + assert "no newer recorded escalation detail is available" in normalized + assert "Do not read below the watermark" in normalized + assert "unfilter or recover an older artifact escalation" in normalized + assert "synthesize an escalation object from `paused_reason`" in normalized + assert ( + "require `paused_reason` to be text containing at least one non-whitespace character" + in normalized + ) + assert "missing, `null`, non-text, or blank after trimming" in normalized + assert "report a malformed resolve context and do not write the resolution marker" in normalized + + shared_requirement = ( + "Using the selected evidence, explain what is ambiguous or contradictory, why it " + "blocks safe implementation, and offer **2–4 concrete resolution options** with a " + "clear recommendation and its trade-offs." + ) + assert shared_requirement in normalized + assert normalized.index(shared_requirement) > normalized.index( + "When the `escalations` array is empty" + ) + assert ( + "\n\n Using the selected evidence, explain what is ambiguous or contradictory" + in presentation_step + ) + + def test_skill_routes_project_artifacts_and_code_work_to_their_distinct_roots(skill_md): """Mentioning both keys is inert unless the skill explains the operational split. From 76145e724a9f42503658147543efb91943ebc1b5 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 18:43:50 -0700 Subject: [PATCH 10/18] sweep dw4-decision-dw-93: DW-93 via bmad-loop --- CHANGELOG.md | 2 + docs/FEATURES.md | 1 + src/bmad_loop/cli.py | 96 +++++++++++--- src/bmad_loop/journal.py | 56 +++++++- src/bmad_loop/runs.py | 215 +++++++++++++++++-------------- src/bmad_loop/runsetup.py | 29 +++-- src/bmad_loop/tui/app.py | 93 ++++++++++---- tests/conftest.py | 10 +- tests/test_cli.py | 174 +++++++++++++++++++++++++ tests/test_diagnostics.py | 7 +- tests/test_engine.py | 9 ++ tests/test_journal.py | 221 +++++++++++++++++++++++++++++++- tests/test_portability_guard.py | 99 +++++++++----- tests/test_runs.py | 206 ++++++++++++++++++++++++++++- tests/test_runsetup.py | 95 +++++++++++++- tests/test_tui_app.py | 188 ++++++++++++++++++++++++++- 16 files changed, 1304 insertions(+), 197 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebc911f6..e9395a19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -270,6 +270,8 @@ breaking changes may land in a minor release. unrecoverably, on a gitignored ledger. The mark and append legs now re-anchor only when the preimage they wrote over is still the bytes the run last claimed; the restore skips and journals `ledger-restore-skipped-diverged`. +- Serialize every run-state writer and control read-modify-write transaction with one canonical per-run advisory lock (DW-93). + - Let interactive resolve present `paused_reason` when watermark filtering leaves no newer recorded escalation detail, without recovering or inventing an escalation (DW-91). diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 947bb044..f130c34f 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -212,6 +212,7 @@ Result` section. Every other spec keeps warn-and-continue, and the record says w - Every run is a resumable on-disk state machine: `bmad-loop resume ` continues from a gate, escalation, or interruption. - A graceful stop (`stop --graceful` / TUI `S`) is resumable too: unlike a hard stop killed mid-item, it lets the in-flight item finish through commit and finalizes cleanly, ending as a `stopped` run that `resume` picks up at the next item. +- Every `state.json` publication is serialized by one advisory lock per run, keyed on the resolved run directory plus the logical `state.json` name and stored under the user state root rather than in git. Ignoring a final-component `state.json` symlink keeps that identity stable when atomic publication replaces the directory entry; alternate spellings of the run directory still converge. Multi-step control mutations (`resolve`, `resume`, code-root restamping, and stop's external fallback) hold that same lock from their authoritative read through atomic publication, so a waiter reloads the state its predecessor left instead of overwriting it from a stale snapshot. A fresh run or sweep likewise holds it from its initial state save through trusted-digest and PID publication, preventing an explicit-id resume from observing resumable state before the composer is live. Readers remain lock-free because publication is atomic. Stop does not hold the lock while it requests, signals, polls, or kills: a live engine must be able to publish its own stopped state; only the fallback's final reload/check/write is serialized. That final check preserves an engine that finished during delivery and retries against any newer live engine generation a concurrent resume published. POSIX lock acquisition blocks, while Windows can surface an `OSError` after its bounded wait; either failure aborts the mutation rather than writing unlocked. - All run state in `.bmad-loop/runs//` (gitignored): `state.json` (which records `repo_root`, the git root code work happens in, so an out-of-process re-arm reads back the tree the run measured, #716 — resume re-stamps it from the config.yaml it just re-read, since that is the tree it arms the engine against, and warns when the root moved: the baselines, preserve refs and branches already recorded name objects in the previous one. `resolve` and the TUI's re-arm re-stamp it themselves, before they re-arm: both re-arm and then resume in one gesture, so resume's own re-stamp lands after the re-arm has already advanced the baseline in whichever tree the mirror still named); `journal.jsonl` (every decision, including the `session-synthesized-from-frontmatter` catch and its `spec-marker-repaired` repair, #276, plus one `verify-command-result` per verifier command — emitted on the dev, repair and review legs alike, carrying `verification_stage` and a per-story `verification_sequence` that orders the passes across all three; the two passes that leave no record are `bmad-loop confirm --reverify`, which runs after the run is over, and any pass with no `[verify] commands` configured, which records nothing because nothing ran — each entry also carrying `spawn_error`, set when the verify command's child could not be started at all — typically because its working directory is missing, is not a directory, or cannot be searched, though any spawn-time `OSError` (a missing shell, EMFILE, ENOMEM) reaches the same field and the wrapped exception is what names the cause — which is an environment fault that pauses the run rather than a command that failed — whose stream pointers name the `verify/` directory below, and one `park-proof-of-work-skipped` per attempt that cleared the dev artifact gate on an `awaiting-operator` park with proof-of-work waived — not per park that committed, since the stages after that gate can still reject the attempt — carrying `zero_diff`: `true` when the waived gate found no non-excluded changes (its exclusions include the spec, board, any restore-patch artifact, and an orchestrator-authored deferred-work ledger append), `false` when it found changes, and `null` when the probe could not answer (a git fault, a git refusal such as an unresolvable baseline, or an attempt with no recorded baseline to measure from) and the gate was waived anyway, so the waiver itself is recorded whatever the probe managed to say. `false` is a statement about the tree, not about who wrote what: the gate this stands in for cannot attribute residue to a session in a shared checkout, and the record inherits that limit rather than improving on it); `tasks//` (per-session prompt + shared artifacts: [`result.json`, `escalation.json`] — respectively the per-session result and escalation outputs — plus adapter-specific breadcrumbs: `session-lifecycle.jsonl` records timeout fires, budget-guard trips (`budget-tripped` / `over-budget-fired`), transport-failure classification (`env-fault-classified`, #194), a mux session lost under the run (`session-vanished`, #489) and the #276 forensics (`spec-status-transition-observed`, `frontmatter-unmodified-refused`, `contract-nudge-sent`); `heartbeat.json` is the wait loop's proof-of-life; `resultless-stops.jsonl` records give-up Stops with a verdict — `no-artifact`, `ambiguous-frontmatter`, `unmodified-since-launch`, `terminal-frontmatter-pending`); `logs/`; `verify/` (verifier command stdout/stderr, one file per stream, pointed at by the journal's `verify-command-result` records — its own store, because every name in `logs/` is a session task id the TUI resolves as a pane log; each stream is retained tail-first up to `[verify] stream_capture_kb` (256 KiB, `0` = keep nothing) and the record carries the full byte count, a truncation flag, and a `capture_error` when the write itself failed); `deferred/`; `resolve/`; `ATTENTION`; `ctl-window` (the control-session window id the last TUI launch minted, so attach/stop follow the live window, #482). - One piece deliberately lives **outside** that directory: the hook-event channel (#494) is at `///events/` under the user-scoped state root (`BMAD_LOOP_STATE_DIR`, see the [transport section](#hook-based-transport-no-pane-scraping) below and the README's env-var table), not `/events/`. The orchestrator still polls the legacy in-tree location, so a project whose installed relay predates the move keeps completing its sessions. `delete`, `archive` and `clean` remove the out-of-tree counterpart along with the run dir, and `clean` sweeps counterparts whose run dir is already gone; an **archived** run's tarball therefore no longer contains `events/` — those files are transient completion signals, consumed while the run was live, and everything an archive is read for later is in the run dir. - `journal.jsonl` records `session-end` for every session unconditionally — even a teardown that throws still lands one (status `aborted` when the outcome is unknowable). A timed-out session's entry carries `fired_at` (wall time the deadline was declared), `teardown_s` (wall seconds from that fire to this entry — the teardown gap), and `expired_clock` (`monotonic` / `wall` / `both` — `wall` alone fingerprints a host suspend that froze the monotonic clock). Every entry whose usage was read carries `tokens` (raw) and `tokens_weighted` (cache reads at `limits.cache_read_weight`), keeping per-session spend reconstructible; both are `null` when the usage read failed, and both are absent on an `aborted` end. `tokens_weighted` is the end-of-session total — distinct from a tripped session's `budget_weighted`, the guard's mid-session sample at trip time. diff --git a/src/bmad_loop/cli.py b/src/bmad_loop/cli.py index f1084369..8371f6d5 100644 --- a/src/bmad_loop/cli.py +++ b/src/bmad_loop/cli.py @@ -70,7 +70,7 @@ validate_document, ) from .engine import Engine -from .journal import Journal, load_state, save_state +from .journal import Journal, load_state, save_state, state_lock from .model import RunState from .platform_util import MAX_SEGMENT, resolve_or_lexical, walk_files_unlinked from .process_host import ProcessHostError @@ -2554,9 +2554,8 @@ def _sweep_dry_run(paths: bmadconfig.ProjectPaths, pol) -> int: return 0 -def _resume_paused_run(project: Path, run_dir: Path) -> int: - """Resume the engine for a paused/interrupted run. Shared by `resume` and - the re-arm step of `resolve`.""" +def _prepare_resume_locked(project: Path, run_dir: Path): + """Publish resume state while the caller holds this run's state lock.""" # An id that aliases a control session (`ctl` / `ctl-<16hex>` — # runs.run_id_aliases_control_session; NOT the mint's broader reservation, # since a historical `ctl-foo` run has a genuine agent session and resumes @@ -2806,6 +2805,30 @@ def _resume_paused_run(project: Path, run_dir: Path) -> int: # SweepEngine and _make_adapters are handed in from this module's namespace so # the test suite's `monkeypatch.setattr(cli, "SweepEngine"/"Engine"/..., ...)` # still applies. + return paths, state, pol, journal, new_digest, profiles + + +def _resume_paused_run(project: Path, run_dir: Path) -> int: + """Resume a paused/interrupted run without holding its lock across execution.""" + with state_lock(run_dir): + # Repeat the command's liveness decision after exclusion. A concurrent + # resume publishes its pid under this same hold, so the waiter refuses + # instead of reloading the predecessor's old paused state and double-driving. + if runs.engine_liveness(run_dir) == "alive": + print( + f"run {run_dir.name} is still live — resuming would double-drive it; " + "stop it first", + file=sys.stderr, + ) + return 1 + prepared = _prepare_resume_locked(project, run_dir) + if isinstance(prepared, int): + return prepared + paths, state, pol, journal, new_digest, profiles = prepared + + # Adapter construction and the engine lifetime are deliberately outside the + # state hold. The pid/state publication above makes a rival control command + # observe this process as live while these unbounded operations proceed. composed = runsetup.compose_resume( project=project, paths=paths, @@ -3255,6 +3278,7 @@ def cmd_resolve(args: argparse.Namespace) -> int: try: paths = bmadconfig.load_paths(project) except (bmadconfig.BmadConfigError, OSError) as e: + paths = None # An observation, so it degrades: without the config this process cannot NAME # the tree, and re-pointing the mirror at a guess is the one outcome worse than # leaving it alone. The re-arm then reads the root the run recorded — precisely @@ -3302,23 +3326,59 @@ def cmd_resolve(args: argparse.Namespace) -> int: # config lecture about a gesture they did not make. if (rc := _reject_isolation_conflict(paths, pol)) is not None: return rc - if (moved := runs.restamp_code_root(run_dir, paths.repo_root)) is not None: - print(f"warning: {moved}", file=sys.stderr) before_entries = runs.journal_entries_or_none(run_dir) outcome: runs.RearmOutcome | None = None try: - outcome = runs.rearm_escalation( - run_dir, - story_key, - restore_patch=restore_patch, - isolated_redrive=pol.scm.isolation == "worktree", - resolution_recorded=resolution_recorded, - # The tree this invocation is acting in, which is also the tree - # `build_context` published a `spec_file` from. `state.project` is where the - # run was LAUNCHED and nothing re-stamps it, so a moved project would have - # the agent edit one file and the re-arm flip another. - project_root=project, - ) + with state_lock(run_dir): + # The pre-session checks intentionally stay lock-free; this is the + # mutation boundary, so repeat every state/liveness precondition from + # the snapshot left by the preceding writer before restamping anything. + fresh_state = load_state(run_dir) + if fresh_state.paused_stage != PAUSE_ESCALATION: + print( + f"run {args.run_id} is not paused at an escalation " + f"(stage: {fresh_state.paused_stage or 'none'})", + file=sys.stderr, + ) + return 1 + fresh_live = runs.engine_liveness(run_dir) + if fresh_live == "alive": + print(f"run {args.run_id} is still live — stop it first", file=sys.stderr) + return 1 + if fresh_live == "unknown" and not args.force: + print( + f"run {args.run_id}: engine may still be live (unverifiable pid) — " + "refusing to re-arm. Confirm the engine process is gone, then re-run " + "with --force (`stop` cannot verify or clear an unverifiable pid).", + file=sys.stderr, + ) + return 1 + fresh_task = fresh_state.tasks.get(story_key) + if fresh_task is None or fresh_task.phase != Phase.ESCALATED: + print(f"no escalated story to resolve in run {args.run_id}", file=sys.stderr) + return 1 + if fresh_task.generation != task.generation: + print( + f"the escalation for {story_key} changed while resolve was in progress " + "— not re-arming", + file=sys.stderr, + ) + return 1 + if paths is not None: + if (moved := runs.restamp_code_root(run_dir, paths.repo_root)) is not None: + print(f"warning: {moved}", file=sys.stderr) + outcome = runs.rearm_escalation( + run_dir, + story_key, + restore_patch=restore_patch, + isolated_redrive=pol.scm.isolation == "worktree", + resolution_recorded=resolution_recorded, + # The tree this invocation is acting in, which is also the tree + # `build_context` published a `spec_file` from. `state.project` is where the + # run was LAUNCHED and nothing re-stamps it, so a moved project would have + # the agent edit one file and the re-arm flip another. + project_root=project, + ) except runs.RearmError as e: print(f"error: {e}", file=sys.stderr) return 1 diff --git a/src/bmad_loop/journal.py b/src/bmad_loop/journal.py index 4ef72b3b..124676cb 100644 --- a/src/bmad_loop/journal.py +++ b/src/bmad_loop/journal.py @@ -4,7 +4,10 @@ import json import os +import threading import time +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path from typing import Any @@ -14,6 +17,7 @@ atomic_replace, atomic_write_text, atomic_write_text_at, + file_lock, is_link_like, open_dir_confined, ) @@ -77,6 +81,8 @@ # name; neither restates the pair. SELF_MINTED_FIELDS: frozenset[str] = frozenset({"log_task", "log_pos"}) +_STATE_LOCK_LOCAL = threading.local() + class Journal: def __init__(self, run_dir: Path): @@ -217,12 +223,52 @@ def entries(self) -> list[dict[str, Any]]: return out +@contextmanager +def state_lock(run_dir: Path) -> Iterator[None]: + """Serialize one run's state mutations, re-entering only for the same run. + + The sidecar identity comes from :func:`runs.lock_path_for`, so alternate path + spellings of one ``state.json`` rendezvous on the same out-of-tree lock. The + import is deliberately lazy: ``runs`` imports this module's persistence helpers. + + Reentrancy is thread-local and intentionally limited to one run. An outer + read-modify-write transaction can call the self-locking :func:`save_state` + without acquiring the OS lock twice, while nested mutation of another run is + refused before a second lock can introduce an ordering cycle. + """ + from . import runs + + lock_path = runs.lock_path_for(run_dir / STATE_FILE, follow_final_symlink=False) + held_path = getattr(_STATE_LOCK_LOCAL, "path", None) + if held_path is not None: + if held_path != lock_path: + raise RuntimeError( + f"cannot nest run-state locks for different runs: {held_path} then {lock_path}" + ) + _STATE_LOCK_LOCAL.depth += 1 + try: + yield + finally: + _STATE_LOCK_LOCAL.depth -= 1 + return + + with file_lock(lock_path): + _STATE_LOCK_LOCAL.path = lock_path + _STATE_LOCK_LOCAL.depth = 1 + try: + yield + finally: + del _STATE_LOCK_LOCAL.depth + del _STATE_LOCK_LOCAL.path + + def save_state(run_dir: Path, state: RunState) -> None: - run_dir.mkdir(parents=True, exist_ok=True) - target = run_dir / STATE_FILE - tmp = target.with_suffix(".json.tmp") - tmp.write_text(json.dumps(state.to_dict(), indent=2), encoding="utf-8") - atomic_replace(tmp, target) + with state_lock(run_dir): + run_dir.mkdir(parents=True, exist_ok=True) + target = run_dir / STATE_FILE + tmp = target.with_suffix(".json.tmp") + tmp.write_text(json.dumps(state.to_dict(), indent=2), encoding="utf-8") + atomic_replace(tmp, target) def load_state(run_dir: Path) -> RunState: diff --git a/src/bmad_loop/runs.py b/src/bmad_loop/runs.py index c453d793..c71e97c2 100644 --- a/src/bmad_loop/runs.py +++ b/src/bmad_loop/runs.py @@ -40,7 +40,7 @@ mux_usable, ) from .frontmatter import auto_dev_baseline_of, parse_frontmatter, status_of -from .journal import STATE_FILE, VERIFY_DIR, Journal, load_state, save_state +from .journal import STATE_FILE, VERIFY_DIR, Journal, load_state, save_state, state_lock from .model import PAUSE_ESCALATION, Phase, RunState, StoryTask from .platform_util import ( MAX_SEGMENT, @@ -1370,7 +1370,7 @@ def accepted_tags(project: Path) -> frozenset[str]: return frozenset({project_tag(project), str(project.resolve())}) -def lock_path_for(data_path: Path) -> Path: +def lock_path_for(data_path: Path, *, follow_final_symlink: bool = True) -> Path: """The advisory-lock sidecar for a mutable data file: ``/locks/-.lock``. @@ -1399,7 +1399,15 @@ def lock_path_for(data_path: Path) -> Path: usable state root (see :func:`state_root`); the caller fails rather than silently locking somewhere else. """ - resolved = data_path.resolve() + # Run-state publication atomically replaces ``state.json``. Its transaction + # lock therefore needs the identity of that *logical directory entry*, not the + # current referent of a planted final-component symlink: following that link + # would change the sidecar halfway through an outer transaction when + # ``save_state`` replaces it. Other mutable artifacts retain the historical + # referent-based behavior by default (notably shared external ledgers). + resolved = ( + data_path.resolve() if follow_final_symlink else data_path.parent.resolve() / data_path.name + ) digest = hashlib.sha256(os.fsencode(str(resolved))).hexdigest()[:16] return state_root() / "locks" / f"{digest}-{resolved.name}.lock" @@ -2100,6 +2108,14 @@ def request_graceful_stop(run_dir: Path) -> str: def stop_run(run_dir: Path) -> bool: + """Stop the engine generation current at completion of the gesture.""" + while True: + result = _stop_run_once(run_dir) + if result is not None: + return result + + +def _stop_run_once(run_dir: Path) -> bool | None: """Stop a live run. Returns False if it was already finished. The request is delivered two ways at once, and the engine wins whichever race @@ -2181,6 +2197,7 @@ def stop_run(run_dir: Path) -> bool: # the pid we recorded is already gone, or was reused by an unrelated # process before stop_run ran — never signal a stranger; mark stopped below. pid = None + addressed_engine = (pid, identity) # Whether this call ever proved the engine dead. Only a confirmed death licenses # the fallback below to discard the request we lodged: while the engine may still # be running, that file is the one channel left that can stop it (on native @@ -2268,8 +2285,53 @@ def stop_run(run_dir: Path) -> bool: # addresses the registry this process exported, and `cleanup`'s legacy pass is # what reaches a session left in an older one. kill_session(run_dir.name) - state = load_state(run_dir) - if state.stopped: + + already_stopped = False + finished_during_stop = False + retry_new_engine = False + clear_request = False + with state_lock(run_dir): + # Authoritative post-delivery snapshot. A rival writer that completed while + # stop was signalling is observed here, after exclusion, rather than being + # overwritten by the stale state loaded at entry. + state = load_state(run_dir) + current_engine = read_pid_identity(run_dir) + current_liveness = engine_liveness(run_dir) + rival_published_engine = current_liveness != "dead" and current_engine != addressed_engine + if state.finished: + # The engine completed while the stop channels were in flight. Its + # terminal state is authoritative; do not rewrite it as a fallback stop. + finished_during_stop = True + clear_request = current_liveness == "dead" + elif rival_published_engine: + # Resume publishes pid + state under this same lock. If that happened + # while this attempt was signalling an older generation, release before + # delivering to the new process and retry from its fresh identity. + retry_new_engine = True + elif state.stopped: + already_stopped = True + clear_request = True + elif engine_may_live and not lodged: + Journal(run_dir).append("run-stop-undelivered", pid=pid) + raise StopRunError( + f"run {run_dir.name}: the stop request could not be written to the run " + "directory and the engine could not be proved dead, so no stop is pending. " + "Its agent session was killed as a backstop. Free space in the run directory " + "and retry, or stop the process yourself" + ) + else: + state.stopped = True + save_state(run_dir, state) + clear_request = not engine_may_live + + if clear_request: + clear_graceful_stop(run_dir) + if finished_during_stop: + return False + if retry_new_engine: + return None + + if already_stopped: # The engine honored the stop and is gone, and its own `run-stop` already # stands in the journal. Stamping `fallback=True` on top would describe an # engine that did its own teardown as one that had to be stopped from @@ -2286,47 +2348,11 @@ def stop_run(run_dir: Path) -> bool: # re-stop at its first item. Safe on the `engine_may_live` paths too: a # written `stopped` *is* the engine reporting it honored the request, so # there is no live consumer left to strand. - clear_graceful_stop(run_dir) return True - # Neither channel was delivered: nothing is lodged, and we never proved the engine - # dead. This is the one outcome `stop` must not report as success — the operator is - # left believing a request is in flight that was never written, while an engine we - # could not signal keeps mutating the project. The pid-reuse guard above already - # refuses for its own path; these are its siblings, and the only reason they stayed - # quiet is that they clear `pid` and skip that block. Not a regression — on the - # merge-base this was the state of *every* refused signal, because `stop_run` cleared - # the request as its first statement — but the earlier decision to report success - # rested on the request being retained, which is exactly what did not happen here. - # - # Placement is load-bearing, twice over. It sits *after* the session backstop - # because refusing to report a stop is no reason to leak the window, and *after* the - # `state.stopped` return because a run the engine already honored must not be - # reported as a failure. Journal the attempt before raising: the `run-stop` append - # below is skipped, and an unrecorded stop attempt is its own trap. - if engine_may_live and not lodged: - Journal(run_dir).append("run-stop-undelivered", pid=pid) - raise StopRunError( - f"run {run_dir.name}: the stop request could not be written to the run " - "directory and the engine could not be proved dead, so no stop is pending. " - "Its agent session was killed as a backstop. Free space in the run directory " - "and retry, or stop the process yourself" - ) - - # Fallback: no live engine (or it never confirmed). Mark it stopped here. Discard - # the request first — nothing is left alive to consume it, and a file outliving - # the run it asked to stop is a trap for the next resume. - # - # Unless we never actually proved that. Where the engine may still be running, - # the request stays lodged and the stop is genuinely still in flight: the engine - # honors the file at its next poll and writes `stopped` itself. Discarding it here - # would leave a live engine with no channel left while we report the run stopped — - # the stale-request trap above is the lesser of the two, and it only bites a run - # that is later resumed, which this one cannot be until that engine exits. - if not engine_may_live: - clear_graceful_stop(run_dir) - state.stopped = True - save_state(run_dir, state) + # The locked branch above performed the external fallback's final + # read-modify-write. The journal remains outside the state transaction: it is + # append-only observation, not part of state publication. Journal(run_dir).append("run-stop", pid=pid, fallback=True) return True @@ -3838,41 +3864,18 @@ def _rearm_commit_landed(run_dir: Path, story_key: str, task: StoryTask) -> bool only if some other writer had minted the same bump, and `phase` alone moves for reasons a re-arm does not own. - Those two conjuncts are a sufficient identity ONLY because `rearm_escalation` runs as - the SOLE writer of this run's `state.json`, and that model is the probe's premise - rather than an assumption left implicit. Exactly TWO call sites reach this - transaction — `cli.cmd_resolve` and `tui.TuiApp._do_rearm` — and each consults - liveness before any side effect: :func:`engine_liveness` in the CLI, its pid-file - sibling :func:`liveness` in the TUI (`probe_liveness` is the shared body). A third - control command, `cli.cmd_resume`, never re-arms but DOES write this run's - `state.json` (through `_resume_paused_run`), which is why the sole-writer claim has - to account for it as well as for the two callers. - `tests/test_portability_guard.py::test_rearm_escalation_called_only_behind_a_liveness_gate` - holds that enumeration, which is otherwise prose a third call site could falsify - silently. - - Those gates establish that no engine is PROVABLY ALIVE — not that one is proven - dead — and the premise rests on the difference, so it is stated rather than rounded - off. `"alive"` is refused outright at all three. `"unknown"` is not: `cmd_resolve` - proceeds on it under `--force`, `cmd_resume` warns and proceeds by design (it is the - recovery path that rewrites engine.pid), and the TUI counts it as blocking only for a - pid-backed run. So the model this probe leans on is the engine stopped AND the - operator driving one control command at a time. Under it only THIS caller can have - moved either field, which is exactly what the exact-phase predicate reports — the - predicate is correct for the reason it is narrow. - - Two overlapping control commands are OUTSIDE that model rather than handled by it, - and deliberately so. `journal.save_state` stages through a FIXED `state.json.tmp` - sibling before its `atomic_replace` — the collision `_write_stop_request` documents - under #379, which names the stop-request file as the ONE control file with genuinely - *concurrent* writers — so two overlapping re-arms lose a `save_state` to - `FileNotFoundError` long before this probe's identity could matter. Answering them - here was weighed and declined: a lock taken by only `rearm_escalation` excludes - nobody (the honest fix is a run-level one shared with `_resume_paused_run` and the - engine's own `save_state`), and a durable per-re-arm token stamped on `StoryTask` - would buy this probe a precision the `save_state` writer beneath it cannot honour, at - the cost of a new persisted model field. Tracked as DW-93; the probe stays two - conjuncts over the reloaded task. + Those two conjuncts are a sufficient identity because the entire re-arm — including + this error-path probe — runs inside :func:`journal.state_lock`. Every state writer + participates through the self-locking :func:`journal.save_state`, and every external + read-modify-write gesture holds the same canonical run sidecar from its deciding read + through publication. Therefore no rival can supply the observed generation/phase + while this transaction is in flight: a waiter reloads only after this hold exits. + + The two operator call sites still repeat liveness under their outer transaction + holds. That is a separate safety rule: serialization prevents stale publication, + while liveness prevents deliberately taking a turn after an engine known to be live. + The portability guard keeps both the writer/transaction inventory and the two re-arm + surface gates executable rather than relying on this prose. Degrades to `False` — roll back, the pre-existing behavior — on ANY failure to read or parse the state file. This is observation feeding a repair decision, and the safe @@ -4011,22 +4014,23 @@ def restamp_code_root(run_dir: Path, repo_root: Path) -> str | None: run has changed repositories, and the paths are the half that would put an attacker-controlled string on their terminal. """ - state = load_state(run_dir) - new = str(repo_root) - if state.repo_root == new: - return None - moved = bool(state.repo_root) - state.repo_root = new - save_state(run_dir, state) - if not moved: - return None - return ( - f"run {run_dir.name}: the code root in _bmad/bmm/config.yaml has changed since " - "this run started — the re-drive works in the tree configured now, while the " - "baselines, preserve refs and branches this run already recorded name objects " - "in the previous one. Restore the previous `repo_root:` value if you did not " - "intend the move." - ) + with state_lock(run_dir): + state = load_state(run_dir) + new = str(repo_root) + if state.repo_root == new: + return None + moved = bool(state.repo_root) + state.repo_root = new + save_state(run_dir, state) + if not moved: + return None + return ( + f"run {run_dir.name}: the code root in _bmad/bmm/config.yaml has changed since " + "this run started — the re-drive works in the tree configured now, while the " + "baselines, preserve refs and branches this run already recorded name objects " + "in the previous one. Restore the previous `repo_root:` value if you did not " + "intend the move." + ) @dataclass(frozen=True) @@ -4076,6 +4080,27 @@ def rearm_escalation( isolated_redrive: bool, resolution_recorded: bool, project_root: Path | None = None, +) -> RearmOutcome: + """Run the complete spec/git/state re-arm transaction under the run lock.""" + with state_lock(run_dir): + return _rearm_escalation_locked( + run_dir, + story_key, + restore_patch=restore_patch, + isolated_redrive=isolated_redrive, + resolution_recorded=resolution_recorded, + project_root=project_root, + ) + + +def _rearm_escalation_locked( + run_dir: Path, + story_key: str | None = None, + *, + restore_patch: str | None = None, + isolated_redrive: bool, + resolution_recorded: bool, + project_root: Path | None = None, ) -> RearmOutcome: """Re-arm an escalation-paused story so the next resume re-drives it. diff --git a/src/bmad_loop/runsetup.py b/src/bmad_loop/runsetup.py index 071c8ada..4b4e258d 100644 --- a/src/bmad_loop/runsetup.py +++ b/src/bmad_loop/runsetup.py @@ -43,7 +43,7 @@ from . import policy as policy_mod from . import runs from .checks import Finding -from .journal import Journal, save_state +from .journal import Journal, save_state, state_lock from .model import RunState from .platform_util import atomic_replace, is_wsl_unc_path from .runs import RUNS_DIR @@ -1025,12 +1025,17 @@ def compose_run( spec_folder=spec_folder, trusted_config_digest=trusted_config_digest, ) - save_state(run_dir, state) - # After the run dir exists (Journal mkdir'd it above) and before the pid lands: - # the ordering `reconcile_orphan_state_dirs` reads runs in, and a stamp that - # cannot be written fails the launch before an observer can see a live run. - runs.write_trusted_config_digest(project, run_id, trusted_config_digest) - runs.write_pid(run_dir) + # State becoming resumable and the pid making this process live are one + # publication. An explicit-id resume waits for the pid rather than entering + # between these writes and double-driving the freshly composed run. + with state_lock(run_dir): + save_state(run_dir, state) + # After the run dir exists (Journal mkdir'd it above) and before the pid + # lands: the ordering `reconcile_orphan_state_dirs` reads runs in, and a + # stamp that cannot be written fails the launch before an observer can + # see a live run. + runs.write_trusted_config_digest(project, run_id, trusted_config_digest) + runs.write_pid(run_dir) adapters = make_adapters(project, run_dir, policy, profiles=profiles) journal.append( "run-start", @@ -1158,10 +1163,12 @@ def compose_sweep( run_type="sweep", trusted_config_digest=trusted_config_digest, ) - save_state(run_dir, state) - # Out of the tree, same ordering and same reason as compose_run's stamp. - runs.write_trusted_config_digest(project, run_id, trusted_config_digest) - runs.write_pid(run_dir) + # Same indivisible state/pid publication as compose_run. + with state_lock(run_dir): + save_state(run_dir, state) + # Out of the tree, same ordering and same reason as compose_run's stamp. + runs.write_trusted_config_digest(project, run_id, trusted_config_digest) + runs.write_pid(run_dir) options = { "prompting": prompting, "decisions_only": decisions_only, diff --git a/src/bmad_loop/tui/app.py b/src/bmad_loop/tui/app.py index 767a2ffd..3b54d202 100644 --- a/src/bmad_loop/tui/app.py +++ b/src/bmad_loop/tui/app.py @@ -26,7 +26,7 @@ from .. import bmadconfig, decisions, devcontract, policy, resolve, runs, stories, verify from ..adapters.multiplexer import MultiplexerError, mux_usable -from ..journal import load_state +from ..journal import load_state, state_lock from ..model import ( PAUSE_EPIC_BOUNDARY, PAUSE_ESCALATION, @@ -34,6 +34,7 @@ PAUSE_SPEC_APPROVAL, PAUSE_STORY_CHECKPOINT, PAUSE_STORY_GATE, + Phase, RunState, StoryTask, ) @@ -723,6 +724,8 @@ def done(verb: str | None) -> None: def _review_escalation(self, run_id: str, run_dir: Path, state: RunState) -> None: story_key = state.paused_story_key or "?" + task = state.tasks.get(story_key) + expected_generation = task.generation if task is not None else None spec_path, spec_text, readable = self._paused_spec(state) title, description = self._story_context(state, story_key) restore_recorded = self._restore_recorded(run_dir, story_key) @@ -751,7 +754,13 @@ def done(verb: str | None) -> None: return self._launch_resolve(run_id) elif verb == "rearm": - self._do_rearm(run_id, run_dir, story_key, restore_recorded=restore_recorded) + self._do_rearm( + run_id, + run_dir, + story_key, + restore_recorded=restore_recorded, + expected_generation=expected_generation, + ) self.push_screen(modal, done) @@ -902,7 +911,13 @@ def _echo_rearm_notices(self, notices: tuple[runs.RearmNotice, ...]) -> None: ) def _do_rearm( - self, run_id: str, run_dir: Path, story_key: str, *, restore_recorded: bool = False + self, + run_id: str, + run_dir: Path, + story_key: str, + *, + restore_recorded: bool = False, + expected_generation: int | None = None, ) -> None: """Re-arm a resolved escalation + resume — the `resolve --no-interactive` path (rearm_escalation handles sentinel auto-delete-with-preservation).""" @@ -945,6 +960,7 @@ def _do_rearm( try: paths = bmadconfig.load_paths(self.project) except (bmadconfig.BmadConfigError, OSError) as e: + paths = None self.notify( f"cannot read the project config to confirm the code root ({e}) — " "re-arming against the root this run recorded", @@ -965,31 +981,58 @@ def _do_rearm( if conflict is not None: self.notify(conflict, severity="error") return - if (moved := runs.restamp_code_root(run_dir, paths.repo_root)) is not None: - self.notify(moved, severity="warning") before_entries = runs.journal_entries_or_none(run_dir) outcome: runs.RearmOutcome | None = None try: - outcome = runs.rearm_escalation( - run_dir, - story_key, - isolated_redrive=isolation == "worktree", - # The live project this dashboard was launched against, matching - # `cli.cmd_resolve`: the re-arm's spec writes have to land in the tree - # the operator is looking at, not in the one the run recorded at launch. - # `self.project` rather than `paths.project` because the `load_paths` - # arm above may have degraded without binding `paths` at all. - project_root=self.project, - # DW-11. This gesture runs no resolve session, so it accepted nothing: - # the escalation watermark must not advance. A `resolution.json` on - # disk is NOT evidence to the contrary here — `_restore_recorded` - # already records the governing fact for this surface, that a stale - # marker is indistinguishable from a fresh one, which is why this path - # declines the restore latch too. Stamping on its presence would bury - # escalations raised since the marker was written. - resolution_recorded=False, - ) - except RearmError as e: + with state_lock(run_dir): + # Repeat the liveness decision after exclusion. Config/policy work + # above is deliberately lock-free; only this bounded restamp+re-arm + # mutation gesture is serialized. + if self._resolve_blocked_by_liveness(run_id, run_dir): + return + fresh_state = load_state(run_dir) + fresh_task = fresh_state.tasks.get(story_key) + if ( + fresh_state.paused_stage != PAUSE_ESCALATION + or fresh_task is None + or fresh_task.phase != Phase.ESCALATED + ): + self.notify( + f"run {run_id} is no longer paused at escalation for {story_key} " + "— not re-arming", + severity="warning", + ) + return + if expected_generation is not None and fresh_task.generation != expected_generation: + self.notify( + f"the escalation for {story_key} changed while its review was open " + "— not re-arming", + severity="warning", + ) + return + if paths is not None: + if (moved := runs.restamp_code_root(run_dir, paths.repo_root)) is not None: + self.notify(moved, severity="warning") + outcome = runs.rearm_escalation( + run_dir, + story_key, + isolated_redrive=isolation == "worktree", + # The live project this dashboard was launched against, matching + # `cli.cmd_resolve`: the re-arm's spec writes have to land in the tree + # the operator is looking at, not in the one the run recorded at launch. + # `self.project` rather than `paths.project` because the `load_paths` + # arm above may have degraded without binding `paths` at all. + project_root=self.project, + # DW-11. This gesture runs no resolve session, so it accepted nothing: + # the escalation watermark must not advance. A `resolution.json` on + # disk is NOT evidence to the contrary here — `_restore_recorded` + # already records the governing fact for this surface, that a stale + # marker is indistinguishable from a fresh one, which is why this path + # declines the restore latch too. Stamping on its presence would bury + # escalations raised since the marker was written. + resolution_recorded=False, + ) + except (RearmError, OSError, runs.StateRootError) as e: self.notify(f"re-arm failed: {e}", severity="error") return finally: diff --git a/tests/conftest.py b/tests/conftest.py index 590e28af..f2d347ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ from bmad_loop.adapters.base import SessionResult, SessionSpec from bmad_loop.bmadconfig import ProjectPaths, load_paths from bmad_loop.checks import ValidationReport -from bmad_loop.journal import save_state +from bmad_loop.journal import STATE_FILE, save_state from bmad_loop.model import PAUSE_ESCALATION, Phase, RunState, SessionRecord, StoryTask from bmad_loop.verify import finalize_commit, rev_parse_head @@ -69,6 +69,14 @@ def _codec_rejects_bad_byte() -> bool: ) +def assert_run_state_lock_held(run_dir: Path) -> None: + """Fail unless this process already owns the canonical logical state lock.""" + sidecar = runs.lock_path_for(run_dir / STATE_FILE, follow_final_symlink=False) + with pytest.raises(OSError): + with platform_util.file_lock(sidecar, blocking=False): + pytest.fail("the run-state publication boundary was outside its outer lock") + + def opencode_runs() -> bool: """Whether this host has an ``opencode`` binary that actually RUNS. diff --git a/tests/test_cli.py b/tests/test_cli.py index 995407de..a1c33411 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,6 +17,7 @@ PROJECT_MARKER_CMD, REPO_ROOT_MARKER_CMD, UNRESOLVABLE, + assert_run_state_lock_held, escalated_run, fault_read_text, git, @@ -2548,6 +2549,88 @@ def test_resolve_force_unknown_proceeds(tmp_path, monkeypatch, capsys): assert load_state(run_dir).tasks["s1"].phase == Phase.PENDING # past the gate, re-armed +def test_resolve_reloads_state_after_waiting_for_mutation_lock(tmp_path, monkeypatch, capsys): + """Ablation: delete cmd_resolve's fresh in-lock state check and both concurrent + gestures reach rearm_escalation instead of the waiter refusing the rival's result.""" + import contextlib + + from bmad_loop import runs + from bmad_loop.journal import load_state, save_state + from bmad_loop.model import Phase + + run_dir = _escalated_run(tmp_path, "r1") + monkeypatch.setattr(runs, "engine_liveness", lambda _rd: "dead") + monkeypatch.setattr(cli, "_resume_paused_run", lambda *_a: pytest.fail("resumed")) + monkeypatch.setattr(runs, "rearm_escalation", lambda *_a, **_k: pytest.fail("double re-armed")) + + @contextlib.contextmanager + def rival_first(_run_dir): + rival = load_state(run_dir) + rival.tasks["s1"].phase = Phase.PENDING + save_state(run_dir, rival) + yield + + monkeypatch.setattr(cli, "state_lock", rival_first) + + rc = cli.main(["resolve", "--project", str(tmp_path), "r1", "--no-interactive", "--resume"]) + + assert rc == 1 + assert "no escalated story" in capsys.readouterr().err + + +def test_resolve_refuses_a_newer_escalation_after_waiting_for_mutation_lock( + tmp_path, monkeypatch, capsys +): + """A same-story re-escalation can have the same phase after a rival re-drive. + + Ablation: delete the generation comparison in ``cmd_resolve`` and this stale + gesture consumes the newer escalation even though its resolve session never saw it. + """ + import contextlib + + from bmad_loop import runs + from bmad_loop.journal import load_state, save_state + + run_dir = _escalated_run(tmp_path, "r1") + original_generation = load_state(run_dir).tasks["s1"].generation + monkeypatch.setattr(runs, "engine_liveness", lambda _rd: "dead") + monkeypatch.setattr(cli, "_resume_paused_run", lambda *_a: pytest.fail("resumed")) + monkeypatch.setattr(runs, "rearm_escalation", lambda *_a, **_k: pytest.fail("stale rearm")) + + @contextlib.contextmanager + def rival_first(_run_dir): + rival = load_state(run_dir) + rival.tasks["s1"].generation = original_generation + 1 + save_state(run_dir, rival) + yield + + monkeypatch.setattr(cli, "state_lock", rival_first) + + rc = cli.main(["resolve", "--project", str(tmp_path), "r1", "--no-interactive", "--resume"]) + + assert rc == 1 + assert "changed while resolve was in progress" in capsys.readouterr().err + + +def test_resolve_retains_outer_lock_through_rearm_call(tmp_path, monkeypatch): + run_dir = _escalated_run(tmp_path, "r1") + rearms: list[Path] = [] + monkeypatch.setattr(runs, "engine_liveness", lambda _rd: "dead") + monkeypatch.setattr(cli, "_resume_paused_run", lambda *_a: 0) + + def checked_rearm(rd, key, **_kwargs): + assert_run_state_lock_held(rd) + rearms.append(rd) + return _rearm_outcome(key) + + monkeypatch.setattr(runs, "rearm_escalation", checked_rearm) + + assert ( + cli.main(["resolve", "--project", str(tmp_path), "r1", "--no-interactive", "--resume"]) == 0 + ) + assert rearms == [run_dir] + + def test_resolve_no_escalated_story(tmp_path, capsys): _make_run_with_state( tmp_path, "r1", paused_stage="escalation", paused_reason="x", paused_story_key="ghost" @@ -5271,10 +5354,101 @@ def _paused_run_for_resume(project, monkeypatch, *, snapshot=LAUNCH_SNAPSHOT, ** **state_kwargs, ) monkeypatch.setattr(runs, "kill_session", lambda rid: None) + # These tests call the private helper repeatedly in one pytest process to inspect + # successive policy snapshots. A real command exits or keeps driving after + # publishing its pid; suppress that unrelated liveness artifact in this harness. + monkeypatch.setattr(runs, "write_pid", lambda _run_dir: None) monkeypatch.setattr(cli, "_make_adapters", lambda *a, **k: {r: None for r in cli.ROLES}) return run_dir +def test_resume_rechecks_liveness_inside_the_state_lock(tmp_path, monkeypatch, capsys): + """Ablation: remove _resume_paused_run's in-lock liveness check and preparation + runs even though a rival resume published its pid while this caller waited.""" + import contextlib + + entered = False + + @contextlib.contextmanager + def recording_lock(_run_dir): + nonlocal entered + entered = True + yield + + monkeypatch.setattr(cli, "state_lock", recording_lock) + + def liveness(_run_dir): + assert entered + return "alive" + + monkeypatch.setattr(cli.runs, "engine_liveness", liveness) + monkeypatch.setattr(cli, "_prepare_resume_locked", lambda *_a: pytest.fail("double drove")) + + assert cli._resume_paused_run(tmp_path, tmp_path / "run") == 1 + assert "double-drive" in capsys.readouterr().err + + +def test_resume_liveness_and_publication_share_one_lock_acquisition(tmp_path, monkeypatch): + """The freshness check and preparation are one uninterrupted transaction. + + Ablation: split ``_resume_paused_run`` into consecutive lock blocks around the + liveness check and preparation; both in-lock assertions still pass, but the + acquisition-count assertion reddens because a rival can enter between them. + """ + import contextlib + + acquisitions = 0 + active = False + + @contextlib.contextmanager + def recording_lock(_run_dir): + nonlocal acquisitions, active + acquisitions += 1 + assert not active + active = True + try: + yield + finally: + active = False + + def liveness(_run_dir): + assert active + return "dead" + + def prepare(_project, _run_dir): + assert active + return 1 + + monkeypatch.setattr(cli, "state_lock", recording_lock) + monkeypatch.setattr(cli.runs, "engine_liveness", liveness) + monkeypatch.setattr(cli, "_prepare_resume_locked", prepare) + + assert cli._resume_paused_run(tmp_path, tmp_path / "run") == 1 + assert acquisitions == 1 + + +def test_resume_retains_outer_lock_through_pid_publication(project, monkeypatch): + run_dir = _paused_run_for_resume(project, monkeypatch) + publications: list[str] = [] + real_save = cli.save_state + + def checked_write_pid(target): + assert_run_state_lock_held(target) + publications.append("pid") + + def checked_save(target, state): + assert_run_state_lock_held(target) + publications.append("state") + real_save(target, state) + + monkeypatch.setattr(cli.runs, "write_pid", checked_write_pid) + monkeypatch.setattr(cli, "save_state", checked_save) + monkeypatch.setattr(cli, "Engine", _StubEngine) + + assert cli._resume_paused_run(project.project, run_dir) == 0 + assert publications == ["pid", "state"] + + def _state_reading_engine(seen): """A stub engine that records state.json as it stood when the engine started. _StubEngine never saves, so anything `seen` contains was written by the CLI diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index 51b62ecc..701c4e0b 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -279,12 +279,13 @@ def test_env_names_the_platform_and_the_win32_on_wsl_path_verdict(project, monke # `collect_env` reaches `get_multiplexer()`, an lru_cache(maxsize=1) that selects # on `sys.platform`; without these clears the patched window caches the Windows # pick for every later test in the worker. + run_dir = _seed_run(project.project) get_multiplexer.cache_clear() try: monkeypatch.setattr(diagnostics.sys, "platform", "win32") pseudo = sanitize.Pseudonymizer() unc = Path("\\\\wsl.localhost\\Ubuntu-24.04\\home\\u\\p") - diag = diagnostics.collect([_seed_run(project.project)], pseudo=pseudo, project=unc) + diag = diagnostics.collect([run_dir], pseudo=pseudo, project=unc) finally: # pytest undoes the patch on its own, but only at teardown — a raise in # `collect` would leave the Windows pick cached past this test without this. @@ -1872,10 +1873,10 @@ def test_events_degrade_to_the_legacy_root_when_the_state_root_is_underivable( count this degradation gives up.""" from bmad_loop import envvars, runs - monkeypatch.delenv(envvars.STATE_DIR, raising=False) - monkeypatch.setattr(runs, "state_root", _raise_no_state_root) run_dir = _seed_bare_run(project.project) _write_events(run_dir / "events", 2) + monkeypatch.delenv(envvars.STATE_DIR, raising=False) + monkeypatch.setattr(runs, "state_root", _raise_no_state_root) group = _events_group(run_dir, project.project) assert group is not None and group.count == 2 diff --git a/tests/test_engine.py b/tests/test_engine.py index d0cef3ad..9d5d6f88 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -11847,6 +11847,7 @@ def test_journal_log_position_covers_post_session_entries(project): ) def test_windows_console_ctrl_signal_is_ignored(project, monkeypatch, signal_name, fallback_signum): import bmad_loop.engine as engine_mod + from bmad_loop import journal as journal_mod signum = getattr(signal, signal_name, fallback_signum) if signal_name == "SIGBREAK": @@ -11865,6 +11866,14 @@ def fake_signal(sig, handler): return previous[sig] monkeypatch.setattr(engine_mod.sys, "platform", "win32") + + @contextlib.contextmanager + def native_test_lock(_path): + # This Linux-hosted test patches the process-wide sys.platform token only to + # drive Engine's Windows signal branch; msvcrt is intentionally unavailable. + yield + + monkeypatch.setattr(journal_mod, "file_lock", native_test_lock) monkeypatch.setattr(signal, "signal", fake_signal) monkeypatch.setattr(engine_mod, "kill_session", lambda rid: None) diff --git a/tests/test_journal.py b/tests/test_journal.py index d3c78b0d..7466da16 100644 --- a/tests/test_journal.py +++ b/tests/test_journal.py @@ -7,12 +7,14 @@ import os import stat +import threading +from contextlib import contextmanager import pytest from bmad_loop import journal as journal_mod -from bmad_loop import platform_util -from bmad_loop.journal import Journal, load_state, save_state +from bmad_loop import platform_util, runs +from bmad_loop.journal import Journal, load_state, save_state, state_lock from bmad_loop.model import RunState @@ -20,6 +22,7 @@ def test_save_state_retries_transient_sharing_violation(tmp_path, monkeypatch): """On win32, os.replace denied by a concurrent reader is retried, not fatal.""" monkeypatch.setattr(platform_util.sys, "platform", "win32") monkeypatch.setattr(platform_util.time, "sleep", lambda _s: None) # no real backoff + monkeypatch.setattr(journal_mod, "file_lock", contextmanager(lambda _path: iter((None,)))) real_replace = os.replace calls = {"n": 0} @@ -38,6 +41,220 @@ def flaky_replace(src, dst): assert load_state(tmp_path).run_id == "r1" +def test_state_lock_holds_the_canonical_run_sidecar(tmp_path): + run_dir = tmp_path / "run" + lock_path = runs.lock_path_for(run_dir / journal_mod.STATE_FILE, follow_final_symlink=False) + + with state_lock(run_dir): + with pytest.raises(OSError): + with platform_util.file_lock(lock_path, blocking=False): + pytest.fail("a rival acquired the held run-state sidecar") + + +def test_state_lock_same_run_nesting_acquires_os_lock_once(tmp_path, monkeypatch): + acquired: list[object] = [] + + @contextmanager + def recording_lock(path): + acquired.append(path) + yield + + monkeypatch.setattr(journal_mod, "file_lock", recording_lock) + + with state_lock(tmp_path): + with state_lock(tmp_path / "."): + save_state( + tmp_path, + RunState(run_id="r1", project="p", started_at="2026-09-01T00:00:00"), + ) + + assert acquired == [ + runs.lock_path_for(tmp_path / journal_mod.STATE_FILE, follow_final_symlink=False) + ] + + +def test_state_lock_same_run_symlink_spellings_acquire_os_lock_once(tmp_path, monkeypatch): + run_dir = tmp_path / "run" + run_dir.mkdir() + alias = tmp_path / "run-alias" + try: + alias.symlink_to(run_dir, target_is_directory=True) + except (NotImplementedError, OSError) as e: + pytest.skip(f"directory symlinks unavailable: {e}") + acquired: list[object] = [] + + @contextmanager + def recording_lock(path): + acquired.append(path) + yield + + monkeypatch.setattr(journal_mod, "file_lock", recording_lock) + + with state_lock(run_dir): + with state_lock(alias): + pass + + assert acquired == [ + runs.lock_path_for(run_dir / journal_mod.STATE_FILE, follow_final_symlink=False) + ] + + +def test_state_lock_identity_survives_replacing_a_final_state_symlink(tmp_path): + """Ablation: follow the final state.json symlink in state_lock and nested + save_state changes sidecars when atomic_replace replaces the link.""" + run_dir = tmp_path / "run" + run_dir.mkdir() + elsewhere = tmp_path / "elsewhere.json" + elsewhere.write_text("{}", encoding="utf-8") + state_path = run_dir / journal_mod.STATE_FILE + state_path.symlink_to(elsewhere) + logical_lock = runs.lock_path_for(state_path, follow_final_symlink=False) + + # The default remains referent-based for ledgers and every other caller. + assert runs.lock_path_for(state_path) == runs.lock_path_for(elsewhere) + assert runs.lock_path_for(state_path) != logical_lock + + with state_lock(run_dir): + save_state( + run_dir, + RunState(run_id="r1", project="p", started_at="2026-09-01T00:00:00"), + ) + assert not state_path.is_symlink() + with state_lock(run_dir / "."): + with pytest.raises(OSError): + with platform_util.file_lock(logical_lock, blocking=False): + pytest.fail("a rival acquired the original logical sidecar") + + assert elsewhere.read_text(encoding="utf-8") == "{}" + assert load_state(run_dir).run_id == "r1" + + +def test_state_lock_refuses_different_run_nesting_before_second_acquire(tmp_path, monkeypatch): + acquired: list[object] = [] + + @contextmanager + def recording_lock(path): + acquired.append(path) + yield + + monkeypatch.setattr(journal_mod, "file_lock", recording_lock) + + with state_lock(tmp_path / "one"): + with pytest.raises(RuntimeError, match="different runs"): + with state_lock(tmp_path / "two"): + pytest.fail("cross-run nesting was allowed") + + assert len(acquired) == 1 + + +def test_state_lock_failure_clears_thread_guard(tmp_path, monkeypatch): + acquired: list[object] = [] + + @contextmanager + def recording_lock(path): + acquired.append(path) + yield + + monkeypatch.setattr(journal_mod, "file_lock", recording_lock) + + with pytest.raises(ValueError, match="boom"): + with state_lock(tmp_path / "one"): + raise ValueError("boom") + with state_lock(tmp_path / "two"): + pass + + assert len(acquired) == 2 + + +def test_save_state_acquisition_error_writes_nothing(tmp_path, monkeypatch): + run_dir = tmp_path / "run" + + @contextmanager + def refusing_lock(_path): + raise OSError("lock unavailable") + yield + + monkeypatch.setattr(journal_mod, "file_lock", refusing_lock) + + with pytest.raises(OSError, match="lock unavailable"): + save_state( + run_dir, + RunState(run_id="r1", project="p", started_at="2026-09-01T00:00:00"), + ) + + assert not run_dir.exists() + + +def test_save_state_root_error_writes_nothing(tmp_path, monkeypatch): + run_dir = tmp_path / "run" + + def no_state_root(_path, **_kwargs): + raise runs.StateRootError("no state root") + + monkeypatch.setattr(runs, "lock_path_for", no_state_root) + + with pytest.raises(runs.StateRootError, match="no state root"): + save_state( + run_dir, + RunState(run_id="r1", project="p", started_at="2026-09-01T00:00:00"), + ) + + assert not run_dir.exists() + + +def test_two_concurrent_saves_never_share_the_fixed_temp_file(tmp_path, monkeypatch): + """Ablation: remove save_state's state_lock and the second replace enters while + the first is paused, so both calls race on state.json.tmp and one loses it.""" + real_replace = journal_mod.atomic_replace + real_file_lock = journal_mod.file_lock + first_entered = threading.Event() + second_attempted = threading.Event() + release_first = threading.Event() + replace_threads: list[str] = [] + + @contextmanager + def observed_file_lock(path): + if threading.current_thread().name == "second": + second_attempted.set() + with real_file_lock(path): + yield + + def controlled_replace(src, dst): + replace_threads.append(threading.current_thread().name) + if len(replace_threads) == 1: + first_entered.set() + assert release_first.wait(2) + real_replace(src, dst) + + monkeypatch.setattr(journal_mod, "atomic_replace", controlled_replace) + monkeypatch.setattr(journal_mod, "file_lock", observed_file_lock) + errors: list[BaseException] = [] + + def writer(run_id: str) -> None: + try: + save_state( + tmp_path, + RunState(run_id=run_id, project="p", started_at="2026-09-01T00:00:00"), + ) + except BaseException as e: + errors.append(e) + + first = threading.Thread(target=writer, args=("first",), name="first") + second = threading.Thread(target=writer, args=("second",), name="second") + first.start() + assert first_entered.wait(2) + second.start() + assert second_attempted.wait(2) + assert replace_threads == ["first"] + release_first.set() + first.join(2) + second.join(2) + + assert errors == [] + assert sorted(replace_threads) == ["first", "second"] + assert load_state(tmp_path).run_id in {"first", "second"} + + def _planted_verify_symlink(tmp_path): """A run dir whose `verify/` a session has already replaced with a link out.""" run_dir, elsewhere = tmp_path / "run", tmp_path / "elsewhere" diff --git a/tests/test_portability_guard.py b/tests/test_portability_guard.py index fd360b58..12e9cc23 100644 --- a/tests/test_portability_guard.py +++ b/tests/test_portability_guard.py @@ -185,27 +185,42 @@ SESSION_TASK_ID_CHOKEPOINT = {"engine.py": "_session_task_id"} # The complete set of ``runs.rearm_escalation`` call sites, as -# ``(file, enclosing function)``. The re-arm transaction's own commit probe -# (``runs._rearm_commit_landed``) proves "did MY save_state land?" with nothing but -# ``(generation, phase)`` over the reloaded task, and that is a sufficient IDENTITY -# only under a sole-writer model: no engine advancing the task underneath, and one -# control command at a time. Its docstring argues that model from this enumeration. -# -# Prose cannot hold it. A third call site — or either existing gate deleted — leaves -# every test in the repo green while the probe's premise quietly becomes false, and -# the failure it opens is DW-79/DW-83's own shape: a spec left re-armed against a task -# the run still calls ESCALATED. So the enumeration is scanned instead of asserted. -# -# Deliberately NOT a lock and not a durable per-re-arm token: the spec's ``Never`` -# forbids both (a lock only ``rearm_escalation`` takes excludes nobody; a token buys a -# precision ``save_state`` cannot honour). It forbids no guard, and this is the cheap -# half — it does not make overlapping callers safe, it makes the day someone adds one -# impossible to miss. Overlapping control commands stay out of the model, as DW-93. +# ``(file, enclosing function)``. Serialization now comes from the shared run-state +# lock, not this liveness inventory. The gates remain independently load-bearing: a +# serialized control command still must not take its turn after an engine known to be +# live, and a new operator surface must make that policy explicit. REARM_ESCALATION_CALLERS = { ("cli.py", "cmd_resolve"), ("tui/app.py", "_do_rearm"), } +# Every production state publication and every explicit multi-step state transaction. +# ``save_state`` itself serializes the leaf write, so a new direct publisher is safe +# from the fixed-temp collision but still appears here for review: if it reads state +# before deciding what to publish, it also belongs in RUN_STATE_TRANSACTIONS with an +# outer hold. Exact inventories make a newly added writer fail loudly instead of +# relying on a reviewer to find it by grep. +SAVE_STATE_CALLERS = { + ("cli.py", "_prepare_resume_locked"), + ("engine.py", "_save"), + ("runs.py", "_rearm_escalation_locked"), + ("runs.py", "restamp_code_root"), + ("runs.py", "_stop_run_once"), + ("runsetup.py", "compose_run"), + ("runsetup.py", "compose_sweep"), +} +RUN_STATE_TRANSACTIONS = { + ("cli.py", "_resume_paused_run"), + ("cli.py", "cmd_resolve"), + ("journal.py", "save_state"), + ("runs.py", "rearm_escalation"), + ("runs.py", "restamp_code_root"), + ("runs.py", "_stop_run_once"), + ("runsetup.py", "compose_run"), + ("runsetup.py", "compose_sweep"), + ("tui/app.py", "_do_rearm"), +} + # What counts as consulting liveness, matched as a substring of the callee's name # because the two sites legitimately spell it differently and neither spelling is more # correct: the CLI calls ``runs.engine_liveness`` directly, the TUI goes through @@ -2222,26 +2237,20 @@ def test_rearm_escalation_called_only_behind_a_liveness_gate(): """``runs.rearm_escalation`` is reached from exactly two places, and each consults liveness before it. - ``runs._rearm_commit_landed`` decides whether the re-arm transaction COMMITTED — - and therefore whether to roll the spec back — from ``(generation, phase)`` over the - reloaded task, nothing more. Those two conjuncts are a sufficient identity only - while ``rearm_escalation`` is the sole writer of that run's ``state.json``, and that - model is argued from this enumeration: two callers, each behind a liveness - consultation, with no engine running. A third caller, or either gate deleted, makes - the premise false — and the defect it reopens is DW-79/DW-83's own: a spec left - flipped against a task the run still calls ESCALATED. + ``runs._rearm_commit_landed`` is protected by the shared run-state transaction + lock, so this enumeration no longer supplies its writer-identity premise. It pins + the separate safety rule that an operator surface refuses a provably-live engine + before entering that serialized mutation turn. Note what the gate does and does not establish. It proves the engine is not PROVABLY alive, not that it is dead: ``"alive"`` is refused outright, while ``"unknown"`` proceeds under ``--force`` in ``cmd_resolve`` and counts as blocking in the TUI only for a pid-backed run. So this grades the falsifiable half — that - an earlier liveness decision BLOCKS fall-through before the call. The rest of the - model (one control command at a time) is out of scope here and tracked as DW-93. + an earlier liveness decision BLOCKS fall-through before the call. - ``cli.cmd_resume`` is deliberately absent: it writes this run's ``state.json`` - through ``_resume_paused_run``, so the sole-writer claim must account for it, but it - never re-arms and so is not a call site. Listing it here would make the enumeration - unfalsifiable in the direction that matters. + ``cli.cmd_resume`` is deliberately absent because it never re-arms. Its state + publication is covered separately by the writer/transaction inventory below; + listing it here would make this call-site enumeration unfalsifiable. ⚠️ What this assertion grades, precisely — the two halves differ, and the difference is the reason the probe rows below exist: @@ -2264,10 +2273,9 @@ def test_rearm_escalation_called_only_behind_a_liveness_gate(): sites = _rearm_callsite_counts(findings) declared = Counter(REARM_ESCALATION_CALLERS) assert sites == declared, ( - "the count of runs.rearm_escalation call sites moved. That enumeration is what " - "runs._rearm_commit_landed's (generation, phase) commit probe argues its " - "sole-writer premise from — a new caller needs that docstring revisited (and " - "DW-93 consulted), not this constant widened:\n" + "the count of runs.rearm_escalation call sites moved. A new operator surface " + "must retain the liveness refusal as well as the shared run-state transaction; " + "do not widen this constant without reviewing both:\n" f" scanned: {sorted(sites.elements())}\n" f" declared: {sorted(declared.elements())}" ) @@ -2279,6 +2287,29 @@ def test_rearm_escalation_called_only_behind_a_liveness_gate(): ) +def _production_call_sites(name: str) -> set[tuple[str, str | None]]: + sites: set[tuple[str, str | None]] = set() + for source in SRC.rglob("*.py"): + tree = ast.parse(source.read_text(encoding="utf-8")) + enclosing = _enclosing_function_names(tree) + rel = source.relative_to(SRC).as_posix() + for node in ast.walk(tree): + if isinstance(node, ast.Call) and _called_name(node.func) == name: + sites.add((rel, enclosing.get(id(node)))) + return sites + + +def test_run_state_writer_and_transaction_inventory_is_complete(): + """Every publisher uses save_state, and every known RMW gesture holds state_lock. + + Ablations: add ``save_state(run_dir, state)`` to a new production function, or + delete the outer ``state_lock`` from ``runs.restamp_code_root``; the respective + exact-set comparison reddens and names the changed site. + """ + assert _production_call_sites("save_state") == SAVE_STATE_CALLERS + assert _production_call_sites("state_lock") == RUN_STATE_TRANSACTIONS + + def _journal_field_offenders(findings) -> list[tuple[str, int, str, str]]: """The routing invariant as a filter, in the two directions a finding can fail: a field name that neither ``diagnostics`` nor the benign inventory accounts for, diff --git a/tests/test_runs.py b/tests/test_runs.py index 40b04062..577ca93e 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -15,7 +15,7 @@ from unittest import mock import pytest -from conftest import escalated_run, git, refuse_to_resolve +from conftest import assert_run_state_lock_held, escalated_run, git, refuse_to_resolve from bmad_loop import envvars, platform_util, runs, verify from bmad_loop.adapters import tmux_base @@ -769,6 +769,116 @@ def test_stop_run_fallback_clears_hard_request(tmp_path, monkeypatch): assert '"fallback": true' in (run_dir / "journal.jsonl").read_text() +def test_stop_run_takes_state_lock_only_after_signal_and_wait(tmp_path, monkeypatch): + """Ablation: move stop_run's state_lock above terminate and this reddens on order; + the engine would be unable to save its own stopped state while stop waits.""" + order: list[str] = [] + alive = True + run_dir = _make_state_run(tmp_path, "r1") + (run_dir / "engine.pid").write_text("4242 100.0") + + def on_terminate(_pid): + nonlocal alive + order.append("terminate") + alive = False + + host = _FakeHost(alive=lambda: alive, identity=100.0, on_terminate=on_terminate) + monkeypatch.setattr(runs, "get_process_host", lambda: host) + monkeypatch.setattr(runs, "kill_session", lambda _rid: order.append("kill-session")) + + @contextlib.contextmanager + def recording_state_lock(_run_dir): + order.append("state-lock") + yield + + monkeypatch.setattr(runs, "state_lock", recording_state_lock) + + assert runs.stop_run(run_dir) is True + assert order == ["terminate", "kill-session", "state-lock"] + + +def test_stop_run_fallback_save_retains_the_outer_state_lock(tmp_path, monkeypatch): + run_dir = _make_state_run(tmp_path, "r1") + monkeypatch.setattr(runs, "kill_session", lambda _rid: None) + real_save = runs.save_state + + def checked_save(target, state): + assert_run_state_lock_held(target) + real_save(target, state) + + monkeypatch.setattr(runs, "save_state", checked_save) + + assert runs.stop_run(run_dir) is True + assert load_state(run_dir).stopped is True + + +def test_stop_run_does_not_fallback_over_engine_completion(tmp_path, monkeypatch): + """The final locked snapshot wins when the engine finishes while stop delivers.""" + run_dir = _make_state_run(tmp_path, "r1") + (run_dir / "engine.pid").write_text("4242 100.0", encoding="utf-8") + monkeypatch.setattr(runs, "kill_session", lambda _rid: None) + + def finish(_pid): + state = load_state(run_dir) + state.finished = True + save_state(run_dir, state) + + monkeypatch.setattr( + runs, + "get_process_host", + lambda: _FakeHost(alive=False, identity=100.0, on_terminate=finish), + ) + + assert runs.stop_run(run_dir) is False + persisted = load_state(run_dir) + assert persisted.finished is True + assert persisted.stopped is False + journal = run_dir / "journal.jsonl" + assert not journal.exists() or '"fallback": true' not in journal.read_text() + + +def test_stop_run_retries_when_resume_publishes_a_new_engine(tmp_path, monkeypatch): + """A rival resume that wins during delivery is itself sent the hard stop.""" + run_dir = _make_state_run(tmp_path, "r1") + (run_dir / "engine.pid").write_text("4242 100.0", encoding="utf-8") + monkeypatch.setattr(runs, "kill_session", lambda _rid: None) + alive = {4242: True, 5252: True} + identities = {4242: 100.0, 5252: 200.0} + + class GenerationalHost(_FakeHost): + def __init__(self): + super().__init__(alive=False) + + def is_alive(self, pid): + return alive.get(pid, False) + + def identity(self, pid): + return identities.get(pid) + + def terminate(self, pid): + self.terminated.append(pid) + alive[pid] = False + if pid == 4242: + # Resume clears the old gesture, then publishes its new pid and + # state atomically under the lock stop will next acquire. + with runs.state_lock(run_dir): + runs.clear_graceful_stop(run_dir) + rival = load_state(run_dir) + rival.crashed = True + (run_dir / "engine.pid").write_text("5252 200.0", encoding="utf-8") + save_state(run_dir, rival) + + host = GenerationalHost() + monkeypatch.setattr(runs, "get_process_host", lambda: host) + + assert runs.stop_run(run_dir) is True + assert host.terminated == [4242, 5252] + persisted = load_state(run_dir) + assert persisted.stopped is True + assert persisted.crashed is True + assert runs.read_stop_request_mode(run_dir) is None + + def test_stop_run_engine_confirmed_leaves_nothing_pending(tmp_path, monkeypatch): """When the engine confirms the stop itself the request is consumed too. The engine normally clears it on the way out; this is the belt-and-braces half, and @@ -2781,6 +2891,43 @@ def test_restamp_code_root_aims_the_mirror_the_rearm_reads(tmp_path, recorded): assert message is None +def test_restamp_code_root_reloads_after_a_rival_writer(tmp_path, monkeypatch): + """Ablation: move restamp_code_root's load above state_lock and the rival's + ``crashed`` update is overwritten by the stale snapshot.""" + run = escalated_run(tmp_path, "r1", story_key="s1") + save_state(run.run_dir, run.state) + + @contextlib.contextmanager + def rival_first(_run_dir): + rival = load_state(run.run_dir) + rival.crashed = True + save_state(run.run_dir, rival) + yield + + monkeypatch.setattr(runs, "state_lock", rival_first) + + runs.restamp_code_root(run.run_dir, tmp_path / "new-code") + + persisted = load_state(run.run_dir) + assert persisted.crashed is True + assert persisted.code_root == tmp_path / "new-code" + + +def test_restamp_code_root_retains_outer_lock_through_save(tmp_path, monkeypatch): + run = escalated_run(tmp_path, "r1", story_key="s1") + save_state(run.run_dir, run.state) + real_save = runs.save_state + + def checked_save(target, state): + assert_run_state_lock_held(target) + real_save(target, state) + + monkeypatch.setattr(runs, "save_state", checked_save) + + runs.restamp_code_root(run.run_dir, tmp_path / "new-code") + assert load_state(run.run_dir).code_root == tmp_path / "new-code" + + _SPEC_WITH_ARR = ( "---\ntitle: t\nstatus: blocked\noperator_actions:\n" " - publish the TXT record\n---\n\n## Intent\n\nbody\n" @@ -2831,6 +2978,63 @@ def test_rearm_plain_mode_sets_ready_for_dev_and_clears_stale_latch(tmp_path): assert entry["restore"] is False +def test_rearm_reloads_state_after_waiting_for_the_run_lock(tmp_path, monkeypatch): + """Ablation: load state before rearm_escalation's state_lock and this stale + gesture re-arms after the rival has already completed it.""" + from bmad_loop.model import Phase + + run_dir, spec = _escalated_run(tmp_path, _SPEC_WITH_ARR) + original = spec.read_bytes() + + @contextlib.contextmanager + def rival_first(_run_dir): + rival = load_state(run_dir) + rival.tasks["1-1-a"].phase = Phase.PENDING + save_state(run_dir, rival) + yield + + monkeypatch.setattr(runs, "state_lock", rival_first) + + with pytest.raises(runs.RearmError, match="is not escalated"): + runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) + + assert spec.read_bytes() == original + + +def test_rearm_lock_acquisition_failure_leaves_spec_and_state_unchanged(tmp_path, monkeypatch): + from bmad_loop import journal as journal_mod + + run_dir, spec = _escalated_run(tmp_path, _SPEC_WITH_ARR) + spec_before = spec.read_bytes() + state_before = (run_dir / journal_mod.STATE_FILE).read_bytes() + + @contextlib.contextmanager + def refusing_lock(_path): + raise OSError("state lock unavailable") + yield + + monkeypatch.setattr(journal_mod, "file_lock", refusing_lock) + + with pytest.raises(OSError, match="state lock unavailable"): + runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) + + assert spec.read_bytes() == spec_before + assert (run_dir / journal_mod.STATE_FILE).read_bytes() == state_before + + +def test_rearm_locked_body_retains_outer_lock_through_state_save(tmp_path, monkeypatch): + run_dir, _spec = _escalated_run(tmp_path, _SPEC_WITH_ARR) + real_save = runs.save_state + + def checked_save(target, state): + assert_run_state_lock_held(target) + real_save(target, state) + + monkeypatch.setattr(runs, "save_state", checked_save) + + runs.rearm_escalation(run_dir, isolated_redrive=False, resolution_recorded=True) + + def test_rearm_aborts_when_the_spec_status_cannot_be_reopened(tmp_path): """The seam that proves the silent-`False` defect mattered. This spec reads as `status: blocked` — the reader resolves the block scalar fine — so it clears diff --git a/tests/test_runsetup.py b/tests/test_runsetup.py index f98a0e77..b1a37a79 100644 --- a/tests/test_runsetup.py +++ b/tests/test_runsetup.py @@ -16,16 +16,19 @@ import dataclasses import shutil +import threading import types +from contextlib import contextmanager from pathlib import Path import pytest from bmad_loop import bmadconfig +from bmad_loop import journal as journal_mod from bmad_loop import policy as policy_mod from bmad_loop import runs, runsetup from bmad_loop.adapters.profile import ProfileError -from bmad_loop.journal import Journal, load_state +from bmad_loop.journal import Journal, load_state, state_lock # A profile overlay carrying the whole launch surface the digest covers. It lives # under .bmad-loop/profiles/, inside the tree every driven session can write. @@ -436,6 +439,96 @@ def test_composition_persists_the_code_root(tmp_path, run_type): assert persisted.code_root != Path(persisted.project) +@pytest.mark.parametrize("run_type", ["run", "sweep"]) +def test_initial_state_and_pid_are_one_locked_publication(tmp_path, monkeypatch, run_type): + """A rival explicit-id resume cannot enter after state.json becomes readable + but before the fresh composer publishes engine.pid.""" + run_dir = runs.run_dir_for(tmp_path, RUN_ID) + stamp_entered = threading.Event() + rival_attempted = threading.Event() + release_stamp = threading.Event() + rival_observed: list[tuple[bool, bool]] = [] + errors: list[BaseException] = [] + real_file_lock = journal_mod.file_lock + + @contextmanager + def observed_file_lock(path, *args, **kwargs): + if threading.current_thread().name == "rival-resume": + rival_attempted.set() + with real_file_lock(path, *args, **kwargs): + yield + + def paused_stamp(_project, _run_id, _digest): + stamp_entered.set() + assert release_stamp.wait(2) + + monkeypatch.setattr(journal_mod, "file_lock", observed_file_lock) + monkeypatch.setattr(runs, "write_trusted_config_digest", paused_stamp) + + def compose() -> None: + try: + if run_type == "run": + runsetup.compose_run( + project=tmp_path, + paths=_fake_paths(tmp_path), + policy=policy_mod.loads(""), + run_id=RUN_ID, + epic_filter=None, + story_filter=None, + max_stories=None, + stories_on=False, + spec_folder="", + sweep_factory=lambda _trigger, *, started: None, + make_adapters=_accepting_adapters, + engine_cls=_AcceptingEngine, + stories_engine_cls=_AcceptingEngine, + trusted_config_digest="deadbeef", + ) + else: + runsetup.compose_sweep( + project=tmp_path, + paths=_fake_paths(tmp_path), + policy=policy_mod.loads(""), + run_id=RUN_ID, + prompting=False, + decisions_only=False, + max_bundles=None, + repeat=None, + max_cycles=None, + trigger="auto", + make_adapters=_accepting_adapters, + sweep_engine_cls=_AcceptingEngine, + trusted_config_digest="deadbeef", + ) + except BaseException as e: + errors.append(e) + + def rival_resume() -> None: + try: + with state_lock(run_dir): + rival_observed.append( + ((run_dir / "state.json").is_file(), (run_dir / "engine.pid").is_file()) + ) + except BaseException as e: + errors.append(e) + + composer = threading.Thread(target=compose, name="composer") + rival = threading.Thread(target=rival_resume, name="rival-resume") + composer.start() + assert stamp_entered.wait(2) + assert (run_dir / "state.json").is_file() + assert not (run_dir / "engine.pid").exists() + rival.start() + assert rival_attempted.wait(2) + assert rival_observed == [] + release_stamp.set() + composer.join(2) + rival.join(2) + + assert errors == [] + assert rival_observed == [(True, True)] + + @pytest.fixture def unwinding(tmp_path): """A project plus a `make_adapters` that fails the way the real one does. diff --git a/tests/test_tui_app.py b/tests/test_tui_app.py index c8ad2620..c50a9371 100644 --- a/tests/test_tui_app.py +++ b/tests/test_tui_app.py @@ -16,6 +16,7 @@ import pytest from conftest import ( + assert_run_state_lock_held, git, install_bmad_config, make_validate_document, @@ -3496,7 +3497,11 @@ def _stories_paused_run( if blocked_result: body += f"\n## Auto Run Result\n\n- Status: blocked\n\n{blocked_result}\n" spec.write_text(body, encoding="utf-8") - task = StoryTask(story_key=story_key, epic=0, phase=Phase.DEV_VERIFY) + task = StoryTask( + story_key=story_key, + epic=0, + phase=Phase.ESCALATED if stage == "escalation" else Phase.DEV_VERIFY, + ) task.spec_file = str(spec) if worktree_path: task.worktree_path = worktree_path @@ -5184,6 +5189,187 @@ def fake_rearm(rd, sk, *, isolated_redrive=False, resolution_recorded=False, pro assert any("the code root in _bmad/bmm/config.yaml has changed" in n for n in notes) +def test_escalation_rearm_rechecks_liveness_inside_state_lock(project, monkeypatch): + """Ablation: delete _do_rearm's second liveness check and the TUI re-arms after + a rival resume published its pid while this gesture waited for the state lock.""" + from bmad_loop import runs + + install_bmad_config(project) + run_dir = project.project / ".bmad-loop" / "runs" / "20260611-100000-aaaa" + checks: list[str] = [] + + def liveness_gate(_self, _run_id, _run_dir): + checks.append("checked") + return len(checks) == 2 + + monkeypatch.setattr(BmadLoopApp, "_resolve_blocked_by_liveness", liveness_gate) + monkeypatch.setattr(runs, "rearm_escalation", lambda *_a, **_k: pytest.fail("double re-armed")) + + BmadLoopApp(project.project)._do_rearm(run_dir.name, run_dir, "1") + + assert checks == ["checked", "checked"] + + +def test_escalation_rearm_reloads_state_before_restamping(project, monkeypatch): + """Ablation: delete _do_rearm's fresh state check and the TUI restamps a run + whose escalation a rival already consumed while this gesture waited for the lock.""" + import contextlib + + from bmad_loop import runs + from bmad_loop.journal import load_state, save_state + from bmad_loop.tui import app as app_mod + + install_bmad_config(project) + run_dir, _spec = _stories_paused_run( + project.project, + stage="escalation", + spec_status="blocked", + spec_checkpoint=False, + blocked_result="Blocked: rival resolved this escalation.", + ) + notes: list[str] = [] + monkeypatch.setattr(BmadLoopApp, "_resolve_blocked_by_liveness", lambda *_a: False) + monkeypatch.setattr( + BmadLoopApp, + "notify", + lambda _self, message, **_kwargs: notes.append(str(message)), + ) + + @contextlib.contextmanager + def rival_first(_run_dir): + rival = load_state(run_dir) + rival.tasks["1"].phase = Phase.PENDING + save_state(run_dir, rival) + yield + + monkeypatch.setattr(app_mod, "state_lock", rival_first) + monkeypatch.setattr(runs, "restamp_code_root", lambda *_a: pytest.fail("stale restamp")) + monkeypatch.setattr(runs, "rearm_escalation", lambda *_a, **_k: pytest.fail("double re-armed")) + + BmadLoopApp(project.project)._do_rearm(run_dir.name, run_dir, "1") + + assert any("no longer paused at escalation" in note for note in notes) + + +def test_escalation_rearm_refuses_a_newer_generation_from_an_open_review(project, monkeypatch): + """An old modal must not consume a later escalation for the same story. + + Ablation: delete ``_do_rearm``'s generation comparison and the rival's newer + escalation reaches ``rearm_escalation`` even though the modal never displayed it. + """ + import contextlib + + from bmad_loop import runs + from bmad_loop.journal import load_state, save_state + from bmad_loop.tui import app as app_mod + + install_bmad_config(project) + run_dir, _spec = _stories_paused_run( + project.project, + stage="escalation", + spec_status="blocked", + spec_checkpoint=False, + blocked_result="Blocked: the original escalation.", + ) + expected_generation = load_state(run_dir).tasks["1"].generation + notes: list[str] = [] + monkeypatch.setattr(BmadLoopApp, "_resolve_blocked_by_liveness", lambda *_a: False) + monkeypatch.setattr( + BmadLoopApp, + "notify", + lambda _self, message, **_kwargs: notes.append(str(message)), + ) + + @contextlib.contextmanager + def rival_first(_run_dir): + rival = load_state(run_dir) + rival.tasks["1"].generation = expected_generation + 1 + save_state(run_dir, rival) + yield + + monkeypatch.setattr(app_mod, "state_lock", rival_first) + monkeypatch.setattr(runs, "restamp_code_root", lambda *_a: pytest.fail("stale restamp")) + monkeypatch.setattr(runs, "rearm_escalation", lambda *_a, **_k: pytest.fail("stale rearm")) + + BmadLoopApp(project.project)._do_rearm( + run_dir.name, + run_dir, + "1", + expected_generation=expected_generation, + ) + + assert any("changed while its review was open" in note for note in notes) + + +def test_escalation_rearm_retains_outer_lock_through_rearm_call(project, monkeypatch): + from bmad_loop import runs + + install_bmad_config(project) + run_dir, _spec = _stories_paused_run( + project.project, + stage="escalation", + spec_status="blocked", + spec_checkpoint=False, + blocked_result="Blocked: needs a human decision.", + ) + rearms: list[Path] = [] + monkeypatch.setattr(BmadLoopApp, "_resolve_blocked_by_liveness", lambda *_a: False) + + def checked_rearm(rd, key, **_kwargs): + assert_run_state_lock_held(rd) + rearms.append(rd) + return _rearm_outcome(key) + + monkeypatch.setattr(runs, "rearm_escalation", checked_rearm) + app = BmadLoopApp(project.project) + monkeypatch.setattr(app, "notify", lambda *_a, **_k: None) + monkeypatch.setattr(app, "_do_resume", lambda _run_id: None) + + app._do_rearm(run_dir.name, run_dir, "1") + + assert rearms == [run_dir] + + +@pytest.mark.parametrize( + "failure", + [OSError("lock unavailable"), runs_mod.StateRootError("no usable state root")], +) +def test_escalation_rearm_reports_state_lock_failures(project, monkeypatch, failure): + import contextlib + + from bmad_loop import runs + from bmad_loop.tui import app as app_mod + + install_bmad_config(project) + run_dir, _spec = _stories_paused_run( + project.project, + stage="escalation", + spec_status="blocked", + spec_checkpoint=False, + blocked_result="Blocked: needs a human decision.", + ) + notes: list[tuple[str, str | None]] = [] + monkeypatch.setattr(BmadLoopApp, "_resolve_blocked_by_liveness", lambda *_a: False) + monkeypatch.setattr(runs, "rearm_escalation", lambda *_a, **_k: pytest.fail("wrote unlocked")) + + @contextlib.contextmanager + def refusing_lock(_run_dir): + raise failure + yield + + monkeypatch.setattr(app_mod, "state_lock", refusing_lock) + app = BmadLoopApp(project.project) + monkeypatch.setattr( + app, + "notify", + lambda message, **kwargs: notes.append((str(message), kwargs.get("severity"))), + ) + + app._do_rearm(run_dir.name, run_dir, "1") + + assert notes == [(f"re-arm failed: {failure}", "error")] + + async def test_escalation_rearm_refuses_the_isolation_conflict_before_it_mutates( project, monkeypatch ): From 8aebd849b3fe1607aa6ce1760cace37de192f966 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 1 Sep 2026 20:03:02 -0700 Subject: [PATCH 11/18] sweep dw5-run-lifecycle-resume-exclusion: DW-94 via bmad-loop --- CHANGELOG.md | 4 + README.md | 10 +- docs/FEATURES.md | 9 +- src/bmad_loop/cli.py | 33 +++-- src/bmad_loop/runs.py | 93 +++++++++++--- src/bmad_loop/runsetup.py | 34 ++++-- src/bmad_loop/tui/app.py | 14 +-- tests/test_cleanup.py | 61 ++++++++++ tests/test_cli.py | 78 +++++++++++- tests/test_portability_guard.py | 2 + tests/test_runs.py | 209 +++++++++++++++++++++++++++++++- tests/test_runsetup.py | 98 ++++++++++++++- tests/test_tui_app.py | 48 ++++++++ 13 files changed, 635 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9395a19..6a9fcc02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -270,6 +270,10 @@ breaking changes may land in a minor release. unrecoverably, on a gitignored ledger. The mark and append legs now re-anchor only when the preimage they wrote over is still the bytes the run last claimed; the restore skips and journals `ledger-restore-skipped-diverged`. +- Serialize run deletion/archive against resume (DW-94), refusing a newly live + engine under the per-run lock and preventing a waiting resume from recreating a + run cleanup already removed. + - Serialize every run-state writer and control read-modify-write transaction with one canonical per-run advisory lock (DW-93). - Let interactive resolve present `paused_reason` when watermark filtering leaves no newer diff --git a/README.md b/README.md index 05cacb27..9a3c86fe 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ bmad-loop tui # …or drive everything from the dashboard | `bmad-loop adapters` | List registered coding-CLI adapter **kinds** — name, builtin/external, whether the family drives a multiplexer, and which profiles select each — the CLI axis's counterpart to `mux`. Unlike `mux` there is no global choice to persist: a kind is selected per profile by its `adapter` field. A profile naming an unregistered kind, and any out-of-tree adapter/profile package that failed to load, get a `warning:` on stderr. | | `bmad-loop run` | Drive the dev → review → verify → commit loop. `--epic N`, `--story KEY`, `--max-stories N`, `--dry-run`. `--spec ` forces **stories mode** (folder+id dispatch off `/stories.yaml`), overriding `[stories].source`; `--story` then filters by story id. | | `bmad-loop sweep` | Triage + execute open `deferred-work.md` entries. `--no-prompt`, `--decisions-only`, `--max-bundles N`, `--repeat`, `--max-cycles N`, `--dry-run`. `--archive [--before DATE]` instead moves closed ledger entries to `deferred-work-archive.md`, leaving id-preserving stubs. | -| `bmad-loop resume ` | Continue a run paused at a gate, escalation, or interruption. | +| `bmad-loop resume ` | Continue a run paused at a gate, escalation, or interruption. The resume command rendezvouses with delete/archive on the run lifecycle lock; if cleanup removed the run while resume waited, resume reports it missing without recreating files or launching an engine. | | `bmad-loop resolve ` | Resolve a CRITICAL escalation: open an interactive resolve agent to fix the frozen spec, then re-arm the story and resume. On an _intent gap_ the re-drive can resume review on the attempted change instead of re-implementing it. `--story KEY`, `--no-interactive`, `--restore-patch ` (intent-gap patch-restore), `--resume` / `--no-resume`, `--force` (proceed when engine liveness is unverifiable; a provably-live engine still blocks). | | `bmad-loop decisions` | Answer deferred-work decisions earlier sweeps left unanswered (skipped by `--no-prompt`, or an abandoned interactive sweep). Recorded so the next sweep acts on them without re-asking. `--list` shows them without answering; `--json` emits them as a stable machine-readable document — id, question, context, recommendation, and every option's key/label/effect/intent/resolution/bundle-name with a derived `recommended` flag. It implies the listing and never prompts, so a script can select an option by policy instead of scraping the text. | | `bmad-loop confirm ` | Complete a story parked at `awaiting-operator` once you have carried out the external actions it owes (buy the domain, publish the DNS record). Acknowledges each action in turn, writes the spec's `## Operator Confirmation` audit section, advances spec and board to `done`, and commits the pair — nothing is re-driven. `--list` shows every parked story and what it owes; `--yes` skips the prompts; `--reverify` re-runs the project's `[verify]` commands first and blocks the confirmation if they fail; `--json` emits the parked set as a stable machine-readable document. Every write is checked and the spec is read back from disk, so a story is never declared done over a write that did not land; a confirmation interrupted before its board write is **finished** by re-running the command, with no second prompt and no second audit section. The index it reads is machine-local, so a park is confirmed on the machine that ran it. | @@ -91,10 +91,10 @@ bmad-loop tui # …or drive everything from the dashboard | `bmad-loop diagnose []` (`diag`) | Emit a **sanitized** diagnostic dump of a run/sweep to hand maintainers when reporting a bug — phase/token/session histograms, escalation counts, adapter/model, env, and run-dir file sizes, with no code, spec content, prompts, transcripts, paths, or PII. Identifiers are pseudonymized to stable per-dump aliases and the output is re-scanned by a fail-closed leak check before writing; a stray pseudonymized identifier is auto-substituted with its alias (disclosed in the report), while PII/secret hits still refuse to emit. Defaults to the latest run. `--all`, `--out`, `--max-journal-entries N`; `--json` emits the dump as a stable JSON document (one object on stdout, no fences) instead of the markdown report. | | `bmad-loop attach []` | tmux-attach to a run's live agent session. | | `bmad-loop stop ` | Stop a live run — the engine and its agent tmux session. `--graceful` instead finishes the in-flight item (a story through commit, a sweep bundle through commit), then stops cleanly and stays resumable, and suppresses pending auto-sweeps; `--cancel-graceful` withdraws a pending request. The hard stop is the default and always wins over a pending graceful one. | -| `bmad-loop delete ` | Delete a run directory. `--force` stops the run first if it is still live. | -| `bmad-loop archive ` | Compress a run into `.bmad-loop/archive` and remove the run dir. `--force` stops the run first if it is still live. | +| `bmad-loop delete ` | Delete a run directory. `--force` stops the run first if it is still live, but cannot override a rival resume that becomes live before removal. Unknown liveness still warns and proceeds. | +| `bmad-loop archive ` | Compress a run into `.bmad-loop/archive` and remove the run dir. `--force` stops the run first if it is still live, but cannot override a rival resume that becomes live before archival. Unknown liveness still warns and proceeds. | | `bmad-loop cleanup` | Remove leftover tmux artifacts **for the current project**: kill `bmad-loop-` sessions for finished/stopped/interrupted runs (and orphans whose run dir is gone) and close parked `bmad-loop-ctl` windows. `--dry-run` lists without killing. Live runs — and any session/window belonging to another project — are never touched. `--json` emits a stable machine-readable document instead of the text — the run ids whose sessions were removed, the live ids left alone, the ctl windows closed, and a `dry_run` flag — so a preview and the real run share one schema and can be compared. | -| `bmad-loop clean` | Reclaim **disk** from concluded runs per `[cleanup]`: tear down git worktrees a mid-flight stop orphaned (freeing their Unity `Library/` + MCP-server builds), trim the heavy `worktrees/` tree from runs kept for history (they stay viewable in the TUI), and archive/delete runs past the retention window. Only finished/stopped runs are touched; `--dry-run` previews, `--keep ` protects, `--retain N` overrides the window, `--hard` deletes instead of archiving. `--json` emits a stable machine-readable document instead of the text — the effective retention policy, `freed_bytes` as a raw integer, and the worktree paths and run ids reclaimed, trimmed, archived, deleted or protected. | +| `bmad-loop clean` | Reclaim **disk** from concluded runs per `[cleanup]`: tear down git worktrees a mid-flight stop orphaned (freeing their Unity `Library/` + MCP-server builds), trim the heavy `worktrees/` tree from runs kept for history (they stay viewable in the TUI), and archive/delete runs past the retention window. Only finished/stopped runs are touched; a run that resumes during clean is recorded as protected or trimmed according to work already done, and siblings continue. `--dry-run` previews, `--keep ` protects, `--retain N` overrides the window, `--hard` deletes instead of archiving. `--json` emits one stable machine-readable document instead of the text — the effective retention policy, `freed_bytes` as a raw integer, and the worktree paths and run ids reclaimed, trimmed, archived, deleted or protected. | | `bmad-loop tui` | The interactive dashboard (needs the `[tui]` extra). `--low-frame-rate` caps it to 15fps + disables animations (fixes repaint tearing over slow/SSH links; also `[tui] low_frame_rate`). | | `bmad-loop probe-adapter ` (`collect-adapter-data`) | Collect + sanitize the data needed to finalize a CLI adapter profile (hook payload shape, transcript location/format, token schema). Default is a zero-launch **scan**; `--probe` opts into a live capture (`--model` picks the probe turn's model, `--timeout` bounds it, default 90s). `--transcript`, `--session-dir`, `--binary` (CLIs with no profile yet), `--out`; `--json` emits the finding as a stable JSON document instead of the report. See the [adapter authoring guide](docs/adapter-authoring-guide.md). | @@ -170,7 +170,7 @@ Press **`g`** to edit `.bmad-loop/policy.toml` in a form grouped by section — | `a` | attach to the live agent session (or the orchestrator window) | | `x` | stop the selected live run immediately (engine + agent session) | | `S` | graceful stop: finish the in-flight item (through commit), then stop cleanly — stays resumable | -| `D` / `A` | delete / archive the selected run (force-stops a live run first) | +| `D` / `A` | delete / archive the selected run (refuses while its engine is live) | | `c` | clean up tmux sessions/windows for finished & stopped runs | | `v` | run `bmad-loop validate`, output in a modal | | `g` | settings editor for `.bmad-loop/policy.toml` | diff --git a/docs/FEATURES.md b/docs/FEATURES.md index f130c34f..f14cf9a1 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -300,6 +300,7 @@ Result` section. Every other spec keeps warn-and-continue, and the record says w - `bmad-loop clean` reclaims **disk** (distinct from `cleanup`, which is only tmux). It tears down git worktrees a mid-flight stop left mounted — the main accumulation source: each carries a real Unity `Library/` (incl. the MCP-server build), which `git worktree remove` cannot reach once the engine was killed before teardown. It then trims the heavy `worktrees/` tree from runs kept for history (the run still lists in the dashboard — discovery reads `state.json`, not the worktree), and archives or deletes runs past the retention window. - It also collects the **out-of-tree** half of a run. Removing a run dir no longer removes everything the run owns (#494), so `delete`/`archive`/`clean` remove the run's control-plane dir under the state root too, and `clean` additionally sweeps this project's orphans there — subtrees whose run dir is gone, from a hand-removed run or a delete that predates this. The sweep keys on the run directory _existing_, not on its `state.json` parsing, so a corrupt run an operator is trying to recover keeps its control plane; a trimmed run keeps its own for the same reason (it is still resumable). Their bytes are not in the reclaim estimate — a state dir holds consumed event files, the run's `config-digest` (#498), and little else. Known limit: the state root is keyed by the project's resolved path, so a project that is deleted, moved or renamed leaves its old subtree unsweepable — after a move the project keys somewhere new, and no project can name the old key. A move does not cost the run its config-change baseline, though: `state.json` carries a second copy that travels with the run directory, and `resume` falls back to it exactly when the out-of-tree file is out of reach (#498). - Safe by construction: only **finished or stopped** runs are touched; running, unknown-host, paused and interrupted (resumable) runs are never reclaimed. `--keep ` protects a specific run (e.g. a finished one whose Editor is still live), `--dry-run` previews, `--retain N`/`--hard` tune the window and archive-vs-delete. +- Delete/archive and resume serialize on the same per-run lifecycle lock. Cleanup re-checks engine liveness after acquiring it and holds exclusion through archive snapshot/publication, run removal, and control-plane cleanup; a provably live engine refuses even under `--force`, while unverifiable liveness remains warn-and-proceed. If cleanup wins first, a waiting resume re-checks existence inside the hold and reports the run missing without recreating files or constructing an engine. `clean` records a racing refusal per run (protected when untouched, trimmed when earlier reclaim steps already ran) and continues with sibling candidates; the TUI reports the refusal and keeps the run visible. - `--json` emits a stable machine-readable document per the [contract below](#machine-readable-output---json) (schema-versioned; the effective retention policy, `freed_bytes` as a raw integer, and the paths and run ids under `worktrees`/`trimmed`/`archived`/`deleted`/`protected`, and `state_dirs_swept` as a count) instead of the text. Plan and outcome share one schema, with `dry_run` saying which one you are holding, so a script can pre-flight a reclaim and compare it against what happened — though values are each invocation's own sample, not a promise the two agree. It names every item the text only counts or renders, and the unverifiable-pid warning text mode writes to stderr becomes `unverifiable_pid` in the document, leaving stderr empty. - Prevention is automatic: every `run`/`sweep` start reconciles worktrees leaked by a prior **finished** run (`[cleanup] auto_clean_on_finish`), and the Unity plugin's `post_run` hook removes the IvanMurzak MCP server's downloaded `/tmp///*.zip` and truncates its unbounded editor log (`[cleanup] clean_tmp`). For recurring housekeeping of stopped runs, schedule `bmad-loop clean`. @@ -318,7 +319,7 @@ Result` section. Every other spec keeps warn-and-continue, and the record says w - `bmad-loop adapters` — list registered coding-CLI adapter **kinds** (name · builtin/external · whether the family drives a multiplexer · which profiles select it), the CLI axis's counterpart to `mux`. Unlike `mux` there is no global choice to persist: a kind is selected per profile by its `adapter` field. A profile referencing an unregistered kind, and any out-of-tree adapter/profile package that failed to load, get a `warning:` on stderr; `validate` reports the same as `adapter.kind` / `adapter.external` / `adapter.external-profile`. - `bmad-loop run` — drive the dev → review → verify → commit loop. - `bmad-loop sweep` — triage + execute open deferred-work entries. -- `bmad-loop resume ` — continue a paused/interrupted run. A resume is fresh intent, so a stop request the prior run left behind is discarded first, in either mode — and if it cannot be removed, resume refuses and names the file rather than re-arming into a run that would stop again at its first item. +- `bmad-loop resume ` — continue a paused/interrupted run. A resume is fresh intent, so a stop request the prior run left behind is discarded first, in either mode — and if it cannot be removed, resume refuses and names the file rather than re-arming into a run that would stop again at its first item. Resume also rendezvouses with delete/archive on the run's lifecycle lock; when cleanup removed the run while resume waited, it reports `no such run` before any state helper can recreate the directory. - `bmad-loop resolve ` — resolve a CRITICAL escalation, then re-arm + resume (`--story`, `--no-interactive`, `--restore-patch ` for intent-gap patch-restore, `--resume`/`--no-resume`). - `bmad-loop decisions` — answer deferred-work decisions past sweeps left unanswered (`--list` to just show them). `--json` instead emits a stable machine-readable document (schema-versioned; per decision the id, question, context, recommendation and every option's key/label/effect/intent/resolution/bundle-name plus a derived `recommended` flag) per the [contract below](#machine-readable-output---json); it implies the listing and never prompts, and nothing pending yields a valid empty document. - `bmad-loop confirm ` — complete a story parked at `awaiting-operator` once you have carried out the external actions it owes: acknowledges each in turn (`--yes` skips the prompts), writes the spec's `## Operator Confirmation` audit section, advances spec and board to `done`, and commits the pair. `--list` shows every parked story and what it owes; `--reverify` re-runs your `[verify]` commands first and blocks on failure; re-running it on an interrupted confirmation finishes that confirmation. `--json` emits a stable machine-readable document per the [contract below](#machine-readable-output---json) — per parked story the key, actions, spec file, spec/board status, the parking run and the `commit` carrying the park (empty until the record is in a commit), plus derived `confirmable`/`resumable` flags, the `confirmation_recorded` reading behind the latter, and a human `drift` reason; it implies the listing and never prompts, and nothing parked yields a valid empty document. @@ -327,11 +328,11 @@ Result` section. Every other spec keeps warn-and-continue, and the record says w - `bmad-loop diagnose []` (`diag`) — emit a sanitized diagnostic dump of a run/sweep to hand maintainers (histograms, counts, env, file sizes — no code/spec/prompts/paths/PII); a stray pseudonymized identifier is auto-substituted with its alias (disclosed in the report), while PII/secret hits still refuse to emit; defaults to the latest run (`--all`, `--out`, `--max-journal-entries`). `--json` emits the same dump as a stable machine-readable document per the [contract below](#machine-readable-output---json) instead of the markdown report. - `bmad-loop attach []` — tmux-attach to a run's live agent session. - `bmad-loop stop ` — stop a live run. The default is a **hard stop**: stop now, abandoning the in-flight item and killing the agent session. `--graceful` instead requests a **graceful stop** — the engine finishes the in-flight item (a story through commit, a sweep bundle through commit, or an in-progress sweep triage — after which no bundles start), then finalizes cleanly and stops as a resumable `stopped` run, suppressing any pending auto-sweeps; `--cancel-graceful` withdraws a pending request. Both modes ride the same `stop-request.json` control file, which carries the mode: a hard stop lodges `mode: "hard"` **before** it signals, and that atomic, project-confined write is also what supersedes a pending graceful request. The engine honors a hard request at the next item boundary and mid-session, where each adapter's wait loop polls it twice per iteration — before and after the loop's own up-to-5s wait — so a quiet session normally lands the stop well inside the 10s grace window. That is the common case rather than a bound: an iteration blocked on a transport call, or waiting out `RESULT_GRACE_S` for an artifact, can exceed the window on either adapter before the next poll — an in-flight socket read or tmux call cannot be interrupted from the polling thread, so no placement of the check makes the interval unconditionally short. What the file does guarantee is reach: a hard stop lands on every platform and multiplexer backend, including one where an inter-process signal is never delivered at all (#319). A nested auto-sweep runs inside its parent but mints its own run dir, so it also polls the _owning_ run's channel: stopping the parent stops the child mid-session rather than leaving it to the force-kill backstop. The child's read of the parent channel is hard-only — a graceful stop already keeps a child sweep from starting, and lets one already in flight finish. Teardown is unbounded on top of that — the opencode HTTP adapter then asks the server to abort and to report usage, and a server that will not answer those leaves the stop to the force-kill backstop, exactly as it would have before #319. SIGTERM still goes out alongside it as the POSIX fast path, but it is no longer the mechanism; the engine stays the single writer of `stopped`, and the external force-kill + `run-stop fallback=True` past the grace window now marks a stop this tool had to finish from outside — a teardown that outran the window reaches it as readily as an engine that never read the request — where before #319 it marked every native-Windows stop. The TUI surfaces the same pair: `x` hard-stops, `S` requests a graceful stop. -- `bmad-loop delete ` — delete a run directory and its out-of-tree control-plane dir (`--force` stops it first if live). -- `bmad-loop archive ` — compress a run into `.bmad-loop/archive` and remove it, control-plane dir included (`--force` stops it first if live). The tarball holds the run dir, so it carries no `events/`. It is staged through an exclusively created temp under a fresh unpredictable name per attempt, so a planted name is never followed or reused, the failure cleanup is provably its own, and a temp stranded by a kill cannot deny later attempts; the tarball is `fsync`ed before the publish — the run dir is removed immediately after, so it is the only remaining copy. A published archive lands at mode `0600` rather than a umask-derived one (#591). +- `bmad-loop delete ` — delete a run directory and its out-of-tree control-plane dir (`--force` stops it first if live). The destructive transaction re-checks liveness after acquiring the per-run lock, so force cannot remove a run a rival resume claimed after that stop; unknown liveness still warns and proceeds. +- `bmad-loop archive ` — compress a run into `.bmad-loop/archive` and remove it, control-plane dir included (`--force` stops it first if live). The destructive transaction re-checks liveness after acquiring the per-run lock, so force cannot archive a run a rival resume claimed after that stop; unknown liveness still warns and proceeds. The hold covers tar snapshot, durable publication, source removal, and control-plane cleanup. The tarball holds the run dir, so it carries no `events/`. It is staged through an exclusively created temp under a fresh unpredictable name per attempt, so a planted name is never followed or reused, the failure cleanup is provably its own, and a temp stranded by a kill cannot deny later attempts; the tarball is `fsync`ed before the publish — the run dir is removed immediately after, so it is the only remaining copy. A published archive lands at mode `0600` rather than a umask-derived one (#591). - Removal refuses while a matching agent session is live that the project cannot prove is another one's, even when the engine is dead: for an untagged session the run dir is the last ownership proof `cleanup` can read, so removing it would leak the session ([#419](https://github.com/bmad-code-org/bmad-loop/issues/419)). A session tagged to another project carries its own proof and never blocks. Run `cleanup` first, having confirmed the session is this project's (`attach`): for an _untagged_ session `cleanup` proves ownership by that same run dir, so two projects sharing a run id can prune each other's. Or pass `--force`, which removes anyway and kills nothing. `clean` leaves such a run untouched and reports it as protected. - `bmad-loop cleanup` — remove leftover tmux artifacts for finished/stopped runs. `--json` emits the sessions and ctl windows removed (or, with `--dry-run`, that would be) as a stable machine-readable document per the [contract below](#machine-readable-output---json). -- `bmad-loop clean` — reclaim disk from concluded runs per `[cleanup]`: tear down worktrees a mid-flight stop orphaned, trim heavy `worktrees/` from runs kept for history, archive/delete past the retention window, and sweep orphaned run control-plane dirs from the out-of-tree state root (`--dry-run`, `--keep`, `--retain N`, `--hard`). `--json` emits what was reclaimed (or would be) as a stable machine-readable document per the [contract below](#machine-readable-output---json), with `freed_bytes` a raw integer. +- `bmad-loop clean` — reclaim disk from concluded runs per `[cleanup]`: tear down worktrees a mid-flight stop orphaned, trim heavy `worktrees/` from runs kept for history, archive/delete past the retention window, and sweep orphaned run control-plane dirs from the out-of-tree state root (`--dry-run`, `--keep`, `--retain N`, `--hard`). A run that resumes before its final removal is classified as protected or trimmed according to work already completed, and unrelated candidates continue. `--json` emits what was reclaimed (or would be) as one stable machine-readable document per the [contract below](#machine-readable-output---json), with `freed_bytes` a raw integer. - `bmad-loop tui` — the interactive dashboard (`--low-frame-rate` for slow/SSH links). - `bmad-loop probe-adapter ` (`collect-adapter-data`) — collect + sanitize adapter-finalization data for a CLI profile; default zero-launch scan, opt-in `--probe` live capture. - Every command takes `--project ` (default: current directory). Any `` accepts a partial — the tail after the last `-`, shortened to any unique prefix. diff --git a/src/bmad_loop/cli.py b/src/bmad_loop/cli.py index 8371f6d5..268cca56 100644 --- a/src/bmad_loop/cli.py +++ b/src/bmad_loop/cli.py @@ -2811,6 +2811,12 @@ def _prepare_resume_locked(project: Path, run_dir: Path): def _resume_paused_run(project: Path, run_dir: Path) -> int: """Resume a paused/interrupted run without holding its lock across execution.""" with state_lock(run_dir): + # Cleanup removes the run under this same hold. A resume that resolved the + # path before cleanup won must not let Journal/save_state recreate it after + # its wait ends. + if not runs.is_run(run_dir): + print(f"no such run: {run_dir.name}", file=sys.stderr) + return 1 # Repeat the command's liveness decision after exclusion. A concurrent # resume publishes its pid under this same hold, so the waiter refuses # instead of reloading the predecessor's old paused state and double-driving. @@ -4099,6 +4105,9 @@ def cmd_delete(args: argparse.Namespace) -> int: return rc try: runs.delete_run(project, run_dir, force=args.force) + except runs.LiveEngineError as e: + print(str(e), file=sys.stderr) + return 1 except runs.LiveSessionError as e: print(f"{e} (or pass --force)", file=sys.stderr) return 1 @@ -4119,6 +4128,9 @@ def cmd_archive(args: argparse.Namespace) -> int: return rc try: dest = runs.archive_run(project, run_dir, force=args.force) + except runs.LiveEngineError as e: + print(str(e), file=sys.stderr) + return 1 except runs.LiveSessionError as e: print(f"{e} (or pass --force)", file=sys.stderr) return 1 @@ -4389,15 +4401,11 @@ def cmd_clean(args: argparse.Namespace) -> int: if not dry: runs.archive_run(project, run_dir) archived.append(run_dir.name) - except runs.LiveSessionError: - # A session appeared between the loop-top guard and here — a resume - # of a stopped run, racing this clean. The chokepoint refused the - # removal; record the run instead of letting one racing run abort - # the whole invocation. Correct the estimate down to what actually - # went. The wider race — every mutation in this loop against a - # concurrent resume — is older than this guard (`reclaimable` is - # sampled in the loop above and never re-read) and is tracked in - # issue #533. + except (runs.LiveEngineError, runs.LiveSessionError) as e: + # A session or engine appeared between the loop-top sample and the + # authoritative removal transaction. Record this run instead of + # letting one racer abort the whole invocation, then continue with + # its siblings. Correct the estimate down to what actually went. freed += heavy_bytes - run_bytes # Classify by what happened, not by what was intended: the steps # above may already have taken this run's worktree and artifacts, @@ -4406,8 +4414,13 @@ def cmd_clean(args: argparse.Namespace) -> int: # trimmed, which is exactly the state it ends in. (trimmed if run_worktrees or shrunk else protected).append(run_dir.name) if not args.json: + reason = ( + "agent session appeared mid-clean" + if isinstance(e, runs.LiveSessionError) + else "engine resumed mid-clean" + ) print( - f"run {run_dir.name}: agent session appeared mid-clean — not removed", + f"run {run_dir.name}: {reason} — not removed", file=sys.stderr, ) elif pol.cleanup.trim_artifacts: diff --git a/src/bmad_loop/runs.py b/src/bmad_loop/runs.py index c71e97c2..8baf3f6e 100644 --- a/src/bmad_loop/runs.py +++ b/src/bmad_loop/runs.py @@ -122,6 +122,16 @@ class LiveSessionError(Exception): message the CLI/TUI surface verbatim.""" +class LiveEngineError(Exception): + """A destructive run-lifecycle transaction found a provably live engine. + + Unlike :class:`LiveSessionError`, this refusal is authoritative even when the + operator requested ``force``: force may stop the engine before entering the + transaction, but it never licenses removing a run a rival resume claimed in + the meantime. ``str()`` is the operator-facing message surfaces report. + """ + + # How long stop_run waits for a signalled engine to exit before falling back to # marking the run stopped itself. _STOP_WAIT_S = 10.0 @@ -2583,10 +2593,24 @@ def _refuse_uncontained_run_dir(project: Path, run_dir: Path, action: str) -> No node = parent -def delete_run(project: Path, run_dir: Path, *, force: bool = False) -> None: - """Permanently remove a run directory. Callers enforce the engine-liveness - guard; the session guard is enforced here (see :func:`_refuse_live_session`), - which raises :class:`LiveSessionError` instead of removing. +def delete_run( + project: Path, + run_dir: Path, + *, + force: bool = False, + _expected_composer_pid: int | None = None, + _expected_composer_claim: os.stat_result | None = None, +) -> None: + """Permanently remove a run directory under one lifecycle transaction. + + Engine liveness is re-checked after acquiring the canonical per-run state + lock and a provably live engine raises :class:`LiveEngineError`. The private + ``_expected_composer_pid`` and ``_expected_composer_claim`` escape exists only + for ``runsetup``'s failed launch unwind: that composer may remove its own live + pid publication only while the freshly read pid and the directory it + exclusively created still match. It does not bypass the independent + live-session guard, and a rival pid publication or replacement directory + refuses the unwind. ``force`` is the operator's explicit override and skips that guard, accepting the leak on their own say-so. It deliberately does not kill the session @@ -2598,21 +2622,48 @@ def delete_run(project: Path, run_dir: Path, *, force: bool = False) -> None: the operator accepting a leaked session, never a licence to rmtree a path outside the runs dir.""" _refuse_uncontained_run_dir(project, run_dir, "delete") - if not force: - _refuse_live_session(project, run_dir.name, "delete") - shutil.rmtree(run_dir) - # after the run dir, never before: a raise above leaves the run whole, and a - # whole run keeps its control plane (see _discard_state_dir). - _discard_state_dir(project, run_dir.name) + with state_lock(run_dir): + if _expected_composer_claim is not None: + try: + current_claim = run_dir.stat(follow_symlinks=False) + except OSError as e: + raise LiveEngineError( + f"run {run_dir.name} changed directory ownership — refusing to delete it" + ) from e + if not os.path.samestat(_expected_composer_claim, current_claim): + raise LiveEngineError( + f"run {run_dir.name} changed directory ownership — refusing to delete it" + ) + published_pid = read_pid(run_dir) + if ( + _expected_composer_pid is not None + and published_pid is not None + and published_pid != _expected_composer_pid + ): + raise LiveEngineError( + f"run {run_dir.name} changed engine ownership — refusing to delete it" + ) + if _expected_composer_pid is None and engine_liveness(run_dir) == "alive": + raise LiveEngineError( + f"run {run_dir.name} is still live — refusing to delete it; stop it first" + ) + if not force: + _refuse_live_session(project, run_dir.name, "delete") + shutil.rmtree(run_dir) + # after the run dir, never before: a raise above leaves the run whole, and a + # whole run keeps its control plane (see _discard_state_dir). + _discard_state_dir(project, run_dir.name) def archive_run(project: Path, run_dir: Path, *, force: bool = False) -> Path: """Compress a run dir into .bmad-loop/archive/.tar.gz and remove the original. The tarball is written to a temp path then atomically replaced into - place so a partial archive never appears. Callers enforce the engine-liveness - guard; the session guard is enforced here (see :func:`_refuse_live_session`, - and :func:`delete_run` for ``force``) and runs before the tarball is written, - so a refusal leaves nothing behind. + place so a partial archive never appears. Engine liveness is re-checked under + the canonical per-run state lock; that exclusion remains held through archive + publication, source removal, and control-state discard. The session guard is + enforced here too (see :func:`_refuse_live_session` and :func:`delete_run` for + ``force``), before any archive path is created, so a refusal leaves nothing + behind. The tarball holds the run dir only, so since #494 an archive no longer carries the run's ``events/``: the channel moved out of the tree, and its files are @@ -2624,8 +2675,18 @@ def archive_run(project: Path, run_dir: Path, *, force: bool = False) -> Path: for the reason the session guard runs early: a refusal must leave no archive directory and no tarball behind.""" _refuse_uncontained_run_dir(project, run_dir, "archive") - if not force: - _refuse_live_session(project, run_dir.name, "archive") + with state_lock(run_dir): + if engine_liveness(run_dir) == "alive": + raise LiveEngineError( + f"run {run_dir.name} is still live — refusing to archive it; stop it first" + ) + if not force: + _refuse_live_session(project, run_dir.name, "archive") + return _archive_run_locked(project, run_dir) + + +def _archive_run_locked(project: Path, run_dir: Path) -> Path: + """Archive ``run_dir`` while its caller owns :func:`state_lock`.""" archive_dir = project / ARCHIVE_DIR archive_dir.mkdir(parents=True, exist_ok=True) dest = archive_dir / f"{run_dir.name}.tar.gz" diff --git a/src/bmad_loop/runsetup.py b/src/bmad_loop/runsetup.py index 4b4e258d..3e701fc0 100644 --- a/src/bmad_loop/runsetup.py +++ b/src/bmad_loop/runsetup.py @@ -32,6 +32,7 @@ import hashlib import json +import os import sys import time from contextlib import suppress @@ -830,7 +831,7 @@ class ComposedRun: journal: Journal -def _claim_run_dir(run_dir: Path) -> None: +def _claim_run_dir(run_dir: Path) -> os.stat_result: """Take exclusive ownership of a fresh run directory, refusing an id that already names a run. @@ -869,9 +870,23 @@ def _claim_run_dir(run_dir: Path) -> None: f"error: run {run_dir.name} already exists — refusing to compose over it. " "`--run-id` must name a run that does not exist yet." ) from e + try: + return run_dir.stat(follow_symlinks=False) + except BaseException: + # The directory is still empty and exclusively ours. If its identity + # cannot be captured, take the just-published claim back here because the + # outer composition unwind cannot safely identify it without the token. + with suppress(OSError): + run_dir.rmdir() + raise -def _unwind_composition(project: Path, run_dir: Path, journal: Journal | None) -> None: +def _unwind_composition( + project: Path, + run_dir: Path, + journal: Journal | None, + composer_claim: os.stat_result, +) -> None: """Remove the run a failed ``compose_*`` had already published, so a launch that aborts partway leaves nothing behind. @@ -929,7 +944,12 @@ def _unwind_composition(project: Path, run_dir: Path, journal: Journal | None) - effect: the operator reads the launch error, and nothing anywhere says the cleanup after it did not happen.""" try: - runs.delete_run(project, run_dir) + runs.delete_run( + project, + run_dir, + _expected_composer_pid=os.getpid(), + _expected_composer_claim=composer_claim, + ) except Exception as e: detail = f"{type(e).__name__}: {e}" print( @@ -999,7 +1019,7 @@ def compose_run( run_dir = project / RUNS_DIR / run_id # Outside the try below, and it must stay there: a collision refusal that # reached `_unwind_composition` would delete the run it exists to protect. - _claim_run_dir(run_dir) + composer_claim = _claim_run_dir(run_dir) # Composition is atomic from the first published artifact onward: everything # below either lands whole or is unwound (see :func:`_unwind_composition`, # which also states why the arm is `BaseException` and not `Exception`). @@ -1064,7 +1084,7 @@ def compose_run( else engine_cls(**common) # pyright: ignore[reportArgumentType] ) except BaseException: - _unwind_composition(project, run_dir, journal) + _unwind_composition(project, run_dir, journal, composer_claim) raise return ComposedRun(engine=engine, run_id=run_id, run_dir=run_dir, state=state, journal=journal) @@ -1147,7 +1167,7 @@ def compose_sweep( run_id = run_id or runs.new_run_id() run_dir = project / RUNS_DIR / run_id # Same claim, same reason, same placement outside the try as in `compose_run`. - _claim_run_dir(run_dir) + composer_claim = _claim_run_dir(run_dir) # Atomic from the first published artifact onward, exactly as in `compose_run` # — same reason, same opening on the statement after the claim, and one more # artifact to unwind (`sweep.json`). @@ -1204,7 +1224,7 @@ def compose_sweep( if on_started is not None: on_started() except BaseException: - _unwind_composition(project, run_dir, journal) + _unwind_composition(project, run_dir, journal, composer_claim) raise return ComposedRun(engine=engine, run_id=run_id, run_dir=run_dir, state=state, journal=journal) diff --git a/src/bmad_loop/tui/app.py b/src/bmad_loop/tui/app.py index 3b54d202..e840c8c9 100644 --- a/src/bmad_loop/tui/app.py +++ b/src/bmad_loop/tui/app.py @@ -1428,10 +1428,10 @@ def done(ok: bool | None) -> None: def _delete_run_worker(self, run_id: str, run_dir: Path) -> None: try: runs.delete_run(self.project, run_dir) - except (OSError, runs.LiveSessionError) as e: - # LiveSessionError is the #419 backstop: the confirm above gates on engine - # liveness, which an orphaned session passes. Surface it like any other - # failed removal rather than letting it kill the worker thread. + except (OSError, runs.StateRootError, runs.LiveEngineError, runs.LiveSessionError) as e: + # The modal's liveness sample is advisory. Surface authoritative + # lifecycle refusals and lock/removal failures here rather than letting + # them kill the worker thread or forgetting a run that still exists. self.call_from_thread(self.notify, f"delete failed: {e}", severity="error") return self.call_from_thread(self._dashboard.forget_run, run_id) @@ -1467,9 +1467,9 @@ def done(ok: bool | None) -> None: def _archive_run_worker(self, run_id: str, run_dir: Path) -> None: try: dest = runs.archive_run(self.project, run_dir) - except (OSError, runs.LiveSessionError) as e: - # see _delete_run_worker: the confirm's guard is engine-keyed, this one - # is session-keyed (#419). + except (OSError, runs.StateRootError, runs.LiveEngineError, runs.LiveSessionError) as e: + # Same worker boundary as delete: report the authoritative transaction, + # not the earlier modal sample. self.call_from_thread(self.notify, f"archive failed: {e}", severity="error") return self.call_from_thread(self._dashboard.forget_run, run_id) diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index 553d8c1b..d36d913c 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -456,6 +456,67 @@ def racing(_project, run_id): assert doc["archived"] == [] and doc["deleted"] == [] +@pytest.mark.parametrize( + "hard, helper_name, result_key, other_key", + [ + (False, "archive_run", "archived", "deleted"), + (True, "delete_run", "deleted", "archived"), + ], +) +def test_cmd_clean_json_records_a_resumed_engine_and_continues_siblings( + project, monkeypatch, capsys, hard, helper_name, result_key, other_key +): + """A runs-layer lifecycle refusal is per-run data, never a partial JSON + document or an abort that prevents later candidates from being reclaimed.""" + install_bmad_config(project) + repo = project.project + racer = repo / ".bmad-loop" / "runs" / "20260101-000000-aaaa" + sibling = repo / ".bmad-loop" / "runs" / "20260101-000001-bbbb" + for run_dir in (racer, sibling): + save_state( + run_dir, + RunState(run_id=run_dir.name, project=str(repo), started_at="x", finished=True), + ) + real_cleanup = getattr(runs, helper_name) + + def racing_cleanup(project_path, run_dir): + if run_dir == racer: + raise runs.LiveEngineError("engine resumed") + return real_cleanup(project_path, run_dir) + + monkeypatch.setattr(runs, helper_name, racing_cleanup) + + extra = ("--hard",) if hard else () + doc = _clean_json(repo, capsys, "--retain", "0", *extra) + + assert doc["protected"] == [racer.name] + assert doc[result_key] == [sibling.name] + assert doc[other_key] == [] + assert racer.is_dir() and not sibling.exists() + + +def test_cmd_clean_text_identifies_a_resumed_engine(project, monkeypatch, capsys): + """Text mode distinguishes an engine resume from an agent-session race.""" + install_bmad_config(project) + repo = project.project + racer = repo / ".bmad-loop" / "runs" / "20260101-000000-aaaa" + save_state( + racer, + RunState(run_id=racer.name, project=str(repo), started_at="x", finished=True), + ) + + def racing_archive(_project, _run_dir): + raise runs.LiveEngineError("engine resumed") + + monkeypatch.setattr(runs, "archive_run", racing_archive) + + assert cli.cmd_clean(_clean_args(repo, retain=0)) == 0 + + _out, err = capsys.readouterr() + assert f"run {racer.name}: engine resumed mid-clean — not removed" in err + assert "agent session appeared mid-clean" not in err + + def test_cmd_clean_reclaims_past_a_session_proven_to_be_another_project_s( project, monkeypatch, capsys ): diff --git a/tests/test_cli.py b/tests/test_cli.py index a1c33411..041063b8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2304,7 +2304,8 @@ def test_delete_force_stops_then_removes(tmp_path, monkeypatch, capsys): from bmad_loop import runs stopped = [] - monkeypatch.setattr(runs, "engine_liveness", lambda _rd: "alive") + samples = iter(("alive", "dead")) + monkeypatch.setattr(runs, "engine_liveness", lambda _rd: next(samples)) monkeypatch.setattr(runs, "stop_run", lambda rd: stopped.append(rd) or True) run_dir = _make_run_with_state(tmp_path, "r1") assert cli.main(["delete", "--project", str(tmp_path), "r1", "--force"]) == 0 @@ -2313,6 +2314,32 @@ def test_delete_force_stops_then_removes(tmp_path, monkeypatch, capsys): assert not run_dir.exists() +@pytest.mark.parametrize("command", ["delete", "archive"]) +def test_force_cannot_remove_a_run_that_resumes_after_the_stop( + tmp_path, monkeypatch, capsys, command +): + """The outer force stop is not an override for the authoritative in-lock + liveness refusal, and its error must not advise retrying with force. + + Ablation: gate the runs-layer probe on ``not force`` and the run disappears. + Verified. + """ + samples = iter(("alive", "alive")) + stopped: list[Path] = [] + monkeypatch.setattr(runs, "engine_liveness", lambda _rd: next(samples)) + monkeypatch.setattr(runs, "stop_run", lambda rd: stopped.append(rd) or True) + run_dir = _make_run_with_state(tmp_path, "r1") + + assert cli.main([command, "--project", str(tmp_path), "r1", "--force"]) == 1 + + err = capsys.readouterr().err + assert "still live" in err and "refusing to" in err + assert "pass --force" not in err + assert stopped == [run_dir] + assert run_dir.is_dir() + assert not (tmp_path / ".bmad-loop" / "archive").exists() + + def test_delete_force_stop_error_blocks(tmp_path, monkeypatch, capsys): # a failed --force stop must propagate, never fall through to deletion from bmad_loop import runs @@ -5376,6 +5403,7 @@ def recording_lock(_run_dir): yield monkeypatch.setattr(cli, "state_lock", recording_lock) + monkeypatch.setattr(cli.runs, "is_run", lambda _run_dir: True) def liveness(_run_dir): assert entered @@ -5420,6 +5448,7 @@ def prepare(_project, _run_dir): return 1 monkeypatch.setattr(cli, "state_lock", recording_lock) + monkeypatch.setattr(cli.runs, "is_run", lambda _run_dir: True) monkeypatch.setattr(cli.runs, "engine_liveness", liveness) monkeypatch.setattr(cli, "_prepare_resume_locked", prepare) @@ -5427,6 +5456,53 @@ def prepare(_project, _run_dir): assert acquisitions == 1 +@pytest.mark.parametrize("operation", [runs.delete_run, runs.archive_run]) +def test_resume_waiter_refuses_a_run_cleanup_removed_without_recreating_it( + tmp_path, monkeypatch, capsys, operation +): + """Cleanup-first ordering: after the waiter enters the lifecycle hold it + checks existence before liveness, Journal construction, adapters, or engine + drive can recreate anything. + + Ablation: remove or move the existence gate below preparation and the injected + preparation failure fires. Verified. + """ + import contextlib + + run_dir = _make_run_with_state(tmp_path, "r1") + + @contextlib.contextmanager + def cleanup_won(_run_dir): + operation(tmp_path, run_dir) + yield + + monkeypatch.setattr(cli, "state_lock", cleanup_won) + + def liveness(target): + if target.exists(): + return "dead" + pytest.fail("resume read liveness after cleanup removed the run") + + monkeypatch.setattr( + cli.runs, + "engine_liveness", + liveness, + ) + monkeypatch.setattr( + cli, "_prepare_resume_locked", lambda *_a: pytest.fail("resume recreated the run") + ) + monkeypatch.setattr( + cli.runsetup, "compose_resume", lambda **_k: pytest.fail("resume built an engine") + ) + + assert cli._resume_paused_run(tmp_path, run_dir) == 1 + + assert "no such run: r1" in capsys.readouterr().err + assert not run_dir.exists() + if operation is runs.archive_run: + assert (tmp_path / ".bmad-loop" / "archive" / "r1.tar.gz").is_file() + + def test_resume_retains_outer_lock_through_pid_publication(project, monkeypatch): run_dir = _paused_run_for_resume(project, monkeypatch) publications: list[str] = [] diff --git a/tests/test_portability_guard.py b/tests/test_portability_guard.py index 12e9cc23..8e6dae3c 100644 --- a/tests/test_portability_guard.py +++ b/tests/test_portability_guard.py @@ -216,6 +216,8 @@ ("runs.py", "rearm_escalation"), ("runs.py", "restamp_code_root"), ("runs.py", "_stop_run_once"), + ("runs.py", "archive_run"), + ("runs.py", "delete_run"), ("runsetup.py", "compose_run"), ("runsetup.py", "compose_sweep"), ("tui/app.py", "_do_rearm"), diff --git a/tests/test_runs.py b/tests/test_runs.py index 577ca93e..a6f0e838 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -2522,6 +2522,127 @@ def test_delete_run(tmp_path): assert not run_dir.exists() +def test_delete_run_refuses_a_live_engine_inside_the_lifecycle_hold(tmp_path, monkeypatch): + """Resume-first ordering: the decisive probe runs after lock acquisition and + leaves both the run and its control plane untouched. + + Ablation: remove the in-lock ``engine_liveness`` gate and the run disappears; + move it above ``state_lock`` and the lock assertion fails. Verified. + """ + run_dir = _make_state_run(tmp_path, "r1") + state_dir = _seed_state_dir(tmp_path, "r1") + + def live(target): + assert_run_state_lock_held(target) + return "alive" + + monkeypatch.setattr(runs, "engine_liveness", live) + monkeypatch.setattr( + runs, + "_refuse_live_session", + lambda *_args: pytest.fail("session guard ran before authoritative engine probe"), + ) + with pytest.raises(runs.LiveEngineError, match="refusing to delete"): + runs.delete_run(tmp_path, run_dir) + + assert run_dir.is_dir() + assert state_dir.is_dir() + + +def test_delete_run_holds_lifecycle_lock_through_removal_and_state_discard(tmp_path, monkeypatch): + """The authoritative probe, directory removal, and control-plane tail are one + uninterrupted transaction. Moving either mutation outside the hold reddens. + """ + run_dir = _make_state_run(tmp_path, "r1") + removed: list[str] = [] + real_rmtree = runs.shutil.rmtree + + def dead(target): + assert_run_state_lock_held(target) + return "dead" + + def checked_rmtree(target, *args, **kwargs): + assert_run_state_lock_held(run_dir) + removed.append("run") + return real_rmtree(target, *args, **kwargs) + + def checked_discard(_project, _run_id): + assert_run_state_lock_held(run_dir) + removed.append("state") + + monkeypatch.setattr(runs, "engine_liveness", dead) + monkeypatch.setattr(runs.shutil, "rmtree", checked_rmtree) + monkeypatch.setattr(runs, "_discard_state_dir", checked_discard) + + runs.delete_run(tmp_path, run_dir) + + assert removed == ["run", "state"] + + +def test_failed_composition_pid_bypasses_only_its_own_live_engine(tmp_path, monkeypatch): + """The narrow composer token keeps the independent session guard active.""" + run_dir = _make_state_run(tmp_path, "r1") + composer_claim = run_dir.stat(follow_symlinks=False) + runs.write_pid(run_dir) + checked: list[str] = [] + + def session_guard(_project, run_id, _action): + checked.append(run_id) + + monkeypatch.setattr(runs, "_refuse_live_session", session_guard) + runs.delete_run( + tmp_path, + run_dir, + _expected_composer_pid=os.getpid(), + _expected_composer_claim=composer_claim, + ) + + assert checked == ["r1"] + assert not run_dir.exists() + + +@pytest.mark.parametrize("operation", [runs.delete_run, runs.archive_run]) +def test_lifecycle_containment_is_checked_before_lock_acquisition(tmp_path, monkeypatch, operation): + """A hostile path is refused without deriving or taking any state lock.""" + project = tmp_path / "project" + outside = tmp_path / "outside" + outside.mkdir() + (outside / "state.json").write_text("{}", encoding="utf-8") + + def unexpected_lock(_run_dir): + pytest.fail("containment must run before state-lock acquisition") + + monkeypatch.setattr(runs, "state_lock", unexpected_lock) + with pytest.raises(platform_util.UnconfinedWriteError): + operation(project, outside) + + assert outside.is_dir() + + +def test_delete_run_refuses_before_removal_when_state_lock_cannot_be_named(tmp_path, monkeypatch): + """The lifecycle lock is mandatory: without a state root cleanup cannot + rendezvous with resume, so failure leaves the run intact and surfaces.""" + run_dir = _make_state_run(tmp_path, "r1") + monkeypatch.setattr(runs, "state_root", _raising(runs.StateRootError("no root"))) + + with pytest.raises(runs.StateRootError, match="no root"): + runs.delete_run(tmp_path, run_dir) + + assert run_dir.is_dir() + + +def test_archive_run_refuses_before_staging_when_state_lock_cannot_be_named(tmp_path, monkeypatch): + """Archive cannot stage or remove anything when its mandatory lock fails.""" + run_dir = _make_state_run(tmp_path, "r1") + monkeypatch.setattr(runs, "state_root", _raising(runs.StateRootError("no root"))) + + with pytest.raises(runs.StateRootError, match="no root"): + runs.archive_run(tmp_path, run_dir) + + assert run_dir.is_dir() + assert not (tmp_path / ".bmad-loop" / "archive").exists() + + def test_delete_run_removes_the_out_of_tree_state_counterpart(tmp_path): """#494 moved the events channel out of the project tree, so removing the run dir stopped removing everything the run owns. Without this tail every delete @@ -2539,19 +2660,18 @@ def test_delete_run_removes_the_out_of_tree_state_counterpart(tmp_path): @pytest.mark.parametrize( "attr, exc", [ - ("state_root", runs.StateRootError("no root")), ("project_tag", OSError("cannot canonicalize")), ("project_tag", RuntimeError("Symlink loop from '/p'")), ], - ids=["no-derivable-state-root", "unresolvable-project", "symlink-loop-project"], + ids=["unresolvable-project", "symlink-loop-project"], ) def test_delete_run_survives_a_counterpart_it_cannot_name(tmp_path, monkeypatch, attr, exc): """The counterpart removal is a never-raise tail (#139 teardown doctrine). - Every row is the counterpart being *unnameable*, which is the only failure - that can escape: an environment with no derivable state root, and a project - the OS refuses to canonicalize (#552). Removal failures are absorbed - separately, by `ignore_errors`. + Every row is a project the OS refuses to canonicalize (#552) only after the + mandatory lifecycle lock was named. Removal failures are absorbed separately, + by `ignore_errors`. A missing state root now refuses before removal because + cleanup cannot safely rendezvous with resume without that lock. The `RuntimeError` row is not a hypothetical type: `project_tag` resolves before digesting, and below 3.13 `Path.resolve` reports a symlink loop as @@ -4461,6 +4581,80 @@ def test_archive_run(tmp_path): assert "20260611-100000-aaaa/journal.jsonl" in names +def test_archive_run_refuses_a_live_engine_before_staging(tmp_path, monkeypatch): + """Resume-first ordering leaves source, destination, and control state whole.""" + run_dir = _make_state_run(tmp_path, "20260611-100000-aaaa") + state_dir = _seed_state_dir(tmp_path, run_dir.name) + + def live(target): + assert_run_state_lock_held(target) + return "alive" + + monkeypatch.setattr(runs, "engine_liveness", live) + monkeypatch.setattr( + runs, + "_refuse_live_session", + lambda *_args: pytest.fail("session guard ran before authoritative engine probe"), + ) + with pytest.raises(runs.LiveEngineError, match="refusing to archive"): + runs.archive_run(tmp_path, run_dir) + + assert run_dir.is_dir() + assert state_dir.is_dir() + assert not (tmp_path / ".bmad-loop" / "archive").exists() + + +def test_archive_run_holds_one_lock_through_snapshot_publish_and_removal(tmp_path, monkeypatch): + """Snapshot, durable publication, source removal, and control cleanup all + remain inside the same lifecycle hold. + + Ablation: moving the liveness gate, tar add, replace, rmtree, or discard outside + the hold fails at that seam. Verified. + """ + run_dir = _make_state_run(tmp_path, "20260611-100000-aaaa") + (run_dir / "payload").write_text("data", encoding="utf-8") + order: list[str] = [] + real_add = runs.tarfile.TarFile.add + real_replace = runs.atomic_replace + real_rmtree = runs.shutil.rmtree + + def dead(target): + assert_run_state_lock_held(target) + order.append("probe") + return "dead" + + def checked_add(self, *args, **kwargs): + assert_run_state_lock_held(run_dir) + order.append("snapshot") + return real_add(self, *args, **kwargs) + + def checked_replace(src, dest): + assert_run_state_lock_held(run_dir) + order.append("publish") + return real_replace(src, dest) + + def checked_rmtree(target, *args, **kwargs): + assert_run_state_lock_held(run_dir) + order.append("remove") + return real_rmtree(target, *args, **kwargs) + + def checked_discard(_project, _run_id): + assert_run_state_lock_held(run_dir) + order.append("discard") + + monkeypatch.setattr(runs, "engine_liveness", dead) + monkeypatch.setattr(runs.tarfile.TarFile, "add", checked_add) + monkeypatch.setattr(runs, "atomic_replace", checked_replace) + monkeypatch.setattr(runs.shutil, "rmtree", checked_rmtree) + monkeypatch.setattr(runs, "_discard_state_dir", checked_discard) + + runs.archive_run(tmp_path, run_dir) + + assert order[0] == "probe" + assert order[-3:] == ["publish", "remove", "discard"] + assert order[1:-3] and set(order[1:-3]) == {"snapshot"} + + def test_archive_run_names_its_temp_after_the_destination(tmp_path, monkeypatch): """#363's filename half, and it needs its own test because NOTHING else grades it: on the happy path `atomic_replace` consumes the temp under either spelling, @@ -4529,6 +4723,9 @@ def boom(src, dst): assert (run_dir / "state.json").is_file() # the run survives a failed archive assert list((tmp_path / ".bmad-loop" / "archive").iterdir()) == [] # no temp left + sidecar = runs.lock_path_for(run_dir / "state.json", follow_final_symlink=False) + with platform_util.file_lock(sidecar, blocking=False): + pass # the original archive error released lifecycle exclusion def test_archive_run_temp_is_created_exclusively_at_0600(tmp_path, monkeypatch): diff --git a/tests/test_runsetup.py b/tests/test_runsetup.py index b1a37a79..0f143512 100644 --- a/tests/test_runsetup.py +++ b/tests/test_runsetup.py @@ -15,6 +15,7 @@ """ import dataclasses +import os import shutil import threading import types @@ -595,6 +596,85 @@ def test_compose_run_unwinds_the_run_when_the_adapters_abort(unwinding): _assert_unwound(unwinding) +def test_failed_composition_identifies_its_narrow_run_ownership(unwinding, monkeypatch): + """The composer may unwind its own live pid publication, but must opt into + that exception explicitly rather than borrowing operator ``force``.""" + real_delete = runs.delete_run + calls: list[tuple[bool, int | None]] = [] + + def checked_delete( + project, + run_dir, + *, + force=False, + _expected_composer_pid=None, + _expected_composer_claim=None, + ): + assert _expected_composer_claim is not None + assert os.path.samestat(_expected_composer_claim, run_dir.stat(follow_symlinks=False)) + calls.append((force, _expected_composer_pid)) + return real_delete( + project, + run_dir, + force=force, + _expected_composer_pid=_expected_composer_pid, + _expected_composer_claim=_expected_composer_claim, + ) + + monkeypatch.setattr(runs, "delete_run", checked_delete) + with pytest.raises(SystemExit, match="not usable on this host"): + _run_compose_sweep(unwinding.project, unwinding.make_adapters) + + assert calls == [(False, os.getpid())] + _assert_unwound(unwinding) + + +def test_failed_composition_refuses_to_unwind_a_rival_pid_publication(unwinding, capsys): + """A resume that replaces the composer's pid publication owns the run now. + + The launch error remains authoritative, while the refused unwind is reported + and leaves both run and control state available to the rival. + """ + + def rival_then_abort(project, run_dir, policy, *, profiles=None): + unwinding.published["run_dir"] = run_dir.is_dir() + unwinding.published["state"] = (run_dir / "state.json").is_file() + unwinding.published["state_dir"] = runs.state_dir_for(project, RUN_ID).is_dir() + (run_dir / runs.PID_FILE).write_text(str(os.getpid() + 1), encoding="utf-8") + raise SystemExit(BOOM) + + with pytest.raises(SystemExit, match="not usable on this host"): + _run_compose_sweep(unwinding.project, rival_then_abort) + + warning = capsys.readouterr().err + assert "changed engine ownership" in warning + assert runs.run_dir_for(unwinding.project, RUN_ID).is_dir() + assert runs.state_dir_for(unwinding.project, RUN_ID).is_dir() + + +def test_failed_composition_refuses_to_unwind_a_replacement_directory(unwinding, capsys): + """A missing pid does not prove the composer's original directory still exists. + + A cleanup can remove that directory after an unverifiable liveness probe and a + later creator can claim the same id before composition unwinds. The directory + identity captured by the original exclusive claim keeps the replacement whole. + """ + + def replace_then_abort(project, run_dir, policy, *, profiles=None): + shutil.rmtree(run_dir) + run_dir.mkdir() + (run_dir / "replacement").write_text("owned elsewhere", encoding="utf-8") + raise SystemExit(BOOM) + + with pytest.raises(SystemExit, match="not usable on this host"): + _run_compose_sweep(unwinding.project, replace_then_abort) + + warning = capsys.readouterr().err + assert "changed directory ownership" in warning + replacement = runs.run_dir_for(unwinding.project, RUN_ID) + assert (replacement / "replacement").read_text(encoding="utf-8") == "owned elsewhere" + + def test_compose_sweep_unwinds_the_run_when_the_adapters_abort(unwinding): """The sweep composer publishes the same artifacts (plus `sweep.json`) ahead of the same `make_adapters` call, so it owns its own unwind — separately, since a @@ -891,7 +971,14 @@ def test_a_failed_unwind_is_reported_and_does_not_replace_the_launch_error( operator is still `make_adapters`', not the cleanup's. A bare `pytest.raises` would pass just as happily for a cleanup failure that replaced it.""" - def boom(project, run_dir, *, force=False): + def boom( + project, + run_dir, + *, + force=False, + _expected_composer_pid=None, + _expected_composer_claim=None, + ): raise OSError(13, "Permission denied") monkeypatch.setattr(runs, "delete_run", boom) @@ -923,7 +1010,14 @@ def test_a_failed_unwind_still_reports_when_the_run_dir_is_already_gone( suppression around the journal write is load-bearing and gets its own test. The stderr report must still land, since it is now the only channel left.""" - def boom(project, run_dir, *, force=False): + def boom( + project, + run_dir, + *, + force=False, + _expected_composer_pid=None, + _expected_composer_claim=None, + ): shutil.rmtree(run_dir) raise RuntimeError("state dir removal failed") diff --git a/tests/test_tui_app.py b/tests/test_tui_app.py index c50a9371..a662d83d 100644 --- a/tests/test_tui_app.py +++ b/tests/test_tui_app.py @@ -4452,6 +4452,54 @@ async def test_archive_live_run_refused_without_calling(project, monkeypatch): assert not isinstance(app.screen, ConfirmModal) +@pytest.mark.parametrize( + "key, helper, failure, expected", + [ + ("D", "delete_run", runs_mod.LiveEngineError("engine resumed"), "delete failed"), + ( + "D", + "delete_run", + runs_mod.StateRootError("no usable state root"), + "delete failed", + ), + ("A", "archive_run", runs_mod.LiveEngineError("engine resumed"), "archive failed"), + ( + "A", + "archive_run", + runs_mod.StateRootError("no usable state root"), + "archive failed", + ), + ], +) +async def test_lifecycle_workers_report_authoritative_failures_and_keep_the_run_visible( + project, monkeypatch, key, helper, failure, expected +): + """The modal's liveness sample is advisory. A later lifecycle or state-lock + refusal is toasted from the worker, and the dashboard forget happens only on + success. + + Ablation: omit either new exception type from the worker catch and the worker + dies without the expected notification. Verified. + """ + monkeypatch.setattr(data, "liveness", lambda _run_dir: "dead") + + def fail(*_args, **_kwargs): + raise failure + + monkeypatch.setattr(runs_mod, helper, fail) + run_dir = make_run(project.project, "20260611-100000-aaaa", finished=True) + app = BmadLoopApp(project.project) + async with app.run_test() as pilot: + await until(pilot, lambda: dashboard(app).selected_run_id == run_dir.name) + await pilot.press(key) + await until(pilot, lambda: isinstance(app.screen, ConfirmModal)) + await pilot.click(await ready(pilot, "#ok")) + await until(pilot, lambda: any(expected in note for note in notifications(app))) + assert dashboard(app).selected_run_id == run_dir.name + + assert run_dir.is_dir() + + # ------------------------------------------------------------ graceful stop (S) From f9a0dc06128999b3c0885d3333f5507357f1df25 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 2 Sep 2026 11:35:30 -0700 Subject: [PATCH 12/18] test(runsetup,diagnostics,engine): stop depending on inode reuse and POSIX-only imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two rows fake a non-Windows `sys.platform` to pin the POSIX half of a gate. The faked branch reaches `import fcntl`, which does not exist on Windows, so they raise ModuleNotFoundError there rather than testing anything. They predate this series; the sweep only extended `diagnostics.collect` far enough to reach that import. Skip them off POSIX — the Linux legs still hold the row. The replacement-directory row deleted the run dir and recreated it, assuming the new directory would land on a different inode. A filesystem is free to reuse the inode it just released, and CI's does: `os.path.samestat` then reads the replacement as the very directory the composer claimed, the guard correctly stays silent, and the row failed on an empty warning. Allocate the replacement while the original is still live — two directories that exist at once cannot share an inode — then rename it into place. --- tests/test_diagnostics.py | 7 +++++++ tests/test_engine.py | 7 +++++++ tests/test_runsetup.py | 12 ++++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index 701c4e0b..3ae4ed62 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -326,6 +326,13 @@ def test_env_names_the_platform_and_the_win32_on_wsl_path_verdict(project, monke assert sanitize.assert_no_leak(js) == [] # general backstop; blind to this shape +@pytest.mark.skipif( + sys.platform == "win32", + reason=( + "pins the POSIX half of the gate by faking sys.platform; the faked branch's\n" + "imports (fcntl) do not exist on Windows, so the row can only run off win32" + ), +) def test_env_win32_on_wsl_path_is_false_off_win32(project, monkeypatch): """The *platform* half of the twin gate, pinned. What #332 names is a mismatched interpreter, not a path shape: the very distro path a win32 interpreter warns about diff --git a/tests/test_engine.py b/tests/test_engine.py index 9d5d6f88..2cf40e3f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -11891,6 +11891,13 @@ def native_test_lock(_path): assert Engine._stop_signals_owner is None +@pytest.mark.skipif( + sys.platform == "win32", + reason=( + "pins the POSIX half of the gate by faking sys.platform; the faked branch's\n" + "imports (fcntl) do not exist on Windows, so the row can only run off win32" + ), +) def test_non_windows_sigint_still_stops_run(project, monkeypatch): import bmad_loop.engine as engine_mod diff --git a/tests/test_runsetup.py b/tests/test_runsetup.py index 0f143512..a7a9be1d 100644 --- a/tests/test_runsetup.py +++ b/tests/test_runsetup.py @@ -661,9 +661,17 @@ def test_failed_composition_refuses_to_unwind_a_replacement_directory(unwinding, """ def replace_then_abort(project, run_dir, policy, *, profiles=None): + # Allocate the replacement while the original still holds its own inode, + # then rename it into place. `rmtree` followed by `mkdir` is free to reuse + # the inode it just released, and on some filesystems it does — leaving + # `os.path.samestat` unable to tell the replacement from the directory the + # composer exclusively claimed, so the guard correctly stays silent and the + # row fails for a reason it is not testing. + stand_in = run_dir.parent / f"{run_dir.name}.replacement" + stand_in.mkdir() + (stand_in / "replacement").write_text("owned elsewhere", encoding="utf-8") shutil.rmtree(run_dir) - run_dir.mkdir() - (run_dir / "replacement").write_text("owned elsewhere", encoding="utf-8") + stand_in.rename(run_dir) raise SystemExit(BOOM) with pytest.raises(SystemExit, match="not usable on this host"): From 14cd2f9149149368d57ede3ccd9e81552cfa78a2 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 2 Sep 2026 12:06:24 -0700 Subject: [PATCH 13/18] test(generic): reuse the shared json recursion probe Second site with the same version-dependent premise as the resolve row: a hardcoded `getrecursionlimit() * 20` that 3.13 cannot decode but 3.14 parses iteratively, so the row failed on "DID NOT RAISE" rather than on the degradation it exists to pin. Take the depth from the conftest probe. --- tests/test_generic_tmux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_generic_tmux.py b/tests/test_generic_tmux.py index dbca0488..3022698a 100644 --- a/tests/test_generic_tmux.py +++ b/tests/test_generic_tmux.py @@ -22,6 +22,7 @@ import pytest import regex +from conftest import json_recursion_payload from bmad_loop import devcontract, runs from bmad_loop.adapters import base as adapter_base @@ -233,8 +234,7 @@ def test_read_result_degrades_a_decoder_recursion_error(tmp_path): adapter = make_adapter(tmp_path) task_dir = adapter.tasks_dir / "t1" task_dir.mkdir(parents=True) - depth = sys.getrecursionlimit() * 20 - nested = '{"value":' + ("[" * depth) + "0" + ("]" * depth) + "}" + nested = '{"value":' + json_recursion_payload() + "}" with pytest.raises(RecursionError): json.loads(nested) From bd1feb54fed10465a70aec813f970ba61c8020e8 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 2 Sep 2026 18:49:50 -0700 Subject: [PATCH 14/18] fix(runs): compare the rival engine against the pid file as recorded, not as cleared MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_stop_run_once` clears its local `pid` on every path that declines to signal — gone, reused, or an identity it cannot read — and then compared the post-delivery pid file against that cleared tuple. An engine whose pid exists but whose identity cannot be read (win32 ERROR_ACCESS_DENIED) is `"unknown"`, not `"dead"`, so the unchanged file read as a rival that had just published a new engine, `_stop_run_once` answered "retry", and `stop_run` — unbounded by design, so a real rival is always sent the stop — never returned. Keep the tuple as read for the generation compare. A rival is a CHANGED pid file, nothing else; the unverifiable engine then takes the fallback the pre-split `stop_run` always took for it. The test wraps `_stop_run_once` so the livelock reddens instead of hanging the suite. --- CHANGELOG.md | 6 ++++++ src/bmad_loop/runs.py | 11 +++++++++-- tests/test_runs.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a9fcc02..a4445dcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -270,6 +270,12 @@ breaking changes may land in a minor release. unrecoverably, on a gitignored ledger. The mark and append legs now re-anchor only when the preimage they wrote over is still the bytes the run last claimed; the restore skips and journals `ledger-restore-skipped-diverged`. +- **`bmad-loop stop` no longer retries forever on an engine whose identity cannot be + read.** The rival-engine compare under the state lock used the local pid after every + declining path had cleared it, so an unchanged pid file with `"unknown"` liveness read + as a freshly published rival on every attempt. The compare now uses the pid file as + recorded; a rival is a changed file, nothing else. + - Serialize run deletion/archive against resume (DW-94), refusing a newly live engine under the per-run lock and preventing a waiting resume from recreating a run cleanup already removed. diff --git a/src/bmad_loop/runs.py b/src/bmad_loop/runs.py index 8baf3f6e..d815176f 100644 --- a/src/bmad_loop/runs.py +++ b/src/bmad_loop/runs.py @@ -2203,11 +2203,18 @@ def _stop_run_once(run_dir: Path) -> bool | None: host = get_process_host() pid, identity = read_pid_identity(run_dir) # identity recorded at run start, not sampled now + # The pid-file tuple AS READ, kept for the generation compare under the lock + # below. The local `pid` is cleared on every path that declines to signal — a + # pid that is gone, reused, or whose identity cannot be read — so comparing the + # post-delivery pid file against `(pid, identity)` AFTER that clearing made an + # unverifiable engine read as a rival that had published a new pid: the file + # was unchanged, its liveness `"unknown"` (not `"dead"`), and `stop_run`'s + # retry loop re-entered forever. A rival is a CHANGED pid file, nothing else. + recorded_engine = (pid, identity) if pid is not None and identity is not None and not host.alive_and_ours(pid, identity): # the pid we recorded is already gone, or was reused by an unrelated # process before stop_run ran — never signal a stranger; mark stopped below. pid = None - addressed_engine = (pid, identity) # Whether this call ever proved the engine dead. Only a confirmed death licenses # the fallback below to discard the request we lodged: while the engine may still # be running, that file is the one channel left that can stop it (on native @@ -2307,7 +2314,7 @@ def _stop_run_once(run_dir: Path) -> bool | None: state = load_state(run_dir) current_engine = read_pid_identity(run_dir) current_liveness = engine_liveness(run_dir) - rival_published_engine = current_liveness != "dead" and current_engine != addressed_engine + rival_published_engine = current_liveness != "dead" and current_engine != recorded_engine if state.finished: # The engine completed while the stop channels were in flight. Its # terminal state is authoritative; do not rewrite it as a fallback stop. diff --git a/tests/test_runs.py b/tests/test_runs.py index a6f0e838..4ca41092 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -879,6 +879,45 @@ def terminate(self, pid): assert runs.read_stop_request_mode(run_dir) is None +def test_stop_run_does_not_loop_on_an_engine_whose_identity_cannot_be_read(tmp_path, monkeypatch): + """A recorded pid that exists but whose identity cannot be read (win32 + ERROR_ACCESS_DENIED is the measured shape) is `"unknown"`, not `"dead"`, and + `alive_and_ours` declines it — so the local pid is cleared and nothing is + signalled. The generation compare under the lock must then read the pid file + AS RECORDED: compared against the cleared local pid, the unchanged file looked + like a rival that had published a new engine, `_stop_run_once` answered "retry", + and `stop_run` — an unbounded loop by design, so a real rival is always sent the + stop — never returned. + + `_stop_run_once` is wrapped to turn that livelock into a failure, or the + ablation would hang the suite rather than redden it. + + Ablation: compare `current_engine` against the post-clearing `(pid, identity)` + again and this reddens on the call count.""" + run_dir = _make_state_run(tmp_path, "r1") + (run_dir / "engine.pid").write_text("4242 100.0", encoding="utf-8") + monkeypatch.setattr(runs, "kill_session", lambda _rid: None) + host = _FakeHost(alive=True, identity=None) # exists, identity unreadable + assert host.liveness_of(4242, 100.0) == "unknown" # MEASURED: the arm under test + monkeypatch.setattr(runs, "get_process_host", lambda: host) + real_once = runs._stop_run_once + calls: list[int] = [] + + def bounded_once(run_dir_): + calls.append(1) + if len(calls) > 3: + raise AssertionError("stop_run is retrying an unchanged engine forever") + return real_once(run_dir_) + + monkeypatch.setattr(runs, "_stop_run_once", bounded_once) + + assert runs.stop_run(run_dir) is True + + assert len(calls) == 1 + assert host.terminated == [] # never signals a pid it cannot prove is ours + assert load_state(run_dir).stopped is True + + def test_stop_run_engine_confirmed_leaves_nothing_pending(tmp_path, monkeypatch): """When the engine confirms the stop itself the request is consumed too. The engine normally clears it on the way out; this is the belt-and-braces half, and From 9d47f495329313746e5e16c15e1ccfef3607344c Mon Sep 17 00:00:00 2001 From: t Date: Wed, 2 Sep 2026 18:49:50 -0700 Subject: [PATCH 15/18] fix(opencode): validate the result-file mixin's task artifacts before writing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `OpencodeHttpAdapter.start_session` refused a redirected `messages.json` — its own file — but not a symlinked, hardlinked, FIFO or device `heartbeat.json`, `resultless-stops.jsonl` or `session-lifecycle.jsonl`, all written by the inherited `_ResultFileMixin`. A reused task directory carrying one of those let the heartbeat overwrite truncate a linked external file and the breadcrumb appends block on or redirect into it, which GenericAdapter already refuses. The three names move into `generic.RESULT_FILE_ARTIFACTS`, handed to `validate_adapter_artifact_paths` by both adapters, so a fourth mixin write cannot reach one adapter's validation and miss the other's. The test is parametrized over that tuple for the same reason. --- CHANGELOG.md | 5 ++++ src/bmad_loop/adapters/generic.py | 24 ++++++++++------ src/bmad_loop/adapters/opencode_http.py | 6 +++- tests/test_opencode_http.py | 37 +++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4445dcc..d45afd21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -276,6 +276,11 @@ breaking changes may land in a minor release. as a freshly published rival on every attempt. The compare now uses the pid file as recorded; a rival is a changed file, nothing else. +- **The OpenCode adapter validates every task artifact it will write.** It refused a + redirected `messages.json` but not a symlinked, hardlinked or special `heartbeat.json`, + `resultless-stops.jsonl` or `session-lifecycle.jsonl`, which the inherited result-file + mixin writes; the three names now live in one `RESULT_FILE_ARTIFACTS` tuple both adapters + validate. - Serialize run deletion/archive against resume (DW-94), refusing a newly live engine under the per-run lock and preventing a waiting resume from recreating a run cleanup already removed. diff --git a/src/bmad_loop/adapters/generic.py b/src/bmad_loop/adapters/generic.py index 04fa4958..6fdda519 100644 --- a/src/bmad_loop/adapters/generic.py +++ b/src/bmad_loop/adapters/generic.py @@ -181,6 +181,21 @@ class _SnapVerdict(enum.Enum): ) +# Every task-directory leaf `_ResultFileMixin` writes during a session, beyond the +# cycle artifacts in `journal.TASK_CYCLE_ARTIFACTS` and the prompt. Both adapters +# that inherit the mixin hand this tuple to `validate_adapter_artifact_paths` +# before their first write: a reused task directory carrying a symlink, hardlink, +# FIFO or device under one of these names would otherwise have the heartbeat +# overwrite truncate a linked external file, or a breadcrumb append block on or +# redirect into it. One tuple, so a fourth mixin write cannot reach one adapter's +# validation and miss the other's. +RESULT_FILE_ARTIFACTS: tuple[str, ...] = ( + "heartbeat.json", + "resultless-stops.jsonl", + "session-lifecycle.jsonl", +) + + class _ResultFileMixin: """Result-file read-back and verdict finalization: acquire the skill-written result dict and fold it into the session's final @@ -561,14 +576,7 @@ def start_session(self, spec: SessionSpec) -> SessionHandle: task_dir = validated_task_directory(self.tasks_dir, spec.task_id) validate_adapter_artifact_paths( task_dir, - tuple( - task_dir / name - for name in ( - "heartbeat.json", - "resultless-stops.jsonl", - "session-lifecycle.jsonl", - ) - ), + tuple(task_dir / name for name in RESULT_FILE_ARTIFACTS), ) validate_adapter_artifact_paths( self.logs_dir, diff --git a/src/bmad_loop/adapters/opencode_http.py b/src/bmad_loop/adapters/opencode_http.py index bba082ad..72a36706 100644 --- a/src/bmad_loop/adapters/opencode_http.py +++ b/src/bmad_loop/adapters/opencode_http.py @@ -164,6 +164,7 @@ BUDGET_NUDGE_TEXT, HEARTBEAT_INTERVAL_S, NUDGE_TEXT, + RESULT_FILE_ARTIFACTS, STALL_NUDGE_TEXT, _DevSynthesisMixin, _ResultFileMixin, @@ -628,9 +629,12 @@ def _await_healthy(self, sess: _ServerSession) -> bool: def start_session(self, spec: SessionSpec) -> SessionHandle: task_dir = validated_task_directory(self.tasks_dir, spec.task_id) + # `messages.json` is this transport's own; the rest are the inherited + # `_ResultFileMixin`'s writes, validated here for the same reason + # GenericAdapter validates them — see `RESULT_FILE_ARTIFACTS`. validate_adapter_artifact_paths( task_dir, - (task_dir / "messages.json",), + (task_dir / "messages.json", *(task_dir / name for name in RESULT_FILE_ARTIFACTS)), ) log_paths = [ self.logs_dir / f"{spec.task_id}.log", diff --git a/tests/test_opencode_http.py b/tests/test_opencode_http.py index 169068c6..7fb00bdb 100644 --- a/tests/test_opencode_http.py +++ b/tests/test_opencode_http.py @@ -1359,6 +1359,43 @@ def test_start_session_refuses_symlinked_task_directory_without_side_effects(tmp assert adapter._sessions == {} +@pytest.mark.parametrize("name", generic.RESULT_FILE_ARTIFACTS) +def test_start_session_refuses_a_redirected_mixin_artifact_without_side_effects(tmp_path, name): + """The inherited `_ResultFileMixin` writes the heartbeat and both breadcrumb + files under the task directory, so a reused directory carrying a symlink under + one of those names is the same hazard the generic adapter refuses: the heartbeat + overwrite truncates the link's target, a breadcrumb append lands outside the run. + This adapter validated `messages.json` alone — its own file — and let the three + mixin writes through. + + Parametrized over `generic.RESULT_FILE_ARTIFACTS` so a fourth mixin write is a + new row here, not a new gap. + + Ablation: validate `(task_dir / "messages.json",)` alone again and every row + reddens on the raise.""" + adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") + spawn_calls = [] + adapter._spawn_server = lambda spec: spawn_calls.append(spec) + outside = tmp_path / "outside-target" + outside.write_text("theirs", encoding="utf-8") + task_id = "reused-task" + task_dir = adapter.tasks_dir / task_id + task_dir.mkdir(parents=True) + try: + (task_dir / name).symlink_to(outside) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + spec = SessionSpec(task_id=task_id, role="triage", prompt="p", cwd=tmp_path) + + with pytest.raises(AdapterTaskDirectoryError, match="symlink or junction"): + adapter.start_session(spec) + + assert outside.read_text(encoding="utf-8") == "theirs" + assert (task_dir / name).is_symlink() + assert not (task_dir / "prompt.txt").exists() # refused before the first write + assert spawn_calls == [] + + def test_start_session_refuses_symlinked_tasks_root_without_side_effects(tmp_path): adapter = make_adapter(tmp_path, binary="definitely-not-a-real-binary-xyz") spawn_calls = [] From 08bc273bbcace18669d89ca8b2cf1e3a4d51640e Mon Sep 17 00:00:00 2001 From: t Date: Thu, 3 Sep 2026 10:03:24 -0700 Subject: [PATCH 16/18] fix(cli,runs): keep a lock-held run from aborting the clean sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-run state lock DW-94/DW-93 added to `delete_run`/`archive_run` is new in this branch — on main neither took a lock — and `file_lock` raises a plain `OSError` when the acquisition fails. That escaped `cmd_clean`'s per-candidate handler and aborted the whole invocation: later candidates were never processed, and runs the loop had already mutated vanished from a report only emitted after the loop. `file_lock` now raises a typed `LockUnavailableError` for a failed acquisition alone (never the locked body, never the lock-file create), and `cmd_clean` catches it beside the lifecycle races, classifying the run by what actually happened and continuing. Typed rather than a bare `OSError` so `UnconfinedWriteError` — also an `OSError` — is never folded into "left untouched". `clean` acquires with `wait_for_lock=False`: `fcntl.flock` never times out, so waiting was unbounded on POSIX, and a lock someone else holds already means what `clean` reports anyway. The flag is threaded through `state_lock` and defaults to blocking, leaving every other caller unchanged. --- CHANGELOG.md | 14 ++++ src/bmad_loop/cli.py | 51 ++++++++++---- src/bmad_loop/journal.py | 18 ++++- src/bmad_loop/platform_util.py | 62 ++++++++++++++--- src/bmad_loop/runs.py | 27 ++++++-- tests/test_cleanup.py | 120 +++++++++++++++++++++++++++++++-- tests/test_engine.py | 2 +- tests/test_journal.py | 18 ++--- tests/test_runs.py | 2 +- 9 files changed, 267 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d45afd21..bf74dab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -270,6 +270,20 @@ breaking changes may land in a minor release. unrecoverably, on a gitignored ledger. The mark and append legs now re-anchor only when the preimage they wrote over is still the bytes the run last claimed; the restore skips and journals `ledger-restore-skipped-diverged`. +- **`bmad-loop clean` no longer aborts the whole sweep when one run's state lock is + held.** The per-run lock DW-94/DW-93 added to `runs.delete_run`/`archive_run` is new in + this branch — on `main` neither took a lock — and its acquisition `OSError` escaped + `cmd_clean`'s per-candidate handler, so a single busy run cost the operator the entire + report: later candidates were never processed, and runs already mutated (worktrees + removed, artifacts trimmed) vanished from a document only emitted after the loop. The + contended run is now classified like the lifecycle races beside it — `trimmed` if + anything reached it, else `protected` — and the sweep continues. `file_lock` raises a + typed `LockUnavailableError` for a failed acquisition, and `clean` acquires + non-blocking: `fcntl.flock` never times out, so waiting was unbounded on POSIX, and a + lock someone else holds already means what `clean` reports anyway. The error is typed + rather than a bare `OSError` so a confinement refusal (`UnconfinedWriteError`, also an + `OSError`) is never folded into "left untouched". + - **`bmad-loop stop` no longer retries forever on an engine whose identity cannot be read.** The rival-engine compare under the state lock used the local pid after every declining path had cleared it, so an unchanged pid file with `"unknown"` liveness read diff --git a/src/bmad_loop/cli.py b/src/bmad_loop/cli.py index 268cca56..6ba5068b 100644 --- a/src/bmad_loop/cli.py +++ b/src/bmad_loop/cli.py @@ -72,7 +72,12 @@ from .engine import Engine from .journal import Journal, load_state, save_state, state_lock from .model import RunState -from .platform_util import MAX_SEGMENT, resolve_or_lexical, walk_files_unlinked +from .platform_util import ( + MAX_SEGMENT, + LockUnavailableError, + resolve_or_lexical, + walk_files_unlinked, +) from .process_host import ProcessHostError # The run-composition helpers now live in runsetup.py (the library layer a non-CLI @@ -4315,7 +4320,15 @@ def cmd_clean(args: argparse.Namespace) -> int: mid-flight stop, trim heavy scaffolding from runs kept for history, and archive/delete runs past the retention window. Only terminal (finished or stopped) runs are touched; running, unknown-host, paused and interrupted - runs are always left intact.""" + runs are always left intact. + + Every per-candidate refusal is DATA, never an abort: this is a sweep, so one + busy run must not cost the operator the report of the runs already reclaimed + around it (they only reach stdout in the post-loop emission). That is why the + removals here take the run's state lock with ``wait_for_lock=False`` — a lock + someone else holds already means what this command reports anyway, and + waiting for it is unbounded on POSIX, where ``fcntl.flock`` never times + out.""" project = _project(args) paths = bmadconfig.load_paths(project) repo = paths.repo_root @@ -4395,17 +4408,28 @@ def cmd_clean(args: argparse.Namespace) -> int: try: if args.hard or not pol.cleanup.archive_old: if not dry: - runs.delete_run(project, run_dir) + runs.delete_run(project, run_dir, wait_for_lock=False) deleted.append(run_dir.name) else: if not dry: - runs.archive_run(project, run_dir) + runs.archive_run(project, run_dir, wait_for_lock=False) archived.append(run_dir.name) - except (runs.LiveEngineError, runs.LiveSessionError) as e: + except (runs.LiveEngineError, runs.LiveSessionError, LockUnavailableError) as e: # A session or engine appeared between the loop-top sample and the - # authoritative removal transaction. Record this run instead of - # letting one racer abort the whole invocation, then continue with - # its siblings. Correct the estimate down to what actually went. + # authoritative removal transaction — or the run's state lock is + # held, which says the same thing one layer down and is the only + # one of the three that reports it in this window: `resume` takes + # the lock FIRST and publishes its pid LAST, so for its whole + # preflight (git work bounded by `[limits] git_timeout_s`) the + # pid/session guards above still read dead and only the lock + # objects. Record this run instead of letting one racer abort the + # whole invocation, then continue with its siblings. Correct the + # estimate down to what actually went. + # + # `LockUnavailableError` and NOT a bare `except OSError`: that + # would also catch `platform_util.UnconfinedWriteError`, an + # OSError subclass raised when a write escapes its root, and file + # a containment refusal as a benign "left untouched". freed += heavy_bytes - run_bytes # Classify by what happened, not by what was intended: the steps # above may already have taken this run's worktree and artifacts, @@ -4414,11 +4438,12 @@ def cmd_clean(args: argparse.Namespace) -> int: # trimmed, which is exactly the state it ends in. (trimmed if run_worktrees or shrunk else protected).append(run_dir.name) if not args.json: - reason = ( - "agent session appeared mid-clean" - if isinstance(e, runs.LiveSessionError) - else "engine resumed mid-clean" - ) + if isinstance(e, runs.LiveSessionError): + reason = "agent session appeared mid-clean" + elif isinstance(e, LockUnavailableError): + reason = "run state locked by another process" + else: + reason = "engine resumed mid-clean" print( f"run {run_dir.name}: {reason} — not removed", file=sys.stderr, diff --git a/src/bmad_loop/journal.py b/src/bmad_loop/journal.py index 124676cb..2d2d94cb 100644 --- a/src/bmad_loop/journal.py +++ b/src/bmad_loop/journal.py @@ -224,9 +224,19 @@ def entries(self) -> list[dict[str, Any]]: @contextmanager -def state_lock(run_dir: Path) -> Iterator[None]: +def state_lock(run_dir: Path, *, blocking: bool = True) -> Iterator[None]: """Serialize one run's state mutations, re-entering only for the same run. + ``blocking=False`` gives up instead of waiting, raising + :class:`platform_util.LockUnavailableError` when another holder has the run. + It is for a caller whose own semantics already say "in use ⇒ leave it alone" + and which must not stall on one busy run — ``cli.cmd_clean`` sweeping many. + The default stays blocking, because every other writer here is mutating one + run it means to mutate, and for those giving up is data loss, not politeness. + That error propagates out of this function UNCAUGHT and unwrapped: the whole + point is that the caller gets to tell contention apart from a real fault, and + a translation here would take that back. + The sidecar identity comes from :func:`runs.lock_path_for`, so alternate path spellings of one ``state.json`` rendezvous on the same out-of-tree lock. The import is deliberately lazy: ``runs`` imports this module's persistence helpers. @@ -234,7 +244,9 @@ def state_lock(run_dir: Path) -> Iterator[None]: Reentrancy is thread-local and intentionally limited to one run. An outer read-modify-write transaction can call the self-locking :func:`save_state` without acquiring the OS lock twice, while nested mutation of another run is - refused before a second lock can introduce an ordering cycle. + refused before a second lock can introduce an ordering cycle. A re-entrant + acquisition ignores ``blocking`` because it acquires nothing: this thread + already holds the run, so there is no one to wait for and nothing to refuse. """ from . import runs @@ -252,7 +264,7 @@ def state_lock(run_dir: Path) -> Iterator[None]: _STATE_LOCK_LOCAL.depth -= 1 return - with file_lock(lock_path): + with file_lock(lock_path, blocking=blocking): _STATE_LOCK_LOCAL.path = lock_path _STATE_LOCK_LOCAL.depth = 1 try: diff --git a/src/bmad_loop/platform_util.py b/src/bmad_loop/platform_util.py index 63aa4634..d92c27e5 100644 --- a/src/bmad_loop/platform_util.py +++ b/src/bmad_loop/platform_util.py @@ -1327,19 +1327,43 @@ def retrying_unlink(path: Path) -> None: _retry_on_sharing_violation(path.unlink) +class LockUnavailableError(OSError): + """:func:`file_lock` could not ACQUIRE the lock: another holder has it, or the + acquisition call itself faulted (EINTR, ENOLCK, EBADF). + + An ``OSError`` subclass for the reason :class:`UnconfinedWriteError` is one — + every site that already degrades on ``OSError`` from a lock acquisition keeps + its behavior unchanged, so this narrows nothing that was previously caught. + What the subclass buys is the arm a caller could not write before: telling + "something else is using this" apart from every other ``OSError`` WITHOUT the + bare ``except OSError`` that would also swallow an ``UnconfinedWriteError`` + and file a confinement refusal as a benign "busy" — the exact fold-to-benign + those two classes exist to keep apart. + + Raised for the ACQUISITION ALONE. A fault raised by the locked body is never + wrapped: reporting a body error as contention is the same fold in the other + direction. The ``os.open`` that provisions the lock file is deliberately left + unwrapped too — failing to create the sidecar is a provisioning fault, not a + holder, and a caller that treats it as "someone is using this run" would + retry forever against a broken path.""" + + @contextmanager def file_lock(path: Path, *, blocking: bool = True) -> Iterator[None]: """Exclusive OS advisory lock on ``path`` (created if missing), released on exit — and by the kernel when the holder dies, so a crashed process never wedges the lock (no stale-lockfile scheme to clean up). ``blocking=False`` - raises ``OSError`` at once when the lock is already held, giving tests a - deterministic exclusion probe instead of a sleep-based negative assertion. + raises :class:`LockUnavailableError` at once when the lock is already held, + giving tests a deterministic exclusion probe instead of a sleep-based negative + assertion — and giving a bulk sweep a way to skip a busy item rather than + stall on it (``cli.cmd_clean``). Lock a dedicated sibling file, never data that is swapped via :func:`atomic_replace` — the lock rides the open fd's inode, and a replace would swap that inode out from under later acquirers. ``fcntl.flock`` on POSIX; ``msvcrt.locking`` on Windows, where the blocking mode's built-in - ~10 s retry bounds the wait and surfaces contention as ``OSError``. + ~10 s retry bounds the wait and surfaces contention as + :class:`LockUnavailableError`. THE WAIT IS PLATFORM-ASYMMETRIC, and a caller has to decide what that means for it: POSIX blocks indefinitely, Windows gives up after ~10 s and raises. @@ -1349,7 +1373,16 @@ def file_lock(path: Path, *, blocking: bool = True) -> Iterator[None]: multi-step git transaction of roughly seven ``git`` spawns, each bounded by ``[limits] git_timeout_s``. So a Windows acquirer can genuinely time out under contention rather than only under a deadlock. Hold it for as short a - span as correctness allows, and handle the ``OSError`` from acquisition. + span as correctness allows, and handle the :class:`LockUnavailableError` from + acquisition. + + AND THE ASYMMETRY IS NOT REMOVED BY THE TYPED ERROR, only made catchable: a + ``blocking=True`` POSIX acquirer still waits forever, because ``fcntl.flock`` + does not time out. :class:`LockUnavailableError` therefore reaches a blocking + caller only on Windows' ~10 s bound and on genuinely exceptional POSIX errnos. + A caller that must not stall — a bulk sweep over many items, where one busy + item is data rather than a reason to stop — has to pass ``blocking=False`` and + handle the refusal; the typed error alone does not buy it. OWNER-ONLY, AND DELIBERATELY NOT MADE TO WORK ACROSS OS USERS. A repository shared between OS users is not a supported configuration (maintainer @@ -1375,15 +1408,22 @@ def file_lock(path: Path, *, blocking: bool = True) -> Iterator[None]: path.parent.mkdir(parents=True, exist_ok=True) fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600) try: - if sys.platform == "win32": - import msvcrt + try: + if sys.platform == "win32": + import msvcrt - # Locks 1 byte at the current position — 0 on a fresh fd. - msvcrt.locking(fd, msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK, 1) - else: - import fcntl + # Locks 1 byte at the current position — 0 on a fresh fd. + msvcrt.locking(fd, msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK, 1) + else: + import fcntl - fcntl.flock(fd, fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)) + fcntl.flock(fd, fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)) + except OSError as e: + # Only this call is wrapped — see LockUnavailableError on why the + # `os.open` above and the `yield` below are deliberately outside it. + # errno/strerror/filename are carried through so a caller that reads + # them, or just prints the exception, sees what the bare OSError said. + raise LockUnavailableError(e.errno, e.strerror or str(e), str(path)) from e try: yield finally: diff --git a/src/bmad_loop/runs.py b/src/bmad_loop/runs.py index d815176f..ade875f1 100644 --- a/src/bmad_loop/runs.py +++ b/src/bmad_loop/runs.py @@ -2605,6 +2605,7 @@ def delete_run( run_dir: Path, *, force: bool = False, + wait_for_lock: bool = True, _expected_composer_pid: int | None = None, _expected_composer_claim: os.stat_result | None = None, ) -> None: @@ -2627,9 +2628,18 @@ def delete_run( The containment guard runs first and is NOT under ``force``: an override is the operator accepting a leaked session, never a licence to rmtree a path - outside the runs dir.""" + outside the runs dir. + + ``wait_for_lock=False`` refuses instead of waiting when another process holds + the run's state lock, raising :class:`platform_util.LockUnavailableError`. It + is NOT an override in the sense ``force`` is — it removes nothing extra and + weakens no guard; it only declines to queue. A bulk caller passes it because + a held lock already means what that caller reports anyway ("in use, left + alone"), and because waiting is unbounded on POSIX where ``fcntl.flock`` never + times out. The default waits, which is what a single-run operator command + wants: there, giving up would turn a brief overlap into a failed command.""" _refuse_uncontained_run_dir(project, run_dir, "delete") - with state_lock(run_dir): + with state_lock(run_dir, blocking=wait_for_lock): if _expected_composer_claim is not None: try: current_claim = run_dir.stat(follow_symlinks=False) @@ -2662,7 +2672,9 @@ def delete_run( _discard_state_dir(project, run_dir.name) -def archive_run(project: Path, run_dir: Path, *, force: bool = False) -> Path: +def archive_run( + project: Path, run_dir: Path, *, force: bool = False, wait_for_lock: bool = True +) -> Path: """Compress a run dir into .bmad-loop/archive/.tar.gz and remove the original. The tarball is written to a temp path then atomically replaced into place so a partial archive never appears. Engine liveness is re-checked under @@ -2680,9 +2692,14 @@ def archive_run(project: Path, run_dir: Path, *, force: bool = False) -> Path: Containment (see :func:`_refuse_uncontained_run_dir`) is checked ahead of both, for the reason the session guard runs early: a refusal must leave no archive - directory and no tarball behind.""" + directory and no tarball behind. + + ``wait_for_lock`` carries the meaning it has on :func:`delete_run`: ``False`` + declines a contended run with :class:`platform_util.LockUnavailableError` + rather than queueing behind its holder, and refuses before the tarball is + written, so a decline — like the guards above it — leaves nothing behind.""" _refuse_uncontained_run_dir(project, run_dir, "archive") - with state_lock(run_dir): + with state_lock(run_dir, blocking=wait_for_lock): if engine_liveness(run_dir) == "alive": raise LiveEngineError( f"run {run_dir.name} is still live — refusing to archive it; stop it first" diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index d36d913c..c7165393 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -7,8 +7,8 @@ import pytest from conftest import install_bmad_config, machine_json -from bmad_loop import cli, runs, verify -from bmad_loop.journal import VERIFY_DIR, save_state +from bmad_loop import cli, platform_util, runs, verify +from bmad_loop.journal import STATE_FILE, VERIFY_DIR, save_state from bmad_loop.model import RunState @@ -479,10 +479,10 @@ def test_cmd_clean_json_records_a_resumed_engine_and_continues_siblings( ) real_cleanup = getattr(runs, helper_name) - def racing_cleanup(project_path, run_dir): + def racing_cleanup(project_path, run_dir, **kw): if run_dir == racer: raise runs.LiveEngineError("engine resumed") - return real_cleanup(project_path, run_dir) + return real_cleanup(project_path, run_dir, **kw) monkeypatch.setattr(runs, helper_name, racing_cleanup) @@ -505,7 +505,7 @@ def test_cmd_clean_text_identifies_a_resumed_engine(project, monkeypatch, capsys RunState(run_id=racer.name, project=str(repo), started_at="x", finished=True), ) - def racing_archive(_project, _run_dir): + def racing_archive(_project, _run_dir, **_kw): raise runs.LiveEngineError("engine resumed") monkeypatch.setattr(runs, "archive_run", racing_archive) @@ -517,6 +517,116 @@ def racing_archive(_project, _run_dir): assert "agent session appeared mid-clean" not in err +def _terminal_runs(repo, *names): + """Terminal (finished) run dirs, created in the order given.""" + made = [] + for name in names: + run_dir = repo / ".bmad-loop" / "runs" / name + save_state( + run_dir, + RunState(run_id=run_dir.name, project=str(repo), started_at="x", finished=True), + ) + made.append(run_dir) + return made + + +@pytest.mark.parametrize( + "hard, result_key, other_key", + [(True, "deleted", "archived"), (False, "archived", "deleted")], +) +def test_cmd_clean_json_records_a_lock_held_run_and_still_sweeps_its_siblings( + project, capsys, hard, result_key, other_key +): + """A run whose state lock someone else holds is per-run data, exactly as the + lifecycle refusals above it are — the sweep must not abort on it. + + THIS is the assertion that characterises the defect, not the single-run one: + every list `clean` reports reaches the operator only in the post-loop + emission, so an acquisition error escaping the per-candidate handler costs + the report of the whole invocation, including siblings it had ALREADY + deleted. The contended run therefore sits BETWEEN two siblings + (`list_run_dirs` is sorted oldest-first), so a passing run proves the sweep + continued past it rather than merely started before it. + + The contention is real — the run's own canonical sidecar, held across the + whole invocation — and carries no timing dependence: `cmd_clean` acquires + with `wait_for_lock=False`, so the rival attempt is refused at once instead + of waiting. It is taken through `platform_util.file_lock` rather than + `journal.state_lock` deliberately: `state_lock`'s reentrancy guard is + thread-local, so acquiring it here would make `cmd_clean` RE-ENTER the lock + it already holds and reclaim the run, passing for the wrong reason. + """ + install_bmad_config(project) + repo = project.project + early, held, late = _terminal_runs( + repo, + "20260101-000000-aaaa", + "20260101-000001-bbbb", + "20260101-000002-cccc", + ) + lock_path = runs.lock_path_for(held / STATE_FILE, follow_final_symlink=False) + extra = ("--hard",) if hard else () + + with platform_util.file_lock(lock_path): + doc = _clean_json(repo, capsys, "--retain", "0", *extra) + + # `_clean_json` is the document assertion: rc 0, stdout parses whole, stderr empty + assert doc["protected"] == [held.name] + assert doc[result_key] == [early.name, late.name] # the run AFTER it too + assert doc[other_key] == [] + assert held.is_dir() and not early.exists() and not late.exists() + + +def test_cmd_clean_text_names_a_lock_held_run_and_keeps_its_siblings(project, capsys): + """Text mode tells a held lock apart from the two races it is classified + with, and still reports the siblings it reclaimed around it.""" + install_bmad_config(project) + repo = project.project + early, held, late = _terminal_runs( + repo, + "20260101-000000-aaaa", + "20260101-000001-bbbb", + "20260101-000002-cccc", + ) + lock_path = runs.lock_path_for(held / STATE_FILE, follow_final_symlink=False) + + with platform_util.file_lock(lock_path): + rc = cli.cmd_clean(_clean_args(repo, retain=0, hard=True)) + + assert rc == 0 # not an aborted clean + out, err = capsys.readouterr() + assert f"run {held.name}: run state locked by another process — not removed" in err + assert "engine resumed mid-clean" not in err + assert "agent session appeared mid-clean" not in err + assert held.is_dir() and not early.exists() and not late.exists() + assert "2 deleted" in out + + +def test_cmd_clean_never_files_a_confinement_refusal_as_protected(project, monkeypatch, capsys): + """The typed arm's whole reason for existing, and the ablation that grades it. + + `platform_util.UnconfinedWriteError` is an `OSError` SUBCLASS, so the bare + `except OSError` that would also have stopped the abort would swallow a + containment refusal — "this write escaped its root" — and file it as a benign + `protected`/"left untouched". A refusal that security-relevant must keep + escaping the per-candidate handler, so widening the arm reddens here. + """ + install_bmad_config(project) + repo = project.project + (run_dir,) = _terminal_runs(repo, "20260101-000000-aaaa") + + def unconfined(_project, _run_dir, **_kw): + raise platform_util.UnconfinedWriteError("run dir escaped the runs root") + + monkeypatch.setattr(runs, "delete_run", unconfined) + + with pytest.raises(platform_util.UnconfinedWriteError): + cli.cmd_clean(_clean_args(repo, retain=0, hard=True)) + + assert run_dir.is_dir() # nothing removed, and nothing reported as reclaimed + capsys.readouterr() + + def test_cmd_clean_reclaims_past_a_session_proven_to_be_another_project_s( project, monkeypatch, capsys ): diff --git a/tests/test_engine.py b/tests/test_engine.py index 2cf40e3f..3bbe6b1d 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -11868,7 +11868,7 @@ def fake_signal(sig, handler): monkeypatch.setattr(engine_mod.sys, "platform", "win32") @contextlib.contextmanager - def native_test_lock(_path): + def native_test_lock(_path, **_kw): # This Linux-hosted test patches the process-wide sys.platform token only to # drive Engine's Windows signal branch; msvcrt is intentionally unavailable. yield diff --git a/tests/test_journal.py b/tests/test_journal.py index 7466da16..3821f941 100644 --- a/tests/test_journal.py +++ b/tests/test_journal.py @@ -22,7 +22,9 @@ def test_save_state_retries_transient_sharing_violation(tmp_path, monkeypatch): """On win32, os.replace denied by a concurrent reader is retried, not fatal.""" monkeypatch.setattr(platform_util.sys, "platform", "win32") monkeypatch.setattr(platform_util.time, "sleep", lambda _s: None) # no real backoff - monkeypatch.setattr(journal_mod, "file_lock", contextmanager(lambda _path: iter((None,)))) + monkeypatch.setattr( + journal_mod, "file_lock", contextmanager(lambda _path, **_kw: iter((None,))) + ) real_replace = os.replace calls = {"n": 0} @@ -55,7 +57,7 @@ def test_state_lock_same_run_nesting_acquires_os_lock_once(tmp_path, monkeypatch acquired: list[object] = [] @contextmanager - def recording_lock(path): + def recording_lock(path, **_kw): acquired.append(path) yield @@ -84,7 +86,7 @@ def test_state_lock_same_run_symlink_spellings_acquire_os_lock_once(tmp_path, mo acquired: list[object] = [] @contextmanager - def recording_lock(path): + def recording_lock(path, **_kw): acquired.append(path) yield @@ -133,7 +135,7 @@ def test_state_lock_refuses_different_run_nesting_before_second_acquire(tmp_path acquired: list[object] = [] @contextmanager - def recording_lock(path): + def recording_lock(path, **_kw): acquired.append(path) yield @@ -151,7 +153,7 @@ def test_state_lock_failure_clears_thread_guard(tmp_path, monkeypatch): acquired: list[object] = [] @contextmanager - def recording_lock(path): + def recording_lock(path, **_kw): acquired.append(path) yield @@ -170,7 +172,7 @@ def test_save_state_acquisition_error_writes_nothing(tmp_path, monkeypatch): run_dir = tmp_path / "run" @contextmanager - def refusing_lock(_path): + def refusing_lock(_path, **_kw): raise OSError("lock unavailable") yield @@ -213,10 +215,10 @@ def test_two_concurrent_saves_never_share_the_fixed_temp_file(tmp_path, monkeypa replace_threads: list[str] = [] @contextmanager - def observed_file_lock(path): + def observed_file_lock(path, **_kw): if threading.current_thread().name == "second": second_attempted.set() - with real_file_lock(path): + with real_file_lock(path, **_kw): yield def controlled_replace(src, dst): diff --git a/tests/test_runs.py b/tests/test_runs.py index 4ca41092..1b397132 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -3168,7 +3168,7 @@ def test_rearm_lock_acquisition_failure_leaves_spec_and_state_unchanged(tmp_path state_before = (run_dir / journal_mod.STATE_FILE).read_bytes() @contextlib.contextmanager - def refusing_lock(_path): + def refusing_lock(_path, **_kw): raise OSError("state lock unavailable") yield From 47bb418b4921fe7a40fb5d31f54b11eb3b058262 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 3 Sep 2026 12:29:35 -0700 Subject: [PATCH 17/18] docs(changelog): condense the clean-sweep lock entry The entry ran 13 lines against 5- and 9-line neighbours, carrying DW ids, the exception taxonomy and root-cause narration that belong in docstrings and git history. Keep the what and the user-facing why in 5 lines. --- CHANGELOG.md | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf74dab8..a384ef2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -271,18 +271,10 @@ breaking changes may land in a minor release. the preimage they wrote over is still the bytes the run last claimed; the restore skips and journals `ledger-restore-skipped-diverged`. - **`bmad-loop clean` no longer aborts the whole sweep when one run's state lock is - held.** The per-run lock DW-94/DW-93 added to `runs.delete_run`/`archive_run` is new in - this branch — on `main` neither took a lock — and its acquisition `OSError` escaped - `cmd_clean`'s per-candidate handler, so a single busy run cost the operator the entire - report: later candidates were never processed, and runs already mutated (worktrees - removed, artifacts trimmed) vanished from a document only emitted after the loop. The - contended run is now classified like the lifecycle races beside it — `trimmed` if - anything reached it, else `protected` — and the sweep continues. `file_lock` raises a - typed `LockUnavailableError` for a failed acquisition, and `clean` acquires - non-blocking: `fcntl.flock` never times out, so waiting was unbounded on POSIX, and a - lock someone else holds already means what `clean` reports anyway. The error is typed - rather than a bare `OSError` so a confinement refusal (`UnconfinedWriteError`, also an - `OSError`) is never folded into "left untouched". + held.** A busy run used to end the invocation, so later candidates went unprocessed and + runs already reclaimed vanished from a report only emitted after the loop. `clean` now + takes each run's lock without waiting and records the contended run like the lifecycle + races beside it — `trimmed` if anything reached it, else `protected` — then continues. - **`bmad-loop stop` no longer retries forever on an engine whose identity cannot be read.** The rival-engine compare under the state lock used the local pid after every From f125d52fb0c1beec3db0cad3c0af9e2a467df504 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 3 Sep 2026 19:43:52 -0700 Subject: [PATCH 18/18] fix(tui): decline a contended run's re-arm instead of freezing the dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_do_rearm` took the run's state lock blocking, on Textual's message loop — no `@work`, and its sole caller is the synchronous `push_screen` dismiss callback. On POSIX that wait is unbounded (`fcntl.flock` never times out), and the realistic holder is a rival `resume`, which keeps the lock across config, skills, profiles and a git preflight bounded only by `[limits] git_timeout_s` (120s by default) while publishing its pid last, so the modal's `_engine_possibly_live` gate reads dead for that whole window. Measured: 3.00s against a 3s holder, 8.00s against an 8s holder. Acquire with `blocking=False` and report the contention as a toast. The new `except LockUnavailableError` arm MUST precede the existing `except (RearmError, OSError, runs.StateRootError)`: the error is an `OSError` subclass, so the reverse order makes it dead code and files contention as "re-arm failed". Waiting bought nothing anyway — the post-lock liveness re-check refuses a re-arm against a live rival. The `finally`'s residue echo is skipped on contention: a refused acquisition ran nothing, so any record appended in that window belongs to the holder. --- CHANGELOG.md | 8 ++ docs/FEATURES.md | 2 +- src/bmad_loop/tui/app.py | 45 +++++++- tests/test_tui_app.py | 224 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 271 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a384ef2f..6f640528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -256,6 +256,14 @@ breaking changes may land in a minor release. ### Fixed +- **The TUI's re-arm declines a contended run instead of waiting for it.** The + gesture runs on Textual's message loop, so taking the run's state lock blocking + froze the whole dashboard for as long as a rival held it — unbounded on POSIX, + where `fcntl.flock` never times out, and a rival `resume` holds it across a git + preflight bounded only by `[limits] git_timeout_s`. It now acquires without + waiting and toasts the contention; the post-lock liveness re-check refused that + re-arm anyway. + - **An orphaned mount's reclaim no longer destroys the orchestrator's own artifacts.** The pre-reclaim snapshot drew its untracked candidates from `untracked_files`, which excludes ignored paths by contract, and the deferred-work ledger, sprint board and bound spec are diff --git a/docs/FEATURES.md b/docs/FEATURES.md index f14cf9a1..168bb315 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -212,7 +212,7 @@ Result` section. Every other spec keeps warn-and-continue, and the record says w - Every run is a resumable on-disk state machine: `bmad-loop resume ` continues from a gate, escalation, or interruption. - A graceful stop (`stop --graceful` / TUI `S`) is resumable too: unlike a hard stop killed mid-item, it lets the in-flight item finish through commit and finalizes cleanly, ending as a `stopped` run that `resume` picks up at the next item. -- Every `state.json` publication is serialized by one advisory lock per run, keyed on the resolved run directory plus the logical `state.json` name and stored under the user state root rather than in git. Ignoring a final-component `state.json` symlink keeps that identity stable when atomic publication replaces the directory entry; alternate spellings of the run directory still converge. Multi-step control mutations (`resolve`, `resume`, code-root restamping, and stop's external fallback) hold that same lock from their authoritative read through atomic publication, so a waiter reloads the state its predecessor left instead of overwriting it from a stale snapshot. A fresh run or sweep likewise holds it from its initial state save through trusted-digest and PID publication, preventing an explicit-id resume from observing resumable state before the composer is live. Readers remain lock-free because publication is atomic. Stop does not hold the lock while it requests, signals, polls, or kills: a live engine must be able to publish its own stopped state; only the fallback's final reload/check/write is serialized. That final check preserves an engine that finished during delivery and retries against any newer live engine generation a concurrent resume published. POSIX lock acquisition blocks, while Windows can surface an `OSError` after its bounded wait; either failure aborts the mutation rather than writing unlocked. +- Every `state.json` publication is serialized by one advisory lock per run, keyed on the resolved run directory plus the logical `state.json` name and stored under the user state root rather than in git. Ignoring a final-component `state.json` symlink keeps that identity stable when atomic publication replaces the directory entry; alternate spellings of the run directory still converge. Multi-step control mutations (`resolve`, `resume`, code-root restamping, and stop's external fallback) hold that same lock from their authoritative read through atomic publication, so a waiter reloads the state its predecessor left instead of overwriting it from a stale snapshot. A fresh run or sweep likewise holds it from its initial state save through trusted-digest and PID publication, preventing an explicit-id resume from observing resumable state before the composer is live. Readers remain lock-free because publication is atomic. Stop does not hold the lock while it requests, signals, polls, or kills: a live engine must be able to publish its own stopped state; only the fallback's final reload/check/write is serialized. That final check preserves an engine that finished during delivery and retries against any newer live engine generation a concurrent resume published. POSIX lock acquisition blocks, while Windows can surface an `OSError` after its bounded wait; either failure aborts the mutation rather than writing unlocked. Two callers acquire without waiting instead: `clean`, sweeping many runs, and the TUI's re-arm, which runs on the dashboard's message loop where a blocking POSIX acquisition would freeze the whole UI for as long as the holder kept the lock. Both report the contended run rather than queueing behind it. - All run state in `.bmad-loop/runs//` (gitignored): `state.json` (which records `repo_root`, the git root code work happens in, so an out-of-process re-arm reads back the tree the run measured, #716 — resume re-stamps it from the config.yaml it just re-read, since that is the tree it arms the engine against, and warns when the root moved: the baselines, preserve refs and branches already recorded name objects in the previous one. `resolve` and the TUI's re-arm re-stamp it themselves, before they re-arm: both re-arm and then resume in one gesture, so resume's own re-stamp lands after the re-arm has already advanced the baseline in whichever tree the mirror still named); `journal.jsonl` (every decision, including the `session-synthesized-from-frontmatter` catch and its `spec-marker-repaired` repair, #276, plus one `verify-command-result` per verifier command — emitted on the dev, repair and review legs alike, carrying `verification_stage` and a per-story `verification_sequence` that orders the passes across all three; the two passes that leave no record are `bmad-loop confirm --reverify`, which runs after the run is over, and any pass with no `[verify] commands` configured, which records nothing because nothing ran — each entry also carrying `spawn_error`, set when the verify command's child could not be started at all — typically because its working directory is missing, is not a directory, or cannot be searched, though any spawn-time `OSError` (a missing shell, EMFILE, ENOMEM) reaches the same field and the wrapped exception is what names the cause — which is an environment fault that pauses the run rather than a command that failed — whose stream pointers name the `verify/` directory below, and one `park-proof-of-work-skipped` per attempt that cleared the dev artifact gate on an `awaiting-operator` park with proof-of-work waived — not per park that committed, since the stages after that gate can still reject the attempt — carrying `zero_diff`: `true` when the waived gate found no non-excluded changes (its exclusions include the spec, board, any restore-patch artifact, and an orchestrator-authored deferred-work ledger append), `false` when it found changes, and `null` when the probe could not answer (a git fault, a git refusal such as an unresolvable baseline, or an attempt with no recorded baseline to measure from) and the gate was waived anyway, so the waiver itself is recorded whatever the probe managed to say. `false` is a statement about the tree, not about who wrote what: the gate this stands in for cannot attribute residue to a session in a shared checkout, and the record inherits that limit rather than improving on it); `tasks//` (per-session prompt + shared artifacts: [`result.json`, `escalation.json`] — respectively the per-session result and escalation outputs — plus adapter-specific breadcrumbs: `session-lifecycle.jsonl` records timeout fires, budget-guard trips (`budget-tripped` / `over-budget-fired`), transport-failure classification (`env-fault-classified`, #194), a mux session lost under the run (`session-vanished`, #489) and the #276 forensics (`spec-status-transition-observed`, `frontmatter-unmodified-refused`, `contract-nudge-sent`); `heartbeat.json` is the wait loop's proof-of-life; `resultless-stops.jsonl` records give-up Stops with a verdict — `no-artifact`, `ambiguous-frontmatter`, `unmodified-since-launch`, `terminal-frontmatter-pending`); `logs/`; `verify/` (verifier command stdout/stderr, one file per stream, pointed at by the journal's `verify-command-result` records — its own store, because every name in `logs/` is a session task id the TUI resolves as a pane log; each stream is retained tail-first up to `[verify] stream_capture_kb` (256 KiB, `0` = keep nothing) and the record carries the full byte count, a truncation flag, and a `capture_error` when the write itself failed); `deferred/`; `resolve/`; `ATTENTION`; `ctl-window` (the control-session window id the last TUI launch minted, so attach/stop follow the live window, #482). - One piece deliberately lives **outside** that directory: the hook-event channel (#494) is at `///events/` under the user-scoped state root (`BMAD_LOOP_STATE_DIR`, see the [transport section](#hook-based-transport-no-pane-scraping) below and the README's env-var table), not `/events/`. The orchestrator still polls the legacy in-tree location, so a project whose installed relay predates the move keeps completing its sessions. `delete`, `archive` and `clean` remove the out-of-tree counterpart along with the run dir, and `clean` sweeps counterparts whose run dir is already gone; an **archived** run's tarball therefore no longer contains `events/` — those files are transient completion signals, consumed while the run was live, and everything an archive is read for later is in the run dir. - `journal.jsonl` records `session-end` for every session unconditionally — even a teardown that throws still lands one (status `aborted` when the outcome is unknowable). A timed-out session's entry carries `fired_at` (wall time the deadline was declared), `teardown_s` (wall seconds from that fire to this entry — the teardown gap), and `expired_clock` (`monotonic` / `wall` / `both` — `wall` alone fingerprints a host suspend that froze the monotonic clock). Every entry whose usage was read carries `tokens` (raw) and `tokens_weighted` (cache reads at `limits.cache_read_weight`), keeping per-session spend reconstructible; both are `null` when the usage read failed, and both are absent on an `aborted` end. `tokens_weighted` is the end-of-session total — distinct from a tripped session's `budget_weighted`, the guard's mid-session sample at trip time. diff --git a/src/bmad_loop/tui/app.py b/src/bmad_loop/tui/app.py index e840c8c9..4f9fc686 100644 --- a/src/bmad_loop/tui/app.py +++ b/src/bmad_loop/tui/app.py @@ -38,7 +38,7 @@ RunState, StoryTask, ) -from ..platform_util import resolve_or_lexical +from ..platform_util import LockUnavailableError, resolve_or_lexical from ..policy import POLICY_FILE from ..process_host import ProcessHostError from ..runs import RUNS_DIR, RearmError, StopRunError @@ -983,8 +983,25 @@ def _do_rearm( return before_entries = runs.journal_entries_or_none(run_dir) outcome: runs.RearmOutcome | None = None + contended = False try: - with state_lock(run_dir): + # `blocking=False` because this runs ON the message loop — the reason + # `_guarded` and `_commit_subject` bound their git calls at `timeout_s=5`. + # `_do_rearm` carries no `@work`, and its only caller is the synchronous + # `push_screen` dismiss callback, so the acquisition happens inline: on + # POSIX a blocking wait is UNBOUNDED (`fcntl.flock` does not time out), + # and the rival that holds this lock is typically `cli._prepare_resume_locked`, + # which holds it across config, skills, profiles and a git preflight each + # bounded only by `[limits] git_timeout_s` (120s by default). The whole + # dashboard freezes for that span, and the `_engine_possibly_live` gate on + # the modal does not head it off: `resume` takes the lock FIRST and + # publishes its pid LAST, so for that entire window liveness still reads + # dead and only the lock objects (the same window `cmd_clean` documents). + # + # Refusing loses nothing a wait would have won, either: the post-lock + # liveness re-check below is what the waiter would reach, and against a + # rival resume it refuses anyway. So the wait's only product is the freeze. + with state_lock(run_dir, blocking=False): # Repeat the liveness decision after exclusion. Config/policy work # above is deliberately lock-free; only this bounded restamp+re-arm # mutation gesture is serialized. @@ -1032,6 +1049,23 @@ def _do_rearm( # escalations raised since the marker was written. resolution_recorded=False, ) + except LockUnavailableError: + # ORDER IS LOAD-BEARING: `LockUnavailableError` SUBCLASSES `OSError`, so + # this arm must precede the one below or contention is reported as + # "re-arm failed" — a fault the operator would go looking for — and the + # non-blocking acquire above buys nothing at all. + # + # Caught here and NOT folded into that arm for `cmd_clean`'s reason: a + # bare `except OSError` cannot tell "someone else has this run" from + # `platform_util.UnconfinedWriteError`, and filing a containment refusal + # as a benign "busy" is the exact fold those two classes exist to prevent. + contended = True + self.notify( + f"run {run_id}: run state locked by another process — not re-arming; " + "wait for it to finish, then re-arm", + severity="warning", + ) + return except (RearmError, OSError, runs.StateRootError) as e: self.notify(f"re-arm failed: {e}", severity="error") return @@ -1046,7 +1080,12 @@ def _do_rearm( # path even after they were unified on routing — and an abort is when the # residue matters most: the re-arm half-ran and the operator has to decide # what to do with the tree. - if outcome is None: + # + # `not contended` because this recovery is for a gesture that RAN and + # aborted. A refused acquisition ran nothing, so every record appended + # between the read above and the refusal belongs to the HOLDER — echoing + # the rival's re-arm as this one's residue. + if outcome is None and not contended: self._echo_rearm_events(run_dir, before_entries) assert outcome is not None self._echo_rearm_notices(outcome.notices) diff --git a/tests/test_tui_app.py b/tests/test_tui_app.py index a662d83d..6b4d157c 100644 --- a/tests/test_tui_app.py +++ b/tests/test_tui_app.py @@ -5258,6 +5258,216 @@ def liveness_gate(_self, _run_id, _run_dir): assert checks == ["checked", "checked"] +def test_escalation_rearm_declines_a_run_whose_state_lock_is_held(project, monkeypatch): + """A contended run is REFUSED with a toast, not queued behind its holder. + + `_do_rearm` runs ON Textual's message loop: it carries no `@work`, and its only + caller is the synchronous `push_screen` dismiss callback, so the acquisition + happens inline and the whole dashboard freezes for however long the holder keeps + the lock — unbounded on POSIX, where `fcntl.flock` never times out. The realistic + holder is a rival `resume`, which takes the lock FIRST and publishes its pid LAST, + so the modal's `_engine_possibly_live` gate reads dead for that entire window (the + same window `cmd_clean` documents) and does not head the freeze off. + + The contention is real — the run's own canonical sidecar — and taken through + `platform_util.file_lock` rather than `journal.state_lock` for the reason + test_cleanup states: `state_lock`'s reentrancy guard is thread-local, so acquiring + it here would let `_do_rearm` RE-ENTER the lock and pass for the wrong reason. The + holder sits on a background thread with a BOUNDED hold so that a regression to a + blocking acquire reddens on `elapsed` instead of hanging the suite. + + Ablations, both of which must redden this: (1) MOVE the `except + LockUnavailableError` arm below the existing `except (RearmError, OSError, + runs.StateRootError)` — the subclass makes the moved arm dead and contention is + filed as "re-arm failed"; (2) drop `blocking=False` and the gesture queues for the + holder's full hold. + """ + import threading + import time + + from bmad_loop import platform_util + from bmad_loop.journal import STATE_FILE + + install_bmad_config(project) + run_dir, _spec = _stories_paused_run( + project.project, + stage="escalation", + spec_status="blocked", + spec_checkpoint=False, + blocked_result="Blocked: a rival process holds the run state.", + ) + notes: list[tuple[str, str]] = [] + monkeypatch.setattr(BmadLoopApp, "_resolve_blocked_by_liveness", lambda *_a: False) + monkeypatch.setattr( + BmadLoopApp, + "notify", + lambda _self, message, **kwargs: notes.append( + (str(message), str(kwargs.get("severity", "information"))) + ), + ) + monkeypatch.setattr( + runs_mod, "rearm_escalation", lambda *_a, **_k: pytest.fail("re-armed a locked run") + ) + + lock_path = runs_mod.lock_path_for(run_dir / STATE_FILE, follow_final_symlink=False) + hold_s = 3.0 + held = threading.Event() + release = threading.Event() + + def hold() -> None: + with platform_util.file_lock(lock_path): + held.set() + release.wait(hold_s) + + holder = threading.Thread(target=hold, daemon=True) + holder.start() + try: + assert held.wait(30), "the rival never acquired the run's state lock" + started = time.monotonic() + BmadLoopApp(project.project)._do_rearm(run_dir.name, run_dir, "1") + elapsed = time.monotonic() - started + finally: + release.set() + holder.join(timeout=30) + + assert elapsed < hold_s / 2 # refused at once, not queued behind the holder + # One toast, and it is the contention one: equality over the list covers both + # negatives at once — no "re-arm failed" fault report, and no residue echo + # crediting this gesture with journal records only the HOLDER can have written. + assert [severity for _message, severity in notes] == ["warning"] + assert "run state locked by another process" in notes[0][0] + assert "re-arm failed" not in notes[0][0] + + +def test_rearm_contention_arm_precedes_the_generic_oserror_arm(): + """`LockUnavailableError` SUBCLASSES `OSError`, so `_do_rearm`'s contention arm is + correct only in that ORDER: below the existing `except (RearmError, OSError, + runs.StateRootError)` it is unreachable and every contention is reported as a + fault, leaving the non-blocking acquire buying nothing. + + The behavioral test above reddens on the same swap; this one names the property, so + the failure says what the invariant is instead of leaving the subclass relation to + be rediscovered from a toast. + + Ablation: move the `except LockUnavailableError` arm after the OSError arm. + """ + import ast + import inspect + import textwrap + + from bmad_loop import platform_util + from bmad_loop.tui import app as app_mod + + assert issubclass(platform_util.LockUnavailableError, OSError) # the whole hazard + + def caught(handler: ast.ExceptHandler) -> set[str]: + if handler.type is None: + return {"BaseException"} + names: set[str] = set() + for node in ast.walk(handler.type): + if isinstance(node, ast.Name): + names.add(node.id) + elif isinstance(node, ast.Attribute): + names.add(node.attr) + return names + + tree = ast.parse(textwrap.dedent(inspect.getsource(app_mod.BmadLoopApp._do_rearm))) + paired = 0 + for node in ast.walk(tree): + if not isinstance(node, ast.Try): + continue + arms = [caught(h) for h in node.handlers] + narrow = [i for i, names in enumerate(arms) if "LockUnavailableError" in names] + wide = [ + i + for i, names in enumerate(arms) + if "LockUnavailableError" not in names and names & {"OSError", "BaseException"} + ] + if not narrow or not wide: + continue + paired += 1 + assert max(narrow) < min(wide), ( + "_do_rearm catches LockUnavailableError after a wider OSError arm — the " + "subclass makes the later arm unreachable, so contention is misreported" + ) + assert paired == 1, "_do_rearm no longer pairs a LockUnavailableError arm with an OSError arm" + + +def test_escalation_rearm_contention_does_not_echo_the_holders_journal_records( + project, monkeypatch +): + """A refused acquisition claims none of the HOLDER's journal records. + + `_do_rearm`'s `finally` recovers the re-arm records a raised call had already + written — abort-only diagnostic recovery, as its docstring says. A refused + acquisition is not that abort: it ran nothing, so every record appended between + the pre-lock read and the refusal was written by the process that HOLDS the lock, + and echoing it credits this gesture with a rival's re-arm. + + The rival's append is injected through `journal_entries_or_none` rather than raced + on a real thread because the window is the microseconds between the pre-lock read + and a non-blocking refusal — a real race would be a coin flip, and a negative + assertion that only sometimes has anything to be negative about proves nothing. + The exclusion itself is still the real sidecar lock, so the refusal is genuine. + + Ablation: drop `and not contended` from the `finally` and the holder's record is + toasted here as this gesture's own residue. + """ + import threading + + from bmad_loop import platform_util + from bmad_loop.journal import STATE_FILE + + install_bmad_config(project) + run_dir, _spec = _stories_paused_run( + project.project, + stage="escalation", + spec_status="blocked", + spec_checkpoint=False, + blocked_result="Blocked: a rival process holds the run state.", + ) + notes: list[str] = [] + monkeypatch.setattr(BmadLoopApp, "_resolve_blocked_by_liveness", lambda *_a: False) + monkeypatch.setattr( + BmadLoopApp, "notify", lambda _self, message, **_kwargs: notes.append(str(message)) + ) + + reads = 0 + + def racing_entries(_run_dir): + # First read is the pre-lock watermark; any later one would be the `finally`, + # by when the holder has appended a re-arm record of its own. + nonlocal reads + reads += 1 + if reads == 1: + return [] + return [{"ts": 0.0, "kind": "stale-restore-excluded", "files": ["rival.py"]}] + + monkeypatch.setattr(runs_mod, "journal_entries_or_none", racing_entries) + + lock_path = runs_mod.lock_path_for(run_dir / STATE_FILE, follow_final_symlink=False) + held = threading.Event() + release = threading.Event() + + def hold() -> None: + with platform_util.file_lock(lock_path): + held.set() + release.wait(3.0) + + holder = threading.Thread(target=hold, daemon=True) + holder.start() + try: + assert held.wait(30), "the rival never acquired the run's state lock" + BmadLoopApp(project.project)._do_rearm(run_dir.name, run_dir, "1") + finally: + release.set() + holder.join(timeout=30) + + assert not any("excluded the abandoned restore" in note for note in notes) + assert [note for note in notes if "run state locked by another process" in note] + assert reads == 1 # the `finally` never took the second read at all + + def test_escalation_rearm_reloads_state_before_restamping(project, monkeypatch): """Ablation: delete _do_rearm's fresh state check and the TUI restamps a run whose escalation a rival already consumed while this gesture waited for the lock.""" @@ -5284,7 +5494,7 @@ def test_escalation_rearm_reloads_state_before_restamping(project, monkeypatch): ) @contextlib.contextmanager - def rival_first(_run_dir): + def rival_first(_run_dir, **_kwargs): rival = load_state(run_dir) rival.tasks["1"].phase = Phase.PENDING save_state(run_dir, rival) @@ -5329,7 +5539,7 @@ def test_escalation_rearm_refuses_a_newer_generation_from_an_open_review(project ) @contextlib.contextmanager - def rival_first(_run_dir): + def rival_first(_run_dir, **_kwargs): rival = load_state(run_dir) rival.tasks["1"].generation = expected_generation + 1 save_state(run_dir, rival) @@ -5380,9 +5590,15 @@ def checked_rearm(rd, key, **_kwargs): @pytest.mark.parametrize( "failure", - [OSError("lock unavailable"), runs_mod.StateRootError("no usable state root")], + [OSError("lock file could not be created"), runs_mod.StateRootError("no usable state root")], ) def test_escalation_rearm_reports_state_lock_failures(project, monkeypatch, failure): + """The other half of the contention split: an acquisition fault that is NOT a + holder still reports "re-arm failed". Neither parameter may be a + `LockUnavailableError` — that subclass is contention, routed to its own arm — and + a plain `OSError` is exactly what `platform_util.file_lock` raises when the + sidecar cannot be PROVISIONED, which its docstring keeps deliberately unwrapped + because a broken path is not "someone is using this run".""" import contextlib from bmad_loop import runs @@ -5401,7 +5617,7 @@ def test_escalation_rearm_reports_state_lock_failures(project, monkeypatch, fail monkeypatch.setattr(runs, "rearm_escalation", lambda *_a, **_k: pytest.fail("wrote unlocked")) @contextlib.contextmanager - def refusing_lock(_run_dir): + def refusing_lock(_run_dir, **_kwargs): raise failure yield