From c78d16b21ec495696d281030051fc6c0a5db193b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 08:14:12 +0000 Subject: [PATCH] Adaptive implementer tier on fix passes; pin ponytail-review to sonnet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up iterations solve progressively smaller findings, but model tiering priced every iteration like the first. Two changes, asymmetric by design because loop economics are asymmetric — under zero-findings convergence a reviewer's hallucinated finding costs a full fix round, so judges must not get cheaper while implementation may: - Fix pass: on iteration >=2 with only minor/nit findings open and no design-level change, an auto-resolved implementer drops one tier (sonnet -> haiku), bumping back if a finding survives its round. Explicitly pinned models never drop. Reviewer models never change mid-loop: the PASS that ends the loop must not come from a weaker judge than the FAIL that opened it. Mirrors the architect's existing revision-pass downgrade. - Default config pins ponytail-review to sonnet instead of auto (which resolved to opus on medium/large): a one-line deletion/leanness lens doesn't earn opus, and sonnet keeps the review floor. docs/shepherd-config.md updated to match; contract + config tests pin both rules; 94 tests pass; SKILL.md at 448/450 lines. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016G4uCkx2btpqfGtJRAPJKd --- .claude/skills/shepherd/SKILL.md | 6 +++++- .claude/skills/shepherd/config.default.json | 2 +- docs/shepherd-config.md | 11 ++++++++++- tests/test_config_registry.py | 7 +++++++ tests/test_orchestrator_contract.py | 11 +++++++++++ 5 files changed, 34 insertions(+), 3 deletions(-) diff --git a/.claude/skills/shepherd/SKILL.md b/.claude/skills/shepherd/SKILL.md index 6d76928..8f1d041 100644 --- a/.claude/skills/shepherd/SKILL.md +++ b/.claude/skills/shepherd/SKILL.md @@ -168,7 +168,11 @@ explicit name (`opus`, `sonnet`, `haiku`) is used verbatim. Resolve `"auto"` as: `final_reviewer` → `opus` (`sonnet` for `trivial`/`small`). `sonnet` is the floor for review — never `haiku`. The panel never resolves entirely onto the implementer's model: keep at least one reviewer — and one final reviewer when the panel has any — on a different model, because a judge -sharing the generator's model favors its output (self-preference bias). Pre-gate stages (verify, explorer, architect, success_criteria) resolve `"auto"` +sharing the generator's model favors its output (self-preference bias). On a fix pass — +iteration ≥2 with only `minor`/`nit` findings open and no design-level change — an auto-resolved +implementer drops one tier (`sonnet` → `haiku`); a finding that survives its fix round bumps it +back. An explicit config model never drops. Reviewer models never change mid-loop — the PASS +that ends the loop must not come from a weaker judge than the FAIL that opened it. Pre-gate stages (verify, explorer, architect, success_criteria) resolve `"auto"` at dispatch time from this table. At step 4 (Design gate), record all picks in `_panel.json`: the pre-gate ones as the record of what ran, the post-gate ones (implementer, reviewers, final reviewers, fulfillment) for the human to edit before approving. diff --git a/.claude/skills/shepherd/config.default.json b/.claude/skills/shepherd/config.default.json index 67972e0..4ec8a67 100644 --- a/.claude/skills/shepherd/config.default.json +++ b/.claude/skills/shepherd/config.default.json @@ -5,7 +5,7 @@ ], "final_reviewers": [ { "use": "thermonuclear", "model": "auto" }, - { "use": "ponytail-review", "model": "auto" } + { "use": "ponytail-review", "model": "sonnet" } ] }, "oracle": { diff --git a/docs/shepherd-config.md b/docs/shepherd-config.md index c50359b..9186387 100644 --- a/docs/shepherd-config.md +++ b/docs/shepherd-config.md @@ -94,7 +94,7 @@ Core/shared or public-contract changes are `medium` at minimum regardless of lin "reviewers": [{ "use": "staff-review", "model": "auto" }], "final_reviewers": [ { "use": "thermonuclear", "model": "auto" }, - { "use": "ponytail-review", "model": "auto" } + { "use": "ponytail-review", "model": "sonnet" } ] }, "oracle": { "commands": [] }, @@ -114,6 +114,15 @@ Single stages may be **model-only** (`{ "model": "..." }` with no `use`): built- that model. All `"auto"` picks are resolved and shown at the design gate, where you can adjust any before approving. +Auto is also adaptive across the loop: on a fix pass (iteration ≥2 with only minor/nit findings +open and no design-level change) an auto-resolved implementer drops one tier, bumping back if a +finding survives its round; explicitly pinned models never drop, and reviewer models never change +mid-loop, so the verdict that ends the loop is never issued by a weaker judge than the one that +raised the findings. The shipped default pins `ponytail-review` to `sonnet`: a one-line +deletion/leanness lens doesn't earn `opus`, while `sonnet` keeps the review floor — under +zero-findings convergence a hallucinated nit costs a full fix round, so review never drops to +`haiku`. + `oracle.commands` should be finite and non-mutating. Good defaults for JS/TS repos are: ```json diff --git a/tests/test_config_registry.py b/tests/test_config_registry.py index 33b68cf..951bfab 100644 --- a/tests/test_config_registry.py +++ b/tests/test_config_registry.py @@ -58,3 +58,10 @@ def test_reviewer_without_use_reported(): bad["stages"]["reviewers"] = [{"model": "sonnet"}] errs = validate(bad, REGISTRY) assert any("reviewers" in e and "use" in e for e in errs) + + +def test_default_config_pins_ponytail_to_sonnet(): + # A one-line deletion lens doesn't earn opus; sonnet stays the review floor because + # under zero-findings convergence a hallucinated nit costs a full fix round. + finals = {e["use"]: e.get("model") for e in load_json(CONFIG)["stages"]["final_reviewers"]} + assert finals["ponytail-review"] == "sonnet" diff --git a/tests/test_orchestrator_contract.py b/tests/test_orchestrator_contract.py index aefdc82..63206a4 100644 --- a/tests/test_orchestrator_contract.py +++ b/tests/test_orchestrator_contract.py @@ -364,3 +364,14 @@ def test_review_panel_never_matches_implementer_model_entirely(): def test_pr_body_scales_with_diff(): assert "mechanism and known limitations" in ORCH + + +def test_implementer_fix_pass_is_adaptive_but_judges_are_stable(): + # Iteration >=2 fix rounds are transcription-like, so auto drops the implementer a tier; + # judges keep verdict continuity — the converging PASS never comes from a weaker model. + assert "On a fix pass" in ORCH + assert "drops one tier" in ORCH + assert "bumps it" in ORCH + assert "An explicit config model never drops" in ORCH + assert "never change mid-loop" in ORCH + assert "weaker judge" in ORCH