From 28080faaa54a72b5d5545649fea5f25c1927f8f4 Mon Sep 17 00:00:00 2001 From: Bram Date: Fri, 4 Sep 2026 23:37:43 +0200 Subject: [PATCH] fix(autoreview): default to Astra with Sol access fallback --- CHANGELOG.md | 3 ++ skills/autoreview/SKILL.md | 30 ++++++------ skills/autoreview/scripts/autoreview | 25 +++++----- skills/autoreview/scripts/autoreview_test.py | 48 ++++++++++---------- 4 files changed, 56 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14763f7..491f0a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ summary: Timeline of guardrail helper changes mirrored from Sweetistics and rela # Changelog +## 2026-09-04: Autoreview Astra Default +- Changed the autoreview skill and helper default to `gpt-6-astra` at high reasoning, using Sol as the access-only fallback and preserving explicit model selection; skill instructions require standard mode. + ## 2026-09-04: PStack Astra Routing - Made `gpt-6-astra` the PStack default for implementation and synthesis, including Poteto playbooks; retained `gpt-5.6-sol` in mixed-model reviews, Arena candidates, and independent audits. All routes use high reasoning and never fast mode. diff --git a/skills/autoreview/SKILL.md b/skills/autoreview/SKILL.md index 775f103..2b39276 100644 --- a/skills/autoreview/SKILL.md +++ b/skills/autoreview/SKILL.md @@ -7,7 +7,7 @@ description: "Pre-commit/ship code review: Codex default; optional Claude or Pi. Run the bundled structured review helper as a closeout check. This is code review, not Guardian `auto_review` approval routing. -Codex review is the default when no engine is set. It uses `gpt-5.6-sol` with `high` reasoning by default, then retries once with `gpt-5.6-terra` only when the account cannot access Sol. Claude review is optional and uses `claude-fable-5` by default. +Codex review is the default when no engine is set. It uses `gpt-6-astra` with `high` reasoning by default, then retries once with `gpt-5.6-sol` only when the account cannot access Astra. Claude review is optional and uses `claude-fable-5` by default. Use standard mode for Codex reviews (`--codex-speed default`); never use fast mode. For user-visible behavior, pair autoreview with `behavior-validator`. Autoreview is source-aware and judges the change bundle; behavior validation is source-blind and judges the running product or tool against a behavior contract. A clean autoreview is not proof that a UI, CLI, API, or generated artifact works from the user's perspective. @@ -31,7 +31,7 @@ Do not require autoreview for a change whose entire diff is prose-only internal - Keep going until structured review returns no accepted/actionable findings only while the work remains inside the original task scope. - If a review-triggered fix changes code, rerun focused tests and rerun the structured review helper. - For security-audit suppression changes, verify accepted findings remain auditable: suppressed findings stay in structured output, active output keeps an unsuppressible suppression notice, and aggregate findings cannot hide unrelated active risk. -- Never switch or override the requested review engine/model except for the documented Codex Sol-to-Terra account-access fallback. Capacity, rate-limit, and unrelated failures keep the same engine/model. +- Never switch or override the requested review engine/model except for the documented Codex Astra-to-Sol account-access fallback. Capacity, rate-limit, and unrelated failures keep the same engine/model. - Be patient with large bundles. Structured review can take up to 30 minutes while the model call is active, especially with Codex tools or web search. - Treat heartbeat lines like `review still running: ... elapsed=... pid=...` as healthy progress, not a hang. Let the helper continue while heartbeats are advancing. Pass `--stream-engine-output` when live engine text is useful; Codex and Claude filter tool/file chatter, other runnable engines pass raw output through. - Do not kill a review just because it has been quiet for 2-5 minutes, or because it is still running under the 30-minute window. Inspect the process only after missing multiple expected heartbeats, after 30 minutes, or after an obviously failed subprocess; prefer letting the same helper command finish. @@ -268,20 +268,20 @@ Run multiple reviewers against one frozen bundle: Set reviewer models and thinking/effort explicitly: ```bash -"$AUTOREVIEW" --reviewers codex,claude --model codex=gpt-5.6-sol --thinking codex=high --model claude=claude-fable-5 --thinking claude=max +"$AUTOREVIEW" --reviewers codex,claude --model codex=gpt-6-astra --thinking codex=high --model claude=claude-fable-5 --thinking claude=max ``` Inline syntax is also supported for simple model IDs: ```bash -"$AUTOREVIEW" --reviewers codex:gpt-5.6-sol:high,claude:claude-fable-5:max +"$AUTOREVIEW" --reviewers codex:gpt-6-astra:high,claude:claude-fable-5:max ``` For models with slashes or extra colons, prefer keyed form: ```bash "$AUTOREVIEW" --engine pi --model anthropic/claude-sonnet-4 --thinking high -"$AUTOREVIEW" --reviewers codex,pi --model codex=gpt-5.6-sol --model pi=anthropic/claude-sonnet-4 +"$AUTOREVIEW" --reviewers codex,pi --model codex=gpt-6-astra --model pi=anthropic/claude-sonnet-4 ``` `--reviewers all` covers Codex, Claude, and Pi. Droid, Copilot, Cursor, and OpenCode selections fail closed because their current CLI contracts cannot confine project instructions, filesystem reads, or network fetches to the review boundary. @@ -294,14 +294,14 @@ Recommended model defaults: | Engine | Default model | Source note | | ------------------- | -------------------------------------------------- | ----------------------------------------------------- | -| **codex** (default) | `gpt-5.6-sol` -> `gpt-5.6-terra` on access failure | OpenClaw org review default | +| **codex** (default) | `gpt-6-astra` -> `gpt-5.6-sol` on access failure | Default Codex review model | | **claude** | `claude-fable-5` | Anthropic's most capable widely released Claude model | CLI flags and environment variables override these defaults. Pi does not get a built-in model default because its provider catalog may vary by installation. Droid, Copilot, Cursor, and OpenCode are currently refused. | Engine | Model flag | Example model IDs | Thinking flag | Accepted levels | | ------------------- | -------------------------- | ---------------------------------------------------------------------------- | ----------------------------- | ---------------------------------------------------------- | -| **codex** (default) | `codex --model X exec ...` | `gpt-5.6-sol`, then `gpt-5.6-terra` on Sol access failure | `-c model_reasoning_effort=Y` | `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max` | +| **codex** (default) | `codex --model X exec ...` | `gpt-6-astra`, then `gpt-5.6-sol` on Astra access failure | `-c model_reasoning_effort=Y` | `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max` | | **claude** | `claude --model X` | `claude-fable-5`, `claude-opus-4-8`, `claude-sonnet-4-6`, `claude-haiku-4-5` | `--effort Y` | `low`, `medium`, `high`, `xhigh`, `max` | | **droid** | currently refused | Factory model IDs | `-r, --reasoning-effort Y` | `off`, `none`, `low`, `medium`, `high`, `xhigh`, `max` | | **copilot** | currently refused | Copilot model aliases | not supported | n/a | @@ -311,19 +311,19 @@ CLI flags and environment variables override these defaults. Pi does not get a b Claude also supports `--fallback-model a,b` for availability-based fallback chains ([model-config](https://code.claude.com/docs/en/model-config)). Current Claude docs note that auth, billing, rate-limit, request-size, and transport errors do not trigger fallback, and the changelog documents interactive-session support in `v2.1.166`. -[OpenAI's model guidance](https://developers.openai.com/api/docs/guides/latest-model) identifies Sol as the GPT-5.6 frontier-capability route and documents `max` support. Autoreview keeps `high` as its default; use `max` only for the hardest quality-first reviews after comparing its latency and cost with `xhigh` on representative changes. +Autoreview defaults to Astra at `high` reasoning. Explicit model requests, including `gpt-5.6-sol`, remain supported; the automatic Sol access fallback applies only to Astra. Examples matching current `main` behavior: ```bash # Codex with explicit model and reasoning -"$AUTOREVIEW" --engine codex --model gpt-5.6-sol --thinking high +"$AUTOREVIEW" --engine codex --model gpt-6-astra --thinking high -# Codex fast mode (priority service tier); needs a model whose catalog lists the tier, silently standard otherwise -"$AUTOREVIEW" --engine codex --codex-speed fast +# Codex standard mode; never request fast mode +"$AUTOREVIEW" --engine codex --codex-speed default # Safe Codex model/response tuning overrides (--codex-speed wins over a service_tier here) -"$AUTOREVIEW" --engine codex --codex-config 'service_tier="fast"' +"$AUTOREVIEW" --engine codex --codex-config 'service_tier="default"' # Claude Code aliases or full model names, with optional availability fallback "$AUTOREVIEW" --engine claude --model claude-fable-5 --thinking max @@ -350,9 +350,9 @@ loader such as an untracked `.envrc`; the helper does not write a config file. | `AUTOREVIEW_MODEL` | Override the built-in default `--model` for all engines | | `AUTOREVIEW_THINKING` | Default `--thinking` for all engines | | `AUTOREVIEW_FALLBACK_MODEL` | Default Claude `--fallback-model` chain | -| `AUTOREVIEW__MODEL` | Per-engine model override, for example `AUTOREVIEW_CODEX_MODEL=gpt-5.6-sol` | +| `AUTOREVIEW__MODEL` | Per-engine model override, for example `AUTOREVIEW_CODEX_MODEL=gpt-6-astra` | | `AUTOREVIEW__THINKING` | Per-engine thinking override | -| `AUTOREVIEW_CODEX_CONFIG` | Safe Codex model/response tuning overrides, semicolon-separated, e.g. `service_tier="fast"`; capability-bearing keys fail closed | +| `AUTOREVIEW_CODEX_CONFIG` | Safe Codex model/response tuning overrides, semicolon-separated, e.g. `service_tier="default"`; capability-bearing keys fail closed | | `AUTOREVIEW_CODEX_SPEED` | Codex service tier override: `fast` (priority), `flex`, or `default`; silently standard when the model does not list the tier | | `AUTOREVIEW_CLAUDE_FALLBACK_MODEL` | Claude-only fallback chain | | `AUTOREVIEW_PROVIDER_ENV_ALLOW` | Comma-separated custom Pi/OpenCode credential variable names; names must end in a recognized credential suffix | @@ -423,7 +423,7 @@ The helper: - supports `--dry-run`, `--parallel-tests`, `--parallel-tests-shell`, `--prompt`, repo-relative `--prompt-file`, repo-relative `--dataset`, `--no-tools`, `--no-web-search`, repeatable Codex-only safe model/response tuning with `--codex-config key=value`, Codex-only `--codex-speed fast|flex|default`, and commit refs - supports `--stream-engine-output` or `AUTOREVIEW_STREAM_ENGINE_OUTPUT=1` for live engine text while preserving structured validation; Codex and Claude hide tool/file event details, emit compact activity summaries, and report usage at turn completion - supports opt-in review panels with `--panel` / `--reviewers`, plus per-engine `--model`, `--thinking`, and Claude `--fallback-model` -- uses built-in defaults `codex=gpt-5.6-sol` with `high` reasoning and an access-only `gpt-5.6-terra` retry, plus `claude=claude-fable-5`; honors `AUTOREVIEW_MODEL`, `AUTOREVIEW_THINKING`, `AUTOREVIEW_FALLBACK_MODEL`, and per-engine `AUTOREVIEW__MODEL` / `AUTOREVIEW__THINKING` environment overrides when CLI flags are omitted +- uses built-in defaults `codex=gpt-6-astra` with `high` reasoning and an access-only `gpt-5.6-sol` retry, plus `claude=claude-fable-5`; honors `AUTOREVIEW_MODEL`, `AUTOREVIEW_THINKING`, `AUTOREVIEW_FALLBACK_MODEL`, and per-engine `AUTOREVIEW__MODEL` / `AUTOREVIEW__THINKING` environment overrides when CLI flags are omitted - gives Codex the bundle in an empty workspace with web search available; Claude receives the bundle plus WebSearch by default and optional domain-constrained WebFetch, and Pi receives the bundle with no tools - runs Claude with `--safe-mode` (`v2.1.169+`), `--setting-sources user`, MCP and auto-memory disabled, no filesystem/shell tools, an empty external workspace, and `--fallback-model` when set - refuses Droid, Copilot, Cursor, and OpenCode reviews until their CLIs expose the required project, filesystem, and network isolation diff --git a/skills/autoreview/scripts/autoreview b/skills/autoreview/scripts/autoreview index 4aae660..5dc1265 100755 --- a/skills/autoreview/scripts/autoreview +++ b/skills/autoreview/scripts/autoreview @@ -781,10 +781,10 @@ TEST_ENV_ALLOWED_EXACT = { } TEST_ENV_ALLOWED_PREFIXES = ("AUTOREVIEW_FAKE_", "LC_") DEFAULT_MODEL_BY_ENGINE = { - "codex": "gpt-5.6-sol", + "codex": "gpt-6-astra", "claude": "claude-fable-5", } -DEFAULT_CODEX_ACCESS_FALLBACK_MODEL = "gpt-5.6-terra" +DEFAULT_CODEX_ACCESS_FALLBACK_MODEL = "gpt-5.6-sol" DEFAULT_THINKING_BY_ENGINE = { "codex": "high", } @@ -11113,12 +11113,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--base") parser.add_argument("--commit", default="HEAD") parser.add_argument("--engine", choices=ENGINE_CHOICES, default=os.environ.get("AUTOREVIEW_ENGINE", "codex")) - parser.add_argument("--reviewers", help="Comma-separated review panel, e.g. codex,claude,pi or codex:gpt-5.6-sol:high.") + parser.add_argument("--reviewers", help="Comma-separated review panel, e.g. codex,claude,pi or codex:gpt-6-astra:high.") parser.add_argument("--panel", action="store_true", help="Run a Codex/Claude review panel unless --engine changes the first reviewer.") parser.add_argument( "--model", action="append", - help="Model for all reviewers or engine=model. Repeatable. Defaults: codex=gpt-5.6-sol with an access-only gpt-5.6-terra retry, claude=claude-fable-5.", + help="Model for all reviewers or engine=model. Repeatable. Defaults: codex=gpt-6-astra with an access-only gpt-5.6-sol retry, claude=claude-fable-5.", ) parser.add_argument("--thinking", action="append", help="Thinking/effort for all reviewers or engine=level. Repeatable. Codex: none, minimal, low, medium, high, xhigh, max. Claude: low, medium, high, xhigh, max. Droid: off, none, low, medium, high. Pi: off, minimal, low, medium, high, xhigh. OpenCode: minimal, low, medium, high, max. Cursor: none.") parser.add_argument( @@ -11559,17 +11559,20 @@ def self_test_config_defaults() -> None: for key in keys: os.environ.pop(key, None) default_codex = reviewer_args(reviewer_test_args(engine="codex"))[0] - if default_codex.model != "gpt-5.6-sol": + if default_codex.model != "gpt-6-astra": raise SystemExit(f"self-test config defaults failed: default codex model={default_codex.model!r}") - if default_codex.fallback_model != "gpt-5.6-terra": + if default_codex.fallback_model != "gpt-5.6-sol": raise SystemExit( f"self-test config defaults failed: default codex fallback={default_codex.fallback_model!r}" ) - explicit_sol = reviewer_args(reviewer_test_args(engine="codex", model=["gpt-5.6-sol"]))[0] - if explicit_sol.fallback_model != "gpt-5.6-terra": + explicit_astra = reviewer_args(reviewer_test_args(engine="codex", model=["gpt-6-astra"]))[0] + if explicit_astra.fallback_model != "gpt-5.6-sol": raise SystemExit( - f"self-test config defaults failed: explicit Sol access fallback={explicit_sol.fallback_model!r}" + f"self-test config defaults failed: explicit Astra access fallback={explicit_astra.fallback_model!r}" ) + explicit_sol = reviewer_args(reviewer_test_args(engine="codex", model=["gpt-5.6-sol"]))[0] + if explicit_sol.model != "gpt-5.6-sol" or explicit_sol.fallback_model is not None: + raise SystemExit("self-test config defaults failed: explicit Sol must stay pinned") if default_codex.thinking != "high": raise SystemExit(f"self-test config defaults failed: default codex thinking={default_codex.thinking!r}") max_effort = reviewer_args(reviewer_test_args(engine="codex", thinking=["max"]))[0] @@ -11684,7 +11687,7 @@ def self_test_fallback_scope() -> None: reviewers = reviewer_args(base) codex = next(r for r in reviewers if r.engine == "codex") claude = next(r for r in reviewers if r.engine == "claude") - if codex.fallback_model != "gpt-5.6-terra": + if codex.fallback_model != "gpt-5.6-sol": raise SystemExit( f"self-test fallback scope failed: codex access fallback={codex.fallback_model!r}" ) @@ -11711,7 +11714,7 @@ def self_test_fallback_scope() -> None: panel = reviewer_args(reviewer_test_args(reviewers="codex,claude", fallback_model=["cli-global"])) panel_codex = next(r for r in panel if r.engine == "codex") panel_claude = next(r for r in panel if r.engine == "claude") - if panel_codex.fallback_model != "gpt-5.6-terra" or panel_claude.fallback_model != "cli-global": + if panel_codex.fallback_model != "gpt-5.6-sol" or panel_claude.fallback_model != "cli-global": raise SystemExit("self-test fallback scope failed: CLI global fallback should apply only to Claude panel reviewers") try: reviewer_args(reviewer_test_args(engine="codex", fallback_model=["cli-global"])) diff --git a/skills/autoreview/scripts/autoreview_test.py b/skills/autoreview/scripts/autoreview_test.py index f3cbc20..348fb21 100644 --- a/skills/autoreview/scripts/autoreview_test.py +++ b/skills/autoreview/scripts/autoreview_test.py @@ -241,13 +241,13 @@ def test_codex_config_status_exposes_keys_only(self) -> None: args = argparse.Namespace(codex_config=['model_verbosity="low"']) self.assertEqual(AUTOREVIEW.codex_config_keys(args), ["model_verbosity"]) - def test_codex_retries_terra_after_sol_access_failure(self) -> None: + def test_codex_retries_sol_after_astra_access_failure(self) -> None: args = argparse.Namespace( codex_bin="codex", codex_config=None, codex_speed=None, - fallback_model="gpt-5.6-terra", - model="gpt-5.6-sol", + fallback_model="gpt-5.6-sol", + model="gpt-6-astra", stream_engine_output=False, thinking="high", tools=True, @@ -258,12 +258,12 @@ def test_codex_retries_terra_after_sol_access_failure(self) -> None: def fake_run(command: list[str], *_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: model = command[command.index("--model") + 1] models.append(model) - if model == "gpt-5.6-sol": + if model == "gpt-6-astra": return subprocess.CompletedProcess( command, 1, "", - "The model `gpt-5.6-sol` does not exist or you do not have access to it.", + "The model `gpt-6-astra` does not exist or you do not have access to it.", ) output_path = Path(command[command.index("--output-last-message") + 1]) output_path.write_text(json.dumps(FINAL_REPORT)) @@ -285,7 +285,7 @@ def fake_run(command: list[str], *_args: object, **_kwargs: object) -> subproces output = AUTOREVIEW.run_codex(args, Path(tmpdir), "review") self.assertEqual(json.loads(output), FINAL_REPORT) - self.assertEqual(models, ["gpt-5.6-sol", "gpt-5.6-terra"]) + self.assertEqual(models, ["gpt-6-astra", "gpt-5.6-sol"]) def test_codex_runs_outside_repo_with_bundle_only_workspace(self) -> None: args = argparse.Namespace( @@ -293,7 +293,7 @@ def test_codex_runs_outside_repo_with_bundle_only_workspace(self) -> None: codex_config=None, codex_speed=None, fallback_model=None, - model="gpt-5.6-sol", + model="gpt-6-astra", stream_engine_output=False, thinking="high", tools=True, @@ -374,8 +374,8 @@ def test_codex_does_not_fallback_after_unrelated_failure(self) -> None: codex_bin="codex", codex_config=None, codex_speed=None, - fallback_model="gpt-5.6-terra", - model="gpt-5.6-sol", + fallback_model="gpt-5.6-sol", + model="gpt-6-astra", stream_engine_output=False, thinking="high", tools=True, @@ -403,15 +403,15 @@ def fake_run(command: list[str], *_args: object, **_kwargs: object) -> subproces with self.assertRaisesRegex(SystemExit, "network timeout"): AUTOREVIEW.run_codex(args, Path(tmpdir), "review") - self.assertEqual(models, ["gpt-5.6-sol"]) + self.assertEqual(models, ["gpt-6-astra"]) def test_codex_does_not_fallback_after_model_capacity_failure(self) -> None: args = argparse.Namespace( codex_bin="codex", codex_config=None, codex_speed=None, - fallback_model="gpt-5.6-terra", - model="gpt-5.6-sol", + fallback_model="gpt-5.6-sol", + model="gpt-6-astra", stream_engine_output=False, thinking="high", tools=True, @@ -425,7 +425,7 @@ def fake_run(command: list[str], *_args: object, **_kwargs: object) -> subproces command, 1, "", - "model_not_available: gpt-5.6-sol is temporarily unavailable due to capacity", + "model_not_available: gpt-6-astra is temporarily unavailable due to capacity", ) with tempfile.TemporaryDirectory(prefix="autoreview-codex-fallback.") as tmpdir, mock.patch.object( @@ -444,30 +444,30 @@ def fake_run(command: list[str], *_args: object, **_kwargs: object) -> subproces with self.assertRaisesRegex(SystemExit, "temporarily unavailable"): AUTOREVIEW.run_codex(args, Path(tmpdir), "review") - self.assertEqual(models, ["gpt-5.6-sol"]) + self.assertEqual(models, ["gpt-6-astra"]) def test_codex_access_fallback_ignores_structured_output_text(self) -> None: result = subprocess.CompletedProcess( ["codex"], 1, - '{"type":"agent_message","text":"gpt-5.6-sol does not exist or you do not have access"}', - '{"type":"agent_message","message":"gpt-5.6-sol does not exist or you do not have access"}', + '{"type":"agent_message","text":"gpt-6-astra does not exist or you do not have access"}', + '{"type":"agent_message","message":"gpt-6-astra does not exist or you do not have access"}', ) self.assertFalse( - AUTOREVIEW.codex_model_access_failure(result, "gpt-5.6-sol") + AUTOREVIEW.codex_model_access_failure(result, "gpt-6-astra") ) def test_codex_access_fallback_accepts_terminal_error_event(self) -> None: result = subprocess.CompletedProcess( ["codex"], 1, - '{"type":"error","message":"gpt-5.6-sol does not exist or you do not have access"}', + '{"type":"error","message":"gpt-6-astra does not exist or you do not have access"}', "", ) self.assertTrue( - AUTOREVIEW.codex_model_access_failure(result, "gpt-5.6-sol") + AUTOREVIEW.codex_model_access_failure(result, "gpt-6-astra") ) def test_codex_access_fallback_accepts_account_model_list_error(self) -> None: @@ -476,25 +476,25 @@ def test_codex_access_fallback_accepts_account_model_list_error(self) -> None: 1, "", ( - "The model gpt-5.6-sol does not appear in the list of models " + "The model gpt-6-astra does not appear in the list of models " "available to your account" ), ) self.assertTrue( - AUTOREVIEW.codex_model_access_failure(result, "gpt-5.6-sol") + AUTOREVIEW.codex_model_access_failure(result, "gpt-6-astra") ) def test_codex_access_fallback_ignores_plain_stdout(self) -> None: - message = "gpt-5.6-sol does not exist or you do not have access" + message = "gpt-6-astra does not exist or you do not have access" stdout_result = subprocess.CompletedProcess(["codex"], 1, message, "") stderr_result = subprocess.CompletedProcess(["codex"], 1, "", message) self.assertFalse( - AUTOREVIEW.codex_model_access_failure(stdout_result, "gpt-5.6-sol") + AUTOREVIEW.codex_model_access_failure(stdout_result, "gpt-6-astra") ) self.assertTrue( - AUTOREVIEW.codex_model_access_failure(stderr_result, "gpt-5.6-sol") + AUTOREVIEW.codex_model_access_failure(stderr_result, "gpt-6-astra") ) def test_extract_json_accepts_dict_result_payload(self) -> None: