From 1a39a41b81ac218e3de97b11cc92739de919788f Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 00:03:44 +0200 Subject: [PATCH 01/18] docs(bench): llms-benchmark row for deepseek/deepseek-v4.1-flash on opencode under 1.13.0 Run 1: 16/16 confirmed (10/3/3), 29m05s, 0.16 USD. Run 2 stopped at 49 min with its report still generating; the command now says a second run already worse than the first is stopped, and records the mission's 669-character length since #620. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .claude/commands/launch-llms-benchmark.md | 18 ++++++++++++++++-- .llms-benchmark/README.md | 4 ++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.claude/commands/launch-llms-benchmark.md b/.claude/commands/launch-llms-benchmark.md index 1fde4e5..5ea2fa2 100644 --- a/.claude/commands/launch-llms-benchmark.md +++ b/.claude/commands/launch-llms-benchmark.md @@ -256,6 +256,19 @@ Steps: ceiling for a run that never produces a turn or declines to drive (step 8): a decline counts as one of the two. + **A second run that is already worse than the first is stopped, not + finished.** Watch it against the first run's figures: once it has + been running longer than the first run's whole duration and its + report is not written yet, or its drive has not started by the time + the first run had finished, kill it by its PID — it can no longer + beat the first on duration, its cost is already spent for nothing, + and the twenty-to-forty minutes it still needs are better given to + the next model. The row is the first run; the pull request records + the second's launch time, the phase it was stopped in and its spend + to that point. Decided on 2026-09-20 by the maintainer after the + second run of one model passed the first's 29-minute total with its + report still being generated, at 49 minutes. + For each run: record the UTC timestamp **before** launching — step 7 needs it to identify the session. Then one headless run, from the repository root, on the work branch. @@ -378,10 +391,11 @@ Steps: opencode it is not**: `opencode run` stores the argument wrapped in one more pair of escaped quotes than it was given (verified on 2026-09-14 with a smoke argument: `"x"` passed, `"\"x\""` stored, after - decoding), so the argument to pass is the same 683-character string + decoding), so the argument to pass is the same 669-character string as under the other two CLIs, and the store's form passed verbatim reaches the run double-wrapped (one run of 2026-09-12 did, at 693 - characters). + characters, when the mission still named a depth and was 683 long — + #620 dropped the depth phrase, and every run since is 669). Add exactly three things to that line and nothing else: that the services' sources are under `.llms-benchmark/src/`; that you want diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 10edf25..92eed63 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -16,7 +16,7 @@ One row per model and CLI, always its latest run. | Rank | Model | CLI | oddyssey | Confirmed / reported | Telemetry / Perf / Behavior | Total | Cost | $/confirmed | | --- | --- | --- | --- | --- | --- | --- | --- | --- | | **#1** | `z-ai/glm-5.3` | opencode | 1.12.0 | 18 / 19 | 12 / 4 / 3 | 12m23s | $1.98 | $0.110 | -| **#2** | `deepseek/deepseek-v4.1-flash` | opencode | 1.12.0 | **10 / 10** | 5 / 3 / 2 | 9m54s | $0.20 | $0.020 | +| **#2** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | $0.010 | | **#3** | `z-ai/glm-5.3-flashx` | opencode | 1.12.0 | 12 / 14 | 3 / 6 / 3 | 16m02s | $0.42 | $0.035 | | **#4** | `openai/gpt-5.6-luna` | copilot | 1.12.0 | 6 / 7 | 3 / 3 / 1 | **7m02s** | **$0.10** | **$0.016** | | **#5** | `google/gemini-3.7-flash` | opencode | 1.12.0 | **7 / 7** | 3 / 3 / 1 | 8m34s | $0.87 | $0.124 | @@ -36,7 +36,7 @@ One row per model and CLI, always its latest run. | Model | CLI | oddyssey | Preflight | Drive | Observation | Turns | Median turn | Input | Output | Cache | Signals | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | `z-ai/glm-5.3` | opencode | 1.12.0 | 1m28s | 2m01s | 8m54s | 46 | 6.1s | 4.2M | 139k | 4.0M | 4/4 | -| `deepseek/deepseek-v4.1-flash` | opencode | 1.12.0 | 1m27s | 2m01s | 6m26s | 48 | 4.2s | 4.1M | 76k | 3.8M | 4/4 | +| `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | 5m59s | 2m02s | 21m04s | 73 | 11.9s | 8.0M | 94k | 7.4M | 4/4 | | `z-ai/glm-5.3-flashx` | opencode | 1.12.0 | 3m32s | 2m02s | 10m28s | 39 | 13.8s | 3.2M | 84k | 3.0M | 4/4 | | `openai/gpt-5.6-luna` | copilot | 1.12.0 | 1m03s | 2m01s | 3m58s | 38 | 3.2s | 2.5M | 15k | 2.5M | 4/4 | | `google/gemini-3.7-flash` | opencode | 1.12.0 | 2m11s | 2m02s | 4m21s | 71 | 4.3s | 4.2M | 31k | 3.5M | 4/4 | From 8583e76526671055fbb08ac2862456f71be64011 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 00:28:50 +0200 Subject: [PATCH 02/18] docs(bench): llms-benchmark row for google/gemini-3.7-flash on opencode under 1.13.0 Run 2 is the row: 8/9 confirmed (4/3/1), 10m17s, 1.08 USD; run 1 7/8 in 9m28s at 1.00 USD. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 92eed63..bbe2b44 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -19,7 +19,7 @@ One row per model and CLI, always its latest run. | **#2** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | $0.010 | | **#3** | `z-ai/glm-5.3-flashx` | opencode | 1.12.0 | 12 / 14 | 3 / 6 / 3 | 16m02s | $0.42 | $0.035 | | **#4** | `openai/gpt-5.6-luna` | copilot | 1.12.0 | 6 / 7 | 3 / 3 / 1 | **7m02s** | **$0.10** | **$0.016** | -| **#5** | `google/gemini-3.7-flash` | opencode | 1.12.0 | **7 / 7** | 3 / 3 / 1 | 8m34s | $0.87 | $0.124 | +| **#5** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | | **#6** | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 14 / 15 | 11 / 3 / 1 | 14m50s | $2.47 | $0.176 | | **#7** | `openai/gpt-5.6-sol` | copilot | 1.12.0 | 11 / 12 | 8 / 3 / 1 | 9m39s | $3.12 | $0.284 | | **#8** | `google/gemini-3.8-flash` | opencode | 1.12.0 | **10 / 10** | 5 / 4 / 1 | 14m59s | $1.96 | $0.196 | @@ -39,7 +39,7 @@ One row per model and CLI, always its latest run. | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | 5m59s | 2m02s | 21m04s | 73 | 11.9s | 8.0M | 94k | 7.4M | 4/4 | | `z-ai/glm-5.3-flashx` | opencode | 1.12.0 | 3m32s | 2m02s | 10m28s | 39 | 13.8s | 3.2M | 84k | 3.0M | 4/4 | | `openai/gpt-5.6-luna` | copilot | 1.12.0 | 1m03s | 2m01s | 3m58s | 38 | 3.2s | 2.5M | 15k | 2.5M | 4/4 | -| `google/gemini-3.7-flash` | opencode | 1.12.0 | 2m11s | 2m02s | 4m21s | 71 | 4.3s | 4.2M | 31k | 3.5M | 4/4 | +| `google/gemini-3.7-flash` | opencode | 1.13.0 | 2m24s | 2m02s | 5m51s | 90 | 4.3s | 6.5M | 32k | 5.8M | 4/4 | | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 1m39s | 2m00s | 11m11s | 64 | 3.9s | 6.5M | 54k | 6.5M | 4/4 | | `openai/gpt-5.6-sol` | copilot | 1.12.0 | 1m38s | 2m00s | 6m01s | 53 | 4.6s | 3.9M | 22k | 3.8M | 4/4 | | `google/gemini-3.8-flash` | opencode | 1.12.0 | 4m26s | 2m02s | 8m31s | 135 | 3.1s | 10.7M | 33k | 9.2M | 4/4 | From 55438d3adf75548bcc85ca7d49c1b2d4fa2bcf51 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 01:02:17 +0200 Subject: [PATCH 03/18] docs(bench): llms-benchmark row for z-ai/glm-5.3 on opencode under 1.13.0 Run 1 is the row: 17/19 confirmed (9/4/4), 19m44s, 1.39 USD; run 2 8/10 in 8m04s at 1.11 USD. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index bbe2b44..ff12564 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -15,7 +15,7 @@ One row per model and CLI, always its latest run. | Rank | Model | CLI | oddyssey | Confirmed / reported | Telemetry / Perf / Behavior | Total | Cost | $/confirmed | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| **#1** | `z-ai/glm-5.3` | opencode | 1.12.0 | 18 / 19 | 12 / 4 / 3 | 12m23s | $1.98 | $0.110 | +| **#1** | `z-ai/glm-5.3` | opencode | 1.13.0 | 17 / 19 | 9 / 4 / 4 | 19m44s | $1.39 | $0.082 | | **#2** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | $0.010 | | **#3** | `z-ai/glm-5.3-flashx` | opencode | 1.12.0 | 12 / 14 | 3 / 6 / 3 | 16m02s | $0.42 | $0.035 | | **#4** | `openai/gpt-5.6-luna` | copilot | 1.12.0 | 6 / 7 | 3 / 3 / 1 | **7m02s** | **$0.10** | **$0.016** | @@ -35,7 +35,7 @@ One row per model and CLI, always its latest run. | Model | CLI | oddyssey | Preflight | Drive | Observation | Turns | Median turn | Input | Output | Cache | Signals | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `z-ai/glm-5.3` | opencode | 1.12.0 | 1m28s | 2m01s | 8m54s | 46 | 6.1s | 4.2M | 139k | 4.0M | 4/4 | +| `z-ai/glm-5.3` | opencode | 1.13.0 | 3m17s | 2m02s | 14m25s | 44 | 9.4s | 4.4M | 121k | 4.0M | 4/4 | | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | 5m59s | 2m02s | 21m04s | 73 | 11.9s | 8.0M | 94k | 7.4M | 4/4 | | `z-ai/glm-5.3-flashx` | opencode | 1.12.0 | 3m32s | 2m02s | 10m28s | 39 | 13.8s | 3.2M | 84k | 3.0M | 4/4 | | `openai/gpt-5.6-luna` | copilot | 1.12.0 | 1m03s | 2m01s | 3m58s | 38 | 3.2s | 2.5M | 15k | 2.5M | 4/4 | From 1c75acdfa4d7251838f79323b6cd8f6e07d76af7 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 01:38:03 +0200 Subject: [PATCH 04/18] docs(bench): llms-benchmark row for z-ai/glm-5.3-flashx on opencode under 1.13.0 Run 1 is the row: 11/13 confirmed (4/4/3), 17m09s, 0.33 USD; run 2 11/13 in 13m51s at 0.42 USD (tie, run 1 cheaper). Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index ff12564..89af742 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -17,7 +17,7 @@ One row per model and CLI, always its latest run. | --- | --- | --- | --- | --- | --- | --- | --- | --- | | **#1** | `z-ai/glm-5.3` | opencode | 1.13.0 | 17 / 19 | 9 / 4 / 4 | 19m44s | $1.39 | $0.082 | | **#2** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | $0.010 | -| **#3** | `z-ai/glm-5.3-flashx` | opencode | 1.12.0 | 12 / 14 | 3 / 6 / 3 | 16m02s | $0.42 | $0.035 | +| **#3** | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 11 / 13 | 4 / 4 / 3 | 17m09s | $0.33 | $0.030 | | **#4** | `openai/gpt-5.6-luna` | copilot | 1.12.0 | 6 / 7 | 3 / 3 / 1 | **7m02s** | **$0.10** | **$0.016** | | **#5** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | | **#6** | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 14 / 15 | 11 / 3 / 1 | 14m50s | $2.47 | $0.176 | @@ -37,7 +37,7 @@ One row per model and CLI, always its latest run. | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | `z-ai/glm-5.3` | opencode | 1.13.0 | 3m17s | 2m02s | 14m25s | 44 | 9.4s | 4.4M | 121k | 4.0M | 4/4 | | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | 5m59s | 2m02s | 21m04s | 73 | 11.9s | 8.0M | 94k | 7.4M | 4/4 | -| `z-ai/glm-5.3-flashx` | opencode | 1.12.0 | 3m32s | 2m02s | 10m28s | 39 | 13.8s | 3.2M | 84k | 3.0M | 4/4 | +| `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 3m10s | 2m01s | 11m58s | 32 | 13.3s | 2.3M | 68k | 2.1M | 4/4 | | `openai/gpt-5.6-luna` | copilot | 1.12.0 | 1m03s | 2m01s | 3m58s | 38 | 3.2s | 2.5M | 15k | 2.5M | 4/4 | | `google/gemini-3.7-flash` | opencode | 1.13.0 | 2m24s | 2m02s | 5m51s | 90 | 4.3s | 6.5M | 32k | 5.8M | 4/4 | | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 1m39s | 2m00s | 11m11s | 64 | 3.9s | 6.5M | 54k | 6.5M | 4/4 | From e7a0765955fcf66d17ab238ff1c725e8c5256c74 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 02:24:36 +0200 Subject: [PATCH 05/18] docs(bench): llms-benchmark row for google/gemini-3.8-flash on opencode under 1.13.0 Run 1 is the row: 12/12 confirmed (6/4/2), 20m39s, 2.29 USD; run 2 9/9 in 21m07s at 2.32 USD. Both runs lost an observe-run session to a provider 400 (corrupted thought signature) and re-drove the scenario before reporting. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 89af742..b0d970d 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -22,7 +22,7 @@ One row per model and CLI, always its latest run. | **#5** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | | **#6** | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 14 / 15 | 11 / 3 / 1 | 14m50s | $2.47 | $0.176 | | **#7** | `openai/gpt-5.6-sol` | copilot | 1.12.0 | 11 / 12 | 8 / 3 / 1 | 9m39s | $3.12 | $0.284 | -| **#8** | `google/gemini-3.8-flash` | opencode | 1.12.0 | **10 / 10** | 5 / 4 / 1 | 14m59s | $1.96 | $0.196 | +| **#8** | `google/gemini-3.8-flash` | opencode | 1.13.0 | **12 / 12** | 6 / 4 / 2 | 20m39s | $2.29 | $0.191 | | **#9** | `anthropic/claude-opus-5` | claude | 1.12.0 | 18 / 19 | 12 / 6 / 1 | 21m55s | $6.33 | $0.352 | | **#10** | `anthropic/claude-fable-5.1` | claude | 1.12.0 | **17 / 17** | 10 / 5 / 2 | 17m08s | $7.55 | $0.444 | | **#11** | `z-ai/glm-5.3-flash` | opencode | 1.12.0 | 13 / 14 | 7 / 4 / 2 | 66m43s | $0.24 | $0.018 | @@ -42,7 +42,7 @@ One row per model and CLI, always its latest run. | `google/gemini-3.7-flash` | opencode | 1.13.0 | 2m24s | 2m02s | 5m51s | 90 | 4.3s | 6.5M | 32k | 5.8M | 4/4 | | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 1m39s | 2m00s | 11m11s | 64 | 3.9s | 6.5M | 54k | 6.5M | 4/4 | | `openai/gpt-5.6-sol` | copilot | 1.12.0 | 1m38s | 2m00s | 6m01s | 53 | 4.6s | 3.9M | 22k | 3.8M | 4/4 | -| `google/gemini-3.8-flash` | opencode | 1.12.0 | 4m26s | 2m02s | 8m31s | 135 | 3.1s | 10.7M | 33k | 9.2M | 4/4 | +| `google/gemini-3.8-flash` | opencode | 1.13.0 | 9m58s | 2m03s | 8m38s | 148 | 4.4s | 12.8M | 59k | 11.1M | 4/4 | | `anthropic/claude-opus-5` | claude | 1.12.0 | 3m36s | 2m00s | 16m19s | 57 | 8.6s | 6.3M | 59k | 6.3M | 4/4 | | `anthropic/claude-fable-5.1` | claude | 1.12.0 | 2m52s | 2m02s | 12m14s | 36 | 3.6s | 3.6M | 60k | 3.6M | 4/4 | | `z-ai/glm-5.3-flash` | opencode | 1.12.0 | 11m45s | 2m00s | 52m58s | 46 | 48.1s | 4.0M | 115k | 2.5M | 4/4 | From 5e80978f7adf063237891983ea12b2a6de6a0bf0 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 02:44:13 +0200 Subject: [PATCH 06/18] docs(bench): llms-benchmark row for openai/gpt-5.6-luna on copilot under 1.13.0 Run 1 is the row: 7/8 confirmed (3/2/2), 6m29s, 0.11 USD at list; run 2 5/6 in 6m53s at 0.12 USD. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index b0d970d..485fa53 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -18,7 +18,7 @@ One row per model and CLI, always its latest run. | **#1** | `z-ai/glm-5.3` | opencode | 1.13.0 | 17 / 19 | 9 / 4 / 4 | 19m44s | $1.39 | $0.082 | | **#2** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | $0.010 | | **#3** | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 11 / 13 | 4 / 4 / 3 | 17m09s | $0.33 | $0.030 | -| **#4** | `openai/gpt-5.6-luna` | copilot | 1.12.0 | 6 / 7 | 3 / 3 / 1 | **7m02s** | **$0.10** | **$0.016** | +| **#4** | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 7 / 8 | 3 / 2 / 2 | **6m29s** | **$0.11** | **$0.016** | | **#5** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | | **#6** | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 14 / 15 | 11 / 3 / 1 | 14m50s | $2.47 | $0.176 | | **#7** | `openai/gpt-5.6-sol` | copilot | 1.12.0 | 11 / 12 | 8 / 3 / 1 | 9m39s | $3.12 | $0.284 | @@ -38,7 +38,7 @@ One row per model and CLI, always its latest run. | `z-ai/glm-5.3` | opencode | 1.13.0 | 3m17s | 2m02s | 14m25s | 44 | 9.4s | 4.4M | 121k | 4.0M | 4/4 | | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | 5m59s | 2m02s | 21m04s | 73 | 11.9s | 8.0M | 94k | 7.4M | 4/4 | | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 3m10s | 2m01s | 11m58s | 32 | 13.3s | 2.3M | 68k | 2.1M | 4/4 | -| `openai/gpt-5.6-luna` | copilot | 1.12.0 | 1m03s | 2m01s | 3m58s | 38 | 3.2s | 2.5M | 15k | 2.5M | 4/4 | +| `openai/gpt-5.6-luna` | copilot | 1.13.0 | 1m02s | 2m01s | 3m26s | 38 | 2.8s | 2.9M | 17k | 2.9M | 4/4 | | `google/gemini-3.7-flash` | opencode | 1.13.0 | 2m24s | 2m02s | 5m51s | 90 | 4.3s | 6.5M | 32k | 5.8M | 4/4 | | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 1m39s | 2m00s | 11m11s | 64 | 3.9s | 6.5M | 54k | 6.5M | 4/4 | | `openai/gpt-5.6-sol` | copilot | 1.12.0 | 1m38s | 2m00s | 6m01s | 53 | 4.6s | 3.9M | 22k | 3.8M | 4/4 | From ff933cf61eebc9753b5ec668d1c95961f0c90d8f Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 03:07:51 +0200 Subject: [PATCH 07/18] docs(bench): llms-benchmark row for openai/gpt-5.6-sol on copilot under 1.13.0 Run 1 is the row: 12/13 confirmed (8/4/0), 9m32s, 1.41 USD at list; run 2 9/10 in 9m59s at 1.31 USD. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 485fa53..bef91e6 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -21,7 +21,7 @@ One row per model and CLI, always its latest run. | **#4** | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 7 / 8 | 3 / 2 / 2 | **6m29s** | **$0.11** | **$0.016** | | **#5** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | | **#6** | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 14 / 15 | 11 / 3 / 1 | 14m50s | $2.47 | $0.176 | -| **#7** | `openai/gpt-5.6-sol` | copilot | 1.12.0 | 11 / 12 | 8 / 3 / 1 | 9m39s | $3.12 | $0.284 | +| **#7** | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 12 / 13 | 8 / 4 / 0 | 9m32s | $1.41 | $0.117 | | **#8** | `google/gemini-3.8-flash` | opencode | 1.13.0 | **12 / 12** | 6 / 4 / 2 | 20m39s | $2.29 | $0.191 | | **#9** | `anthropic/claude-opus-5` | claude | 1.12.0 | 18 / 19 | 12 / 6 / 1 | 21m55s | $6.33 | $0.352 | | **#10** | `anthropic/claude-fable-5.1` | claude | 1.12.0 | **17 / 17** | 10 / 5 / 2 | 17m08s | $7.55 | $0.444 | @@ -41,7 +41,7 @@ One row per model and CLI, always its latest run. | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 1m02s | 2m01s | 3m26s | 38 | 2.8s | 2.9M | 17k | 2.9M | 4/4 | | `google/gemini-3.7-flash` | opencode | 1.13.0 | 2m24s | 2m02s | 5m51s | 90 | 4.3s | 6.5M | 32k | 5.8M | 4/4 | | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 1m39s | 2m00s | 11m11s | 64 | 3.9s | 6.5M | 54k | 6.5M | 4/4 | -| `openai/gpt-5.6-sol` | copilot | 1.12.0 | 1m38s | 2m00s | 6m01s | 53 | 4.6s | 3.9M | 22k | 3.8M | 4/4 | +| `openai/gpt-5.6-sol` | copilot | 1.13.0 | 1m21s | 2m00s | 6m11s | 48 | 4.2s | 3.7M | 28k | 3.5M | 4/4 | | `google/gemini-3.8-flash` | opencode | 1.13.0 | 9m58s | 2m03s | 8m38s | 148 | 4.4s | 12.8M | 59k | 11.1M | 4/4 | | `anthropic/claude-opus-5` | claude | 1.12.0 | 3m36s | 2m00s | 16m19s | 57 | 8.6s | 6.3M | 59k | 6.3M | 4/4 | | `anthropic/claude-fable-5.1` | claude | 1.12.0 | 2m52s | 2m02s | 12m14s | 36 | 3.6s | 3.6M | 60k | 3.6M | 4/4 | From 1a63a1961c5400d12765cc52bfa2ababacea8120 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 03:24:52 +0200 Subject: [PATCH 08/18] docs(bench): llms-benchmark row for openai/gpt-5.6-terra on copilot under 1.13.0 Run 2 is the row: 7/8 confirmed (2/3/2), 5m58s, 0.88 USD at list, the root observing without a subagent; run 1 6/7 in 7m44s at 1.25 USD. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index bef91e6..3c9c246 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -20,7 +20,7 @@ One row per model and CLI, always its latest run. | **#3** | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 11 / 13 | 4 / 4 / 3 | 17m09s | $0.33 | $0.030 | | **#4** | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 7 / 8 | 3 / 2 / 2 | **6m29s** | **$0.11** | **$0.016** | | **#5** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | -| **#6** | `openai/gpt-5.6-terra` | copilot | 1.12.0 | 14 / 15 | 11 / 3 / 1 | 14m50s | $2.47 | $0.176 | +| **#6** | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 7 / 8 | 2 / 3 / 2 | 5m58s | $0.88 | $0.126 | | **#7** | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 12 / 13 | 8 / 4 / 0 | 9m32s | $1.41 | $0.117 | | **#8** | `google/gemini-3.8-flash` | opencode | 1.13.0 | **12 / 12** | 6 / 4 / 2 | 20m39s | $2.29 | $0.191 | | **#9** | `anthropic/claude-opus-5` | claude | 1.12.0 | 18 / 19 | 12 / 6 / 1 | 21m55s | $6.33 | $0.352 | @@ -40,7 +40,7 @@ One row per model and CLI, always its latest run. | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 3m10s | 2m01s | 11m58s | 32 | 13.3s | 2.3M | 68k | 2.1M | 4/4 | | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 1m02s | 2m01s | 3m26s | 38 | 2.8s | 2.9M | 17k | 2.9M | 4/4 | | `google/gemini-3.7-flash` | opencode | 1.13.0 | 2m24s | 2m02s | 5m51s | 90 | 4.3s | 6.5M | 32k | 5.8M | 4/4 | -| `openai/gpt-5.6-terra` | copilot | 1.12.0 | 1m39s | 2m00s | 11m11s | 64 | 3.9s | 6.5M | 54k | 6.5M | 4/4 | +| `openai/gpt-5.6-terra` | copilot | 1.13.0 | 0m36s | 2m01s | 3m21s | 26 | 3.2s | 2.4M | 13k | 2.4M | 4/4 | | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 1m21s | 2m00s | 6m11s | 48 | 4.2s | 3.7M | 28k | 3.5M | 4/4 | | `google/gemini-3.8-flash` | opencode | 1.13.0 | 9m58s | 2m03s | 8m38s | 148 | 4.4s | 12.8M | 59k | 11.1M | 4/4 | | `anthropic/claude-opus-5` | claude | 1.12.0 | 3m36s | 2m00s | 16m19s | 57 | 8.6s | 6.3M | 59k | 6.3M | 4/4 | From 11679ecb63b284b9bc41ef66ea25ef033ff6f2af Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 04:02:10 +0200 Subject: [PATCH 09/18] docs(bench): llms-benchmark row for anthropic/claude-sonnet-5 on claude under 1.13.0 Run 1 is the row: 6/6 confirmed (2/3/1), 13m55s, 3.37 USD. Run 2 is void: its root dispatched the observation with model: opus, and the command now says a run whose report another model wrote is no row. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .claude/commands/launch-llms-benchmark.md | 11 +++++++++++ .llms-benchmark/README.md | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.claude/commands/launch-llms-benchmark.md b/.claude/commands/launch-llms-benchmark.md index 5ea2fa2..8ea87b9 100644 --- a/.claude/commands/launch-llms-benchmark.md +++ b/.claude/commands/launch-llms-benchmark.md @@ -561,6 +561,17 @@ Steps: result must equal it, and the transcript is `~/.claude/projects//.jsonl`. Anything else, stop and say so. + - **the observation must have run on the benchmarked model.** The + root session may dispatch `observe-run` through the Agent tool with + a `model` of its own choosing — on 2026-09-20 a `claude-sonnet-5` + root passed `model: 'opus'` on one run of two, and the subagent's + 89 requests, 90 % of the spend and the whole report were opus-5's. + Read `modelUsage`'s keys: the benchmarked model's key, the + `claude-haiku-4-5` background key, and nothing else carrying spend. + A run whose report was written by another model measured that + model: it is no row for this one, whatever it found — record it as + void in the pull request with its cost, and the other run is the + row (or re-run when it was the only one). - reconstruct the cost from the transcripts, root and `subagents/*.jsonl` together, at Anthropic's published list prices for the model: an `assistant` line is written once per content diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 3c9c246..1a19f5c 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -28,7 +28,7 @@ One row per model and CLI, always its latest run. | **#11** | `z-ai/glm-5.3-flash` | opencode | 1.12.0 | 13 / 14 | 7 / 4 / 2 | 66m43s | $0.24 | $0.018 | | **#12** | `qwen/qwen3.8-27b` | opencode | 1.12.0 | 14 / 16 | 10 / 3 / 3 | 43m52s | $2.09 | $0.150 | | **#13** | `qwen/qwen3.8-max-0902` | opencode | 1.12.0 | **14 / 14** | 8 / 3 / 3 | 66m09s | $2.88 | $0.206 | -| **#14** | `anthropic/claude-sonnet-5` | claude | 1.12.0 | **5 / 5** | 2 / 2 / 1 | 12m20s | $2.55 | $0.511 | +| **#14** | `anthropic/claude-sonnet-5` | claude | 1.13.0 | **6 / 6** | 2 / 3 / 1 | 13m55s | $3.37 | $0.561 |
Run detail — phases, turns, tokens @@ -48,7 +48,7 @@ One row per model and CLI, always its latest run. | `z-ai/glm-5.3-flash` | opencode | 1.12.0 | 11m45s | 2m00s | 52m58s | 46 | 48.1s | 4.0M | 115k | 2.5M | 4/4 | | `qwen/qwen3.8-27b` | opencode | 1.12.0 | 10m51s | 2m01s | 31m00s | 80 | 24.8s | 11.5M | 121k | 10.7M | 4/4 | | `qwen/qwen3.8-max-0902` | opencode | 1.12.0 | 9m00s | 2m02s | 55m07s | 47 | 51.1s | 5.5M | 153k | 5.2M | 4/4 | -| `anthropic/claude-sonnet-5` | claude | 1.12.0 | 2m11s | 2m00s | 8m09s | 67 | 2.1s | 7.5M | 43k | 7.5M | 4/4 | +| `anthropic/claude-sonnet-5` | claude | 1.13.0 | 2m36s | 2m02s | 9m17s | 84 | 2.0s | 10.6M | 52k | 10.6M | 4/4 | Token counts are rounded; the exact figures are in each run's pull request. Input includes the cached share, so Input and Cache overlap by From e00c44eb9bdab7a6274aa281f5278eb4792d686e Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 04:48:29 +0200 Subject: [PATCH 10/18] docs(bench): llms-benchmark row for anthropic/claude-opus-5 on claude under 1.13.0 Run 2 is the row: 17/17 confirmed (8/6/3), 19m31s, 6.15 USD; run 1 17/18 in 22m07s at 7.51 USD (tie, run 2 cheaper). Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 1a19f5c..81b0c0a 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -23,7 +23,7 @@ One row per model and CLI, always its latest run. | **#6** | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 7 / 8 | 2 / 3 / 2 | 5m58s | $0.88 | $0.126 | | **#7** | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 12 / 13 | 8 / 4 / 0 | 9m32s | $1.41 | $0.117 | | **#8** | `google/gemini-3.8-flash` | opencode | 1.13.0 | **12 / 12** | 6 / 4 / 2 | 20m39s | $2.29 | $0.191 | -| **#9** | `anthropic/claude-opus-5` | claude | 1.12.0 | 18 / 19 | 12 / 6 / 1 | 21m55s | $6.33 | $0.352 | +| **#9** | `anthropic/claude-opus-5` | claude | 1.13.0 | **17 / 17** | 8 / 6 / 3 | 19m31s | $6.15 | $0.362 | | **#10** | `anthropic/claude-fable-5.1` | claude | 1.12.0 | **17 / 17** | 10 / 5 / 2 | 17m08s | $7.55 | $0.444 | | **#11** | `z-ai/glm-5.3-flash` | opencode | 1.12.0 | 13 / 14 | 7 / 4 / 2 | 66m43s | $0.24 | $0.018 | | **#12** | `qwen/qwen3.8-27b` | opencode | 1.12.0 | 14 / 16 | 10 / 3 / 3 | 43m52s | $2.09 | $0.150 | @@ -43,7 +43,7 @@ One row per model and CLI, always its latest run. | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 0m36s | 2m01s | 3m21s | 26 | 3.2s | 2.4M | 13k | 2.4M | 4/4 | | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 1m21s | 2m00s | 6m11s | 48 | 4.2s | 3.7M | 28k | 3.5M | 4/4 | | `google/gemini-3.8-flash` | opencode | 1.13.0 | 9m58s | 2m03s | 8m38s | 148 | 4.4s | 12.8M | 59k | 11.1M | 4/4 | -| `anthropic/claude-opus-5` | claude | 1.12.0 | 3m36s | 2m00s | 16m19s | 57 | 8.6s | 6.3M | 59k | 6.3M | 4/4 | +| `anthropic/claude-opus-5` | claude | 1.13.0 | 2m47s | 2m02s | 14m42s | 53 | 6.6s | 5.8M | 62k | 5.8M | 4/4 | | `anthropic/claude-fable-5.1` | claude | 1.12.0 | 2m52s | 2m02s | 12m14s | 36 | 3.6s | 3.6M | 60k | 3.6M | 4/4 | | `z-ai/glm-5.3-flash` | opencode | 1.12.0 | 11m45s | 2m00s | 52m58s | 46 | 48.1s | 4.0M | 115k | 2.5M | 4/4 | | `qwen/qwen3.8-27b` | opencode | 1.12.0 | 10m51s | 2m01s | 31m00s | 80 | 24.8s | 11.5M | 121k | 10.7M | 4/4 | From f6da1a40e9656cc62b9e859e711170801ef939e2 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 07:18:43 +0200 Subject: [PATCH 11/18] docs(bench): llms-benchmark row for qwen/qwen3.8-max-0902 on opencode under 1.13.0 Run 1 is the row: 15/16 confirmed (8/4/3), 30m25s, 1.50 USD; run 2 14/14 in 30m15s at 1.55 USD. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 81b0c0a..fea324c 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -27,7 +27,7 @@ One row per model and CLI, always its latest run. | **#10** | `anthropic/claude-fable-5.1` | claude | 1.12.0 | **17 / 17** | 10 / 5 / 2 | 17m08s | $7.55 | $0.444 | | **#11** | `z-ai/glm-5.3-flash` | opencode | 1.12.0 | 13 / 14 | 7 / 4 / 2 | 66m43s | $0.24 | $0.018 | | **#12** | `qwen/qwen3.8-27b` | opencode | 1.12.0 | 14 / 16 | 10 / 3 / 3 | 43m52s | $2.09 | $0.150 | -| **#13** | `qwen/qwen3.8-max-0902` | opencode | 1.12.0 | **14 / 14** | 8 / 3 / 3 | 66m09s | $2.88 | $0.206 | +| **#13** | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 15 / 16 | 8 / 4 / 3 | 30m25s | $1.50 | $0.100 | | **#14** | `anthropic/claude-sonnet-5` | claude | 1.13.0 | **6 / 6** | 2 / 3 / 1 | 13m55s | $3.37 | $0.561 |
@@ -47,7 +47,7 @@ One row per model and CLI, always its latest run. | `anthropic/claude-fable-5.1` | claude | 1.12.0 | 2m52s | 2m02s | 12m14s | 36 | 3.6s | 3.6M | 60k | 3.6M | 4/4 | | `z-ai/glm-5.3-flash` | opencode | 1.12.0 | 11m45s | 2m00s | 52m58s | 46 | 48.1s | 4.0M | 115k | 2.5M | 4/4 | | `qwen/qwen3.8-27b` | opencode | 1.12.0 | 10m51s | 2m01s | 31m00s | 80 | 24.8s | 11.5M | 121k | 10.7M | 4/4 | -| `qwen/qwen3.8-max-0902` | opencode | 1.12.0 | 9m00s | 2m02s | 55m07s | 47 | 51.1s | 5.5M | 153k | 5.2M | 4/4 | +| `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 4m04s | 2m01s | 24m20s | 34 | 22.5s | 2.8M | 67k | 2.5M | 4/4 | | `anthropic/claude-sonnet-5` | claude | 1.13.0 | 2m36s | 2m02s | 9m17s | 84 | 2.0s | 10.6M | 52k | 10.6M | 4/4 | Token counts are rounded; the exact figures are in each run's pull From 16e505bd63fbfe889cb1cd1fa879c39e225d7aee Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 07:53:05 +0200 Subject: [PATCH 12/18] docs(bench): llms-benchmark row for z-ai/glm-5.3-flash on opencode under 1.13.0 Attempt 2 is the row: 7/8 confirmed (3/4/0), 32m15s, 0.10 USD; attempt 1 was killed after four 504s on the report turn. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index fea324c..3ce937f 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -25,7 +25,7 @@ One row per model and CLI, always its latest run. | **#8** | `google/gemini-3.8-flash` | opencode | 1.13.0 | **12 / 12** | 6 / 4 / 2 | 20m39s | $2.29 | $0.191 | | **#9** | `anthropic/claude-opus-5` | claude | 1.13.0 | **17 / 17** | 8 / 6 / 3 | 19m31s | $6.15 | $0.362 | | **#10** | `anthropic/claude-fable-5.1` | claude | 1.12.0 | **17 / 17** | 10 / 5 / 2 | 17m08s | $7.55 | $0.444 | -| **#11** | `z-ai/glm-5.3-flash` | opencode | 1.12.0 | 13 / 14 | 7 / 4 / 2 | 66m43s | $0.24 | $0.018 | +| **#11** | `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 7 / 8 | 3 / 4 / 0 | 32m15s | $0.10 | $0.014 | | **#12** | `qwen/qwen3.8-27b` | opencode | 1.12.0 | 14 / 16 | 10 / 3 / 3 | 43m52s | $2.09 | $0.150 | | **#13** | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 15 / 16 | 8 / 4 / 3 | 30m25s | $1.50 | $0.100 | | **#14** | `anthropic/claude-sonnet-5` | claude | 1.13.0 | **6 / 6** | 2 / 3 / 1 | 13m55s | $3.37 | $0.561 | @@ -45,7 +45,7 @@ One row per model and CLI, always its latest run. | `google/gemini-3.8-flash` | opencode | 1.13.0 | 9m58s | 2m03s | 8m38s | 148 | 4.4s | 12.8M | 59k | 11.1M | 4/4 | | `anthropic/claude-opus-5` | claude | 1.13.0 | 2m47s | 2m02s | 14m42s | 53 | 6.6s | 5.8M | 62k | 5.8M | 4/4 | | `anthropic/claude-fable-5.1` | claude | 1.12.0 | 2m52s | 2m02s | 12m14s | 36 | 3.6s | 3.6M | 60k | 3.6M | 4/4 | -| `z-ai/glm-5.3-flash` | opencode | 1.12.0 | 11m45s | 2m00s | 52m58s | 46 | 48.1s | 4.0M | 115k | 2.5M | 4/4 | +| `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 6m42s | 2m03s | 23m30s | 34 | 22.3s | 2.3M | 74k | 1.8M | 4/4 | | `qwen/qwen3.8-27b` | opencode | 1.12.0 | 10m51s | 2m01s | 31m00s | 80 | 24.8s | 11.5M | 121k | 10.7M | 4/4 | | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 4m04s | 2m01s | 24m20s | 34 | 22.5s | 2.8M | 67k | 2.5M | 4/4 | | `anthropic/claude-sonnet-5` | claude | 1.13.0 | 2m36s | 2m02s | 9m17s | 84 | 2.0s | 10.6M | 52k | 10.6M | 4/4 | From 07cb0a38d2df88f7143b7911ea5c9f58d6374e54 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 08:35:30 +0200 Subject: [PATCH 13/18] docs(bench): llms-benchmark row for qwen/qwen3.8-27b on opencode under 1.13.0 Attempt 2 is the row: 11/11 confirmed (5/3/3), 40m25s, 1.62 USD; attempt 1 was killed after a 14-minute silent stream at 45 min. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 3ce937f..e6276e7 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -26,7 +26,7 @@ One row per model and CLI, always its latest run. | **#9** | `anthropic/claude-opus-5` | claude | 1.13.0 | **17 / 17** | 8 / 6 / 3 | 19m31s | $6.15 | $0.362 | | **#10** | `anthropic/claude-fable-5.1` | claude | 1.12.0 | **17 / 17** | 10 / 5 / 2 | 17m08s | $7.55 | $0.444 | | **#11** | `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 7 / 8 | 3 / 4 / 0 | 32m15s | $0.10 | $0.014 | -| **#12** | `qwen/qwen3.8-27b` | opencode | 1.12.0 | 14 / 16 | 10 / 3 / 3 | 43m52s | $2.09 | $0.150 | +| **#12** | `qwen/qwen3.8-27b` | opencode | 1.13.0 | **11 / 11** | 5 / 3 / 3 | 40m25s | $1.62 | $0.147 | | **#13** | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 15 / 16 | 8 / 4 / 3 | 30m25s | $1.50 | $0.100 | | **#14** | `anthropic/claude-sonnet-5` | claude | 1.13.0 | **6 / 6** | 2 / 3 / 1 | 13m55s | $3.37 | $0.561 | @@ -46,7 +46,7 @@ One row per model and CLI, always its latest run. | `anthropic/claude-opus-5` | claude | 1.13.0 | 2m47s | 2m02s | 14m42s | 53 | 6.6s | 5.8M | 62k | 5.8M | 4/4 | | `anthropic/claude-fable-5.1` | claude | 1.12.0 | 2m52s | 2m02s | 12m14s | 36 | 3.6s | 3.6M | 60k | 3.6M | 4/4 | | `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 6m42s | 2m03s | 23m30s | 34 | 22.3s | 2.3M | 74k | 1.8M | 4/4 | -| `qwen/qwen3.8-27b` | opencode | 1.12.0 | 10m51s | 2m01s | 31m00s | 80 | 24.8s | 11.5M | 121k | 10.7M | 4/4 | +| `qwen/qwen3.8-27b` | opencode | 1.13.0 | 3m49s | 2m00s | 34m36s | 48 | 16.6s | 5.9M | 133k | 3.7M | 4/4 | | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 4m04s | 2m01s | 24m20s | 34 | 22.5s | 2.8M | 67k | 2.5M | 4/4 | | `anthropic/claude-sonnet-5` | claude | 1.13.0 | 2m36s | 2m02s | 9m17s | 84 | 2.0s | 10.6M | 52k | 10.6M | 4/4 | From 3a8869f236d5626e0117c1a015e9639da4c8a67f Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 08:36:36 +0200 Subject: [PATCH 14/18] docs(bench): re-sort the llms-benchmark table for the 1.13.0 campaign Thirteen rows re-measured under 1.13.0, one row (claude-fable-5.1) kept at 1.12.0 and marked provisional; the rank weighs findings, cost and duration together. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index e6276e7..33b63f7 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -15,19 +15,19 @@ One row per model and CLI, always its latest run. | Rank | Model | CLI | oddyssey | Confirmed / reported | Telemetry / Perf / Behavior | Total | Cost | $/confirmed | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| **#1** | `z-ai/glm-5.3` | opencode | 1.13.0 | 17 / 19 | 9 / 4 / 4 | 19m44s | $1.39 | $0.082 | -| **#2** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | $0.010 | -| **#3** | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 11 / 13 | 4 / 4 / 3 | 17m09s | $0.33 | $0.030 | -| **#4** | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 7 / 8 | 3 / 2 / 2 | **6m29s** | **$0.11** | **$0.016** | -| **#5** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | -| **#6** | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 7 / 8 | 2 / 3 / 2 | 5m58s | $0.88 | $0.126 | -| **#7** | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 12 / 13 | 8 / 4 / 0 | 9m32s | $1.41 | $0.117 | +| **#1** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | **$0.010** | +| **#2** | `z-ai/glm-5.3` | opencode | 1.13.0 | 17 / 19 | 9 / 4 / 4 | 19m44s | $1.39 | $0.082 | +| **#3** | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 12 / 13 | 8 / 4 / 0 | 9m32s | $1.41 | $0.117 | +| **#4** | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 11 / 13 | 4 / 4 / 3 | 17m09s | $0.33 | $0.030 | +| **#5** | `anthropic/claude-opus-5` | claude | 1.13.0 | **17 / 17** | 8 / 6 / 3 | 19m31s | $6.15 | $0.362 | +| **#6** | `anthropic/claude-fable-5.1` | claude | 1.12.0 ⚠︎ | **17 / 17** | 10 / 5 / 2 | 17m08s | $7.55 | $0.444 | +| **#7** | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 15 / 16 | 8 / 4 / 3 | 30m25s | $1.50 | $0.100 | | **#8** | `google/gemini-3.8-flash` | opencode | 1.13.0 | **12 / 12** | 6 / 4 / 2 | 20m39s | $2.29 | $0.191 | -| **#9** | `anthropic/claude-opus-5` | claude | 1.13.0 | **17 / 17** | 8 / 6 / 3 | 19m31s | $6.15 | $0.362 | -| **#10** | `anthropic/claude-fable-5.1` | claude | 1.12.0 | **17 / 17** | 10 / 5 / 2 | 17m08s | $7.55 | $0.444 | -| **#11** | `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 7 / 8 | 3 / 4 / 0 | 32m15s | $0.10 | $0.014 | +| **#9** | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 7 / 8 | 3 / 2 / 2 | 6m29s | $0.11 | $0.016 | +| **#10** | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 7 / 8 | 2 / 3 / 2 | **5m58s** | $0.88 | $0.126 | +| **#11** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | | **#12** | `qwen/qwen3.8-27b` | opencode | 1.13.0 | **11 / 11** | 5 / 3 / 3 | 40m25s | $1.62 | $0.147 | -| **#13** | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 15 / 16 | 8 / 4 / 3 | 30m25s | $1.50 | $0.100 | +| **#13** | `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 7 / 8 | 3 / 4 / 0 | 32m15s | **$0.10** | $0.014 | | **#14** | `anthropic/claude-sonnet-5` | claude | 1.13.0 | **6 / 6** | 2 / 3 / 1 | 13m55s | $3.37 | $0.561 |
@@ -35,19 +35,19 @@ One row per model and CLI, always its latest run. | Model | CLI | oddyssey | Preflight | Drive | Observation | Turns | Median turn | Input | Output | Cache | Signals | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `z-ai/glm-5.3` | opencode | 1.13.0 | 3m17s | 2m02s | 14m25s | 44 | 9.4s | 4.4M | 121k | 4.0M | 4/4 | | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | 5m59s | 2m02s | 21m04s | 73 | 11.9s | 8.0M | 94k | 7.4M | 4/4 | +| `z-ai/glm-5.3` | opencode | 1.13.0 | 3m17s | 2m02s | 14m25s | 44 | 9.4s | 4.4M | 121k | 4.0M | 4/4 | +| `openai/gpt-5.6-sol` | copilot | 1.13.0 | 1m21s | 2m00s | 6m11s | 48 | 4.2s | 3.7M | 28k | 3.5M | 4/4 | | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 3m10s | 2m01s | 11m58s | 32 | 13.3s | 2.3M | 68k | 2.1M | 4/4 | +| `anthropic/claude-opus-5` | claude | 1.13.0 | 2m47s | 2m02s | 14m42s | 53 | 6.6s | 5.8M | 62k | 5.8M | 4/4 | +| `anthropic/claude-fable-5.1` | claude | 1.12.0 ⚠︎ | 2m52s | 2m02s | 12m14s | 36 | 3.6s | 3.6M | 60k | 3.6M | 4/4 | +| `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 4m04s | 2m01s | 24m20s | 34 | 22.5s | 2.8M | 67k | 2.5M | 4/4 | +| `google/gemini-3.8-flash` | opencode | 1.13.0 | 9m58s | 2m03s | 8m38s | 148 | 4.4s | 12.8M | 59k | 11.1M | 4/4 | | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 1m02s | 2m01s | 3m26s | 38 | 2.8s | 2.9M | 17k | 2.9M | 4/4 | -| `google/gemini-3.7-flash` | opencode | 1.13.0 | 2m24s | 2m02s | 5m51s | 90 | 4.3s | 6.5M | 32k | 5.8M | 4/4 | | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 0m36s | 2m01s | 3m21s | 26 | 3.2s | 2.4M | 13k | 2.4M | 4/4 | -| `openai/gpt-5.6-sol` | copilot | 1.13.0 | 1m21s | 2m00s | 6m11s | 48 | 4.2s | 3.7M | 28k | 3.5M | 4/4 | -| `google/gemini-3.8-flash` | opencode | 1.13.0 | 9m58s | 2m03s | 8m38s | 148 | 4.4s | 12.8M | 59k | 11.1M | 4/4 | -| `anthropic/claude-opus-5` | claude | 1.13.0 | 2m47s | 2m02s | 14m42s | 53 | 6.6s | 5.8M | 62k | 5.8M | 4/4 | -| `anthropic/claude-fable-5.1` | claude | 1.12.0 | 2m52s | 2m02s | 12m14s | 36 | 3.6s | 3.6M | 60k | 3.6M | 4/4 | -| `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 6m42s | 2m03s | 23m30s | 34 | 22.3s | 2.3M | 74k | 1.8M | 4/4 | +| `google/gemini-3.7-flash` | opencode | 1.13.0 | 2m24s | 2m02s | 5m51s | 90 | 4.3s | 6.5M | 32k | 5.8M | 4/4 | | `qwen/qwen3.8-27b` | opencode | 1.13.0 | 3m49s | 2m00s | 34m36s | 48 | 16.6s | 5.9M | 133k | 3.7M | 4/4 | -| `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 4m04s | 2m01s | 24m20s | 34 | 22.5s | 2.8M | 67k | 2.5M | 4/4 | +| `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 6m42s | 2m03s | 23m30s | 34 | 22.3s | 2.3M | 74k | 1.8M | 4/4 | | `anthropic/claude-sonnet-5` | claude | 1.13.0 | 2m36s | 2m02s | 9m17s | 84 | 2.0s | 10.6M | 52k | 10.6M | 4/4 | Token counts are rounded; the exact figures are in each run's pull From 1ccb2a5c4fcad20fcc4f857100f88fd92c4119b8 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 08:42:06 +0200 Subject: [PATCH 15/18] docs(bench): the kind split counts confirmed findings, as every row does Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 33b63f7..1e0d859 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -60,7 +60,7 @@ design. - **Rank** weighs findings, cost and duration together. It is decided in each row's pull request, never computed: findings alone would rank a 67-minute run first, duration alone rewards whoever gives up soonest, cost alone rewards whoever barely looks. Adding a model re-sorts the whole table. - **Confirmed / reported** is the grade: how many of the findings the model reported held up when checked against the telemetry it cited and the code it accused. 3 / 3 beats 4 / 12. Anomalies and telemetry gaps both count; a restatement counts once; a row bundling several defects counts once per defect. -- **Telemetry / Perf / Behavior** splits the reported findings by kind. +- **Telemetry / Perf / Behavior** splits the confirmed findings by kind. - **$/confirmed** is what one trustworthy finding costs. - **CLI** is the coding-agent CLI the mission ran in; its version is in the row's pull request. Model and CLI identify a row; the oddyssey version does not, a new run replaces the row. - **Signals**: how many of metrics, traces, logs and profiles the run queried. Not part of the grade, the context to read it in. From 825af37bef8848525ec456a9f7f37885437765af Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 10:26:44 +0200 Subject: [PATCH 16/18] docs(bench): re-sort the llms-benchmark table with cost and duration weighing heavier The maintainer asked for a heavier negative weight on duration and cost: sol, flashx and luna lead; deepseek and glm-5.3 follow on their counts; opus-5, fable-5.1, glm-5.3-flash, sonnet-5 and qwen3.8-27b close the table on cost or wall clock. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 1e0d859..3f0bfa4 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -15,40 +15,40 @@ One row per model and CLI, always its latest run. | Rank | Model | CLI | oddyssey | Confirmed / reported | Telemetry / Perf / Behavior | Total | Cost | $/confirmed | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| **#1** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | **$0.010** | -| **#2** | `z-ai/glm-5.3` | opencode | 1.13.0 | 17 / 19 | 9 / 4 / 4 | 19m44s | $1.39 | $0.082 | -| **#3** | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 12 / 13 | 8 / 4 / 0 | 9m32s | $1.41 | $0.117 | -| **#4** | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 11 / 13 | 4 / 4 / 3 | 17m09s | $0.33 | $0.030 | -| **#5** | `anthropic/claude-opus-5` | claude | 1.13.0 | **17 / 17** | 8 / 6 / 3 | 19m31s | $6.15 | $0.362 | -| **#6** | `anthropic/claude-fable-5.1` | claude | 1.12.0 ⚠︎ | **17 / 17** | 10 / 5 / 2 | 17m08s | $7.55 | $0.444 | -| **#7** | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 15 / 16 | 8 / 4 / 3 | 30m25s | $1.50 | $0.100 | +| **#1** | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 12 / 13 | 8 / 4 / 0 | 9m32s | $1.41 | $0.117 | +| **#2** | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 11 / 13 | 4 / 4 / 3 | 17m09s | $0.33 | $0.030 | +| **#3** | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 7 / 8 | 3 / 2 / 2 | 6m29s | $0.11 | $0.016 | +| **#4** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | **$0.010** | +| **#5** | `z-ai/glm-5.3` | opencode | 1.13.0 | 17 / 19 | 9 / 4 / 4 | 19m44s | $1.39 | $0.082 | +| **#6** | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 7 / 8 | 2 / 3 / 2 | **5m58s** | $0.88 | $0.126 | +| **#7** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | | **#8** | `google/gemini-3.8-flash` | opencode | 1.13.0 | **12 / 12** | 6 / 4 / 2 | 20m39s | $2.29 | $0.191 | -| **#9** | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 7 / 8 | 3 / 2 / 2 | 6m29s | $0.11 | $0.016 | -| **#10** | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 7 / 8 | 2 / 3 / 2 | **5m58s** | $0.88 | $0.126 | -| **#11** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | -| **#12** | `qwen/qwen3.8-27b` | opencode | 1.13.0 | **11 / 11** | 5 / 3 / 3 | 40m25s | $1.62 | $0.147 | -| **#13** | `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 7 / 8 | 3 / 4 / 0 | 32m15s | **$0.10** | $0.014 | -| **#14** | `anthropic/claude-sonnet-5` | claude | 1.13.0 | **6 / 6** | 2 / 3 / 1 | 13m55s | $3.37 | $0.561 | +| **#9** | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 15 / 16 | 8 / 4 / 3 | 30m25s | $1.50 | $0.100 | +| **#10** | `anthropic/claude-opus-5` | claude | 1.13.0 | **17 / 17** | 8 / 6 / 3 | 19m31s | $6.15 | $0.362 | +| **#11** | `anthropic/claude-fable-5.1` | claude | 1.12.0 ⚠︎ | **17 / 17** | 10 / 5 / 2 | 17m08s | $7.55 | $0.444 | +| **#12** | `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 7 / 8 | 3 / 4 / 0 | 32m15s | **$0.10** | $0.014 | +| **#13** | `anthropic/claude-sonnet-5` | claude | 1.13.0 | **6 / 6** | 2 / 3 / 1 | 13m55s | $3.37 | $0.561 | +| **#14** | `qwen/qwen3.8-27b` | opencode | 1.13.0 | **11 / 11** | 5 / 3 / 3 | 40m25s | $1.62 | $0.147 |
Run detail — phases, turns, tokens | Model | CLI | oddyssey | Preflight | Drive | Observation | Turns | Median turn | Input | Output | Cache | Signals | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | 5m59s | 2m02s | 21m04s | 73 | 11.9s | 8.0M | 94k | 7.4M | 4/4 | -| `z-ai/glm-5.3` | opencode | 1.13.0 | 3m17s | 2m02s | 14m25s | 44 | 9.4s | 4.4M | 121k | 4.0M | 4/4 | | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 1m21s | 2m00s | 6m11s | 48 | 4.2s | 3.7M | 28k | 3.5M | 4/4 | | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 3m10s | 2m01s | 11m58s | 32 | 13.3s | 2.3M | 68k | 2.1M | 4/4 | -| `anthropic/claude-opus-5` | claude | 1.13.0 | 2m47s | 2m02s | 14m42s | 53 | 6.6s | 5.8M | 62k | 5.8M | 4/4 | -| `anthropic/claude-fable-5.1` | claude | 1.12.0 ⚠︎ | 2m52s | 2m02s | 12m14s | 36 | 3.6s | 3.6M | 60k | 3.6M | 4/4 | -| `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 4m04s | 2m01s | 24m20s | 34 | 22.5s | 2.8M | 67k | 2.5M | 4/4 | -| `google/gemini-3.8-flash` | opencode | 1.13.0 | 9m58s | 2m03s | 8m38s | 148 | 4.4s | 12.8M | 59k | 11.1M | 4/4 | | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 1m02s | 2m01s | 3m26s | 38 | 2.8s | 2.9M | 17k | 2.9M | 4/4 | +| `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | 5m59s | 2m02s | 21m04s | 73 | 11.9s | 8.0M | 94k | 7.4M | 4/4 | +| `z-ai/glm-5.3` | opencode | 1.13.0 | 3m17s | 2m02s | 14m25s | 44 | 9.4s | 4.4M | 121k | 4.0M | 4/4 | | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 0m36s | 2m01s | 3m21s | 26 | 3.2s | 2.4M | 13k | 2.4M | 4/4 | | `google/gemini-3.7-flash` | opencode | 1.13.0 | 2m24s | 2m02s | 5m51s | 90 | 4.3s | 6.5M | 32k | 5.8M | 4/4 | -| `qwen/qwen3.8-27b` | opencode | 1.13.0 | 3m49s | 2m00s | 34m36s | 48 | 16.6s | 5.9M | 133k | 3.7M | 4/4 | +| `google/gemini-3.8-flash` | opencode | 1.13.0 | 9m58s | 2m03s | 8m38s | 148 | 4.4s | 12.8M | 59k | 11.1M | 4/4 | +| `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 4m04s | 2m01s | 24m20s | 34 | 22.5s | 2.8M | 67k | 2.5M | 4/4 | +| `anthropic/claude-opus-5` | claude | 1.13.0 | 2m47s | 2m02s | 14m42s | 53 | 6.6s | 5.8M | 62k | 5.8M | 4/4 | +| `anthropic/claude-fable-5.1` | claude | 1.12.0 ⚠︎ | 2m52s | 2m02s | 12m14s | 36 | 3.6s | 3.6M | 60k | 3.6M | 4/4 | | `z-ai/glm-5.3-flash` | opencode | 1.13.0 | 6m42s | 2m03s | 23m30s | 34 | 22.3s | 2.3M | 74k | 1.8M | 4/4 | | `anthropic/claude-sonnet-5` | claude | 1.13.0 | 2m36s | 2m02s | 9m17s | 84 | 2.0s | 10.6M | 52k | 10.6M | 4/4 | +| `qwen/qwen3.8-27b` | opencode | 1.13.0 | 3m49s | 2m00s | 34m36s | 48 | 16.6s | 5.9M | 133k | 3.7M | 4/4 | Token counts are rounded; the exact figures are in each run's pull request. Input includes the cached share, so Input and Cache overlap by @@ -58,7 +58,7 @@ design. **How to read the table** -- **Rank** weighs findings, cost and duration together. It is decided in each row's pull request, never computed: findings alone would rank a 67-minute run first, duration alone rewards whoever gives up soonest, cost alone rewards whoever barely looks. Adding a model re-sorts the whole table. +- **Rank** weighs findings, cost and duration together, cost and duration the heavier since the 1.13.0 campaign. It is decided in each row's pull request, never computed: findings alone would rank a 67-minute run first, duration alone rewards whoever gives up soonest, cost alone rewards whoever barely looks. Adding a model re-sorts the whole table. - **Confirmed / reported** is the grade: how many of the findings the model reported held up when checked against the telemetry it cited and the code it accused. 3 / 3 beats 4 / 12. Anomalies and telemetry gaps both count; a restatement counts once; a row bundling several defects counts once per defect. - **Telemetry / Perf / Behavior** splits the confirmed findings by kind. - **$/confirmed** is what one trustworthy finding costs. From a0ea1a7706f261e0af4c012cd557f6ef1b579090 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 11:30:29 +0200 Subject: [PATCH 17/18] docs(bench): re-sort the llms-benchmark table - flashx, luna and terra lead, sol and deepseek move down on cost and wall clock Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .llms-benchmark/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.llms-benchmark/README.md b/.llms-benchmark/README.md index 3f0bfa4..b5a0b2f 100644 --- a/.llms-benchmark/README.md +++ b/.llms-benchmark/README.md @@ -15,13 +15,13 @@ One row per model and CLI, always its latest run. | Rank | Model | CLI | oddyssey | Confirmed / reported | Telemetry / Perf / Behavior | Total | Cost | $/confirmed | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| **#1** | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 12 / 13 | 8 / 4 / 0 | 9m32s | $1.41 | $0.117 | -| **#2** | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 11 / 13 | 4 / 4 / 3 | 17m09s | $0.33 | $0.030 | -| **#3** | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 7 / 8 | 3 / 2 / 2 | 6m29s | $0.11 | $0.016 | -| **#4** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | **$0.010** | -| **#5** | `z-ai/glm-5.3` | opencode | 1.13.0 | 17 / 19 | 9 / 4 / 4 | 19m44s | $1.39 | $0.082 | -| **#6** | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 7 / 8 | 2 / 3 / 2 | **5m58s** | $0.88 | $0.126 | -| **#7** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | +| **#1** | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 11 / 13 | 4 / 4 / 3 | 17m09s | $0.33 | $0.030 | +| **#2** | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 7 / 8 | 3 / 2 / 2 | 6m29s | $0.11 | $0.016 | +| **#3** | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 7 / 8 | 2 / 3 / 2 | **5m58s** | $0.88 | $0.126 | +| **#4** | `openai/gpt-5.6-sol` | copilot | 1.13.0 | 12 / 13 | 8 / 4 / 0 | 9m32s | $1.41 | $0.117 | +| **#5** | `google/gemini-3.7-flash` | opencode | 1.13.0 | 8 / 9 | 4 / 3 / 1 | 10m17s | $1.08 | $0.135 | +| **#6** | `z-ai/glm-5.3` | opencode | 1.13.0 | 17 / 19 | 9 / 4 / 4 | 19m44s | $1.39 | $0.082 | +| **#7** | `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | **16 / 16** | 10 / 3 / 3 | 29m05s | $0.16 | **$0.010** | | **#8** | `google/gemini-3.8-flash` | opencode | 1.13.0 | **12 / 12** | 6 / 4 / 2 | 20m39s | $2.29 | $0.191 | | **#9** | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 15 / 16 | 8 / 4 / 3 | 30m25s | $1.50 | $0.100 | | **#10** | `anthropic/claude-opus-5` | claude | 1.13.0 | **17 / 17** | 8 / 6 / 3 | 19m31s | $6.15 | $0.362 | @@ -35,13 +35,13 @@ One row per model and CLI, always its latest run. | Model | CLI | oddyssey | Preflight | Drive | Observation | Turns | Median turn | Input | Output | Cache | Signals | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `openai/gpt-5.6-sol` | copilot | 1.13.0 | 1m21s | 2m00s | 6m11s | 48 | 4.2s | 3.7M | 28k | 3.5M | 4/4 | | `z-ai/glm-5.3-flashx` | opencode | 1.13.0 | 3m10s | 2m01s | 11m58s | 32 | 13.3s | 2.3M | 68k | 2.1M | 4/4 | | `openai/gpt-5.6-luna` | copilot | 1.13.0 | 1m02s | 2m01s | 3m26s | 38 | 2.8s | 2.9M | 17k | 2.9M | 4/4 | -| `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | 5m59s | 2m02s | 21m04s | 73 | 11.9s | 8.0M | 94k | 7.4M | 4/4 | -| `z-ai/glm-5.3` | opencode | 1.13.0 | 3m17s | 2m02s | 14m25s | 44 | 9.4s | 4.4M | 121k | 4.0M | 4/4 | | `openai/gpt-5.6-terra` | copilot | 1.13.0 | 0m36s | 2m01s | 3m21s | 26 | 3.2s | 2.4M | 13k | 2.4M | 4/4 | +| `openai/gpt-5.6-sol` | copilot | 1.13.0 | 1m21s | 2m00s | 6m11s | 48 | 4.2s | 3.7M | 28k | 3.5M | 4/4 | | `google/gemini-3.7-flash` | opencode | 1.13.0 | 2m24s | 2m02s | 5m51s | 90 | 4.3s | 6.5M | 32k | 5.8M | 4/4 | +| `z-ai/glm-5.3` | opencode | 1.13.0 | 3m17s | 2m02s | 14m25s | 44 | 9.4s | 4.4M | 121k | 4.0M | 4/4 | +| `deepseek/deepseek-v4.1-flash` | opencode | 1.13.0 | 5m59s | 2m02s | 21m04s | 73 | 11.9s | 8.0M | 94k | 7.4M | 4/4 | | `google/gemini-3.8-flash` | opencode | 1.13.0 | 9m58s | 2m03s | 8m38s | 148 | 4.4s | 12.8M | 59k | 11.1M | 4/4 | | `qwen/qwen3.8-max-0902` | opencode | 1.13.0 | 4m04s | 2m01s | 24m20s | 34 | 22.5s | 2.8M | 67k | 2.5M | 4/4 | | `anthropic/claude-opus-5` | claude | 1.13.0 | 2m47s | 2m02s | 14m42s | 53 | 6.6s | 5.8M | 62k | 5.8M | 4/4 | From 86f18f09b8168946b3d376dd39ed7b314f26c3e5 Mon Sep 17 00:00:00 2001 From: using-system Date: Sun, 20 Sep 2026 11:34:14 +0200 Subject: [PATCH 18/18] chore(harness): test-plugin-harnessing defaults to openai/gpt-5.6-luna on copilot The fastest and cheapest row of the 1.13.0 table becomes the instrument: --cli copilot and --model openai/gpt-5.6-luna are the defaults of measure_phase.py and run_samples.py, analyze_run.py falls back to copilot, and copilot syncs no user scope (its deploy lives in the clone) instead of being refused for a missing --scope; the skill says so. Refs #636 Co-Authored-By: Claude Opus 5 (1M context) --- .../skills/test-plugin-harnessing/SKILL.md | 13 ++++++------ .../scripts/analyze_run.py | 4 ++-- .../scripts/measure_phase.py | 10 +++++----- .../scripts/run_samples.py | 20 +++++++++++-------- .../test_run_samples.py | 8 ++++++++ 5 files changed, 34 insertions(+), 21 deletions(-) diff --git a/.claude/skills/test-plugin-harnessing/SKILL.md b/.claude/skills/test-plugin-harnessing/SKILL.md index c4f24d6..3b08e5a 100644 --- a/.claude/skills/test-plugin-harnessing/SKILL.md +++ b/.claude/skills/test-plugin-harnessing/SKILL.md @@ -47,9 +47,9 @@ the path of `/odd-observe`, `/odd-verify` or `/odd-status`. phase durations, its turn count and its **median turn**. Read it from `origin/main`, never from the working tree. - **A model whose median turn is small**, as the canonical - `vendor/name` id - default `deepseek/deepseek-v4.1-flash`; others - `google/gemini-3.7-flash`, `anthropic/claude-haiku-4.5`, - `openai/gpt-5.6-sol`: the scripts hand each CLI its own form. The published median is the instrument's + `vendor/name` id - default `openai/gpt-5.6-luna` on `copilot`; others + `deepseek/deepseek-v4.1-flash`, `google/gemini-3.7-flash`, + `anthropic/claude-haiku-4.5`: the scripts hand each CLI its own form. The published median is the instrument's precision: a model at 3 s per turn measures the harness, one at 20 s measures the provider. Prefer the fastest row in the table for that CLI, whatever its findings score — this is not a quality test. @@ -126,7 +126,7 @@ the path of `/odd-observe`, `/odd-verify` or `/odd-status`. taken another. Its whole surface, so `--help` has nothing to add: `--cli` (default - `opencode`), `--model` (default `deepseek/deepseek-v4.1-flash`), + `copilot`), `--model` (default `openai/gpt-5.6-luna`), `--tag`, `--phase`, one of `--prompt` / `--prompt-file`, `--out`, plus `--end-pattern` (a regular expression over the run's own lines — opencode's log, claude's transcripts, @@ -171,8 +171,9 @@ the path of `/odd-observe`, `/odd-verify` or `/odd-status`. rewritten `opencode.json` — a lab dirty in any other way is refused before the launch), the fake user scope is synced from the branch's deploy and checked identical (`--scope :`, - repeatable; opencode's two pairs are the default, the other CLIs - state theirs), `--scratch ` is cleared when given (the CLI's + repeatable; copilot syncs nothing - its deploy lives in the clone - + opencode's two pairs are known to the script, claude states its + own), `--scratch ` is cleared when given (the CLI's scratch directory; nothing outside the study is touched otherwise), the measurement above is launched (once more when it exits non-zero within 30 s — a launch that died measured nothing; `SAMPLE diff --git a/.claude/skills/test-plugin-harnessing/scripts/analyze_run.py b/.claude/skills/test-plugin-harnessing/scripts/analyze_run.py index 551bc2f..6d4fc3a 100755 --- a/.claude/skills/test-plugin-harnessing/scripts/analyze_run.py +++ b/.claude/skills/test-plugin-harnessing/scripts/analyze_run.py @@ -359,11 +359,11 @@ def main() -> int: cli, run_id, stdout, usage = args.cli, args.run_id, args.stdout, args.usage if args.record: record = json.loads(Path(args.record).read_text()) - cli = cli or record.get("cli") or "opencode" + cli = cli or record.get("cli") or "copilot" run_id = run_id or record.get("run_id") stdout = stdout or record.get("stdout") usage = usage or record.get("usage") - cli = cli or "opencode" + cli = cli or "copilot" if not run_id: print("no run id - pass --run-id or --record", file=sys.stderr) return 1 diff --git a/.claude/skills/test-plugin-harnessing/scripts/measure_phase.py b/.claude/skills/test-plugin-harnessing/scripts/measure_phase.py index 7450d8a..81f49b0 100755 --- a/.claude/skills/test-plugin-harnessing/scripts/measure_phase.py +++ b/.claude/skills/test-plugin-harnessing/scripts/measure_phase.py @@ -18,7 +18,7 @@ - a run that dies looks exactly like a run that thinks, so the watch fails loudly instead of returning a fast, wrong time. - measure_phase.py --cli opencode --model deepseek/deepseek-v4.1-flash --tag d1 \ + measure_phase.py --cli copilot --model openai/gpt-5.6-luna --tag l1 \ --phase preflight --prompt-file mission.txt --out /tmp/study The model is the canonical `vendor/name` id whatever the CLI; each CLI is @@ -305,13 +305,13 @@ def main() -> int: ap.add_argument( "--cli", choices=CLIS, - default="opencode", - help="the CLI the run is driven by (default opencode)", + default="copilot", + help="the CLI the run is driven by (default copilot)", ) ap.add_argument( "--model", - default="deepseek/deepseek-v4.1-flash", - help="the canonical vendor/name model id (default deepseek/deepseek-v4.1-flash)", + default="openai/gpt-5.6-luna", + help="the canonical vendor/name model id (default openai/gpt-5.6-luna)", ) ap.add_argument("--tag", required=True, help="names this measurement's files") ap.add_argument( diff --git a/.claude/skills/test-plugin-harnessing/scripts/run_samples.py b/.claude/skills/test-plugin-harnessing/scripts/run_samples.py index 1711ef7..ff4b5a1 100755 --- a/.claude/skills/test-plugin-harnessing/scripts/run_samples.py +++ b/.claude/skills/test-plugin-harnessing/scripts/run_samples.py @@ -2,7 +2,7 @@ """Run a study's samples in order, each on its lab branch, and journal them. python3 run_samples.py --lab --fake-home --out \\ - --cli opencode --model deepseek/deepseek-v4.1-flash --phase whole \\ + --cli copilot --model openai/gpt-5.6-luna --phase whole \\ base1=lab-main:mission.txt after1=lab-after:mission.txt \\ after2=lab-after:mission.txt base2=lab-main:mission.txt @@ -47,10 +47,13 @@ HERE = Path(__file__).resolve().parent RELAUNCH_WITHIN = 30 # seconds: a run gone before this measured nothing -# what the opencode deploy in the lab writes, and where the fake user scope -# expects it (launch-llms-benchmark step 3 states the scopes per CLI); the -# other CLIs' scopes are passed as --scope : +# what a CLI's deploy in the lab writes, and where the fake user scope +# expects it (launch-llms-benchmark step 3 states the scopes per CLI): +# copilot reads the deploy in the clone and syncs nothing, opencode also +# reads the user scope; claude's scopes are passed as +# --scope : DEFAULT_SCOPES = { + "copilot": [], "opencode": [ (".agents/skills", ".claude/skills"), (".opencode/agents", ".claude/agents"), @@ -218,20 +221,20 @@ def main() -> int: ap.add_argument("--lab", required=True, help="the lab clone the runs launch in") ap.add_argument("--fake-home", required=True, help="the HOME the runs see") ap.add_argument("--out", required=True, help="the study directory") - ap.add_argument("--cli", default="opencode", choices=CLIS) + ap.add_argument("--cli", default="copilot", choices=CLIS) ap.add_argument( "--scope", action="append", metavar="LAB_PATH:HOME_PATH", help="a deployed directory to sync into the fake home, both paths relative " - "(repeatable; the opencode pairs are the default, the other CLIs need theirs stated)", + "(repeatable; copilot syncs nothing, opencode's pairs are known, claude states its own)", ) ap.add_argument( "--scratch", metavar="DIR", help="the CLI's scratch directory to clear before each sample (none by default)", ) - ap.add_argument("--model", default="deepseek/deepseek-v4.1-flash") + ap.add_argument("--model", default="openai/gpt-5.6-luna") ap.add_argument( "--phase", default="whole", @@ -275,6 +278,7 @@ def main() -> int: raise SystemExit(f"no such mission file: {mission}") check_pair_order(samples) scopes = list(DEFAULT_SCOPES.get(args.cli, [])) + known = args.cli in DEFAULT_SCOPES for pair in args.scope or []: if ":" not in pair: raise SystemExit(f"--scope takes :, got {pair!r}") @@ -286,7 +290,7 @@ def main() -> int: f"never absolute, never through ..: got {pair!r}" ) scopes.append((src, dst)) - if not scopes: + if not scopes and not known: raise SystemExit(f"--cli {args.cli} needs its --scope pairs stated") branches = git(lab, "branch", "--format=%(refname:short)").splitlines() tips: dict[str, str] = {} diff --git a/tests/skills/test-plugin-harnessing/test_run_samples.py b/tests/skills/test-plugin-harnessing/test_run_samples.py index 226065e..c1d866f 100644 --- a/tests/skills/test-plugin-harnessing/test_run_samples.py +++ b/tests/skills/test-plugin-harnessing/test_run_samples.py @@ -315,6 +315,14 @@ def test_a_missing_scope_source_is_refused_and_scratch_is_cleared_only_when_name lab, kit, out, f"s4=lab-main:{kit['mission']}", extra=("--cli", "claude") ) assert p.returncode == 1 and "--scope" in p.stderr + # copilot reads the deploy in the clone: no user-scope pair to sync, nothing refused + p = run_samples( + lab, kit, out, f"s5=lab-main:{kit['mission']}", extra=("--cli", "copilot") + ) + assert p.returncode == 0, p.stderr + p.stdout + assert "--cli copilot" in " ".join( + json.loads((out / "s5.record.json").read_text())["argv"] + ) def test_a_sample_spec_needs_all_three_parts():