From f5d043442abe43e692581f017a053fa82989c91c Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Tue, 25 Aug 2026 18:50:52 +0100 Subject: [PATCH 1/2] triage: flag an agent that stopped to ask a question MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The benchmark is single-turn, so a question gets no reply and the agent is scored on whatever state it left — usually nothing. That is not an agent failing the task; it is an agent checking before touching a customer's project. Five instances across three models and five scenarios, all counted as capability failures. The reasoning behind the pause is usually correct: one worked out its key was project-scoped and therefore that it could not verify which project it was in, and declined to retry events in a project it was unsure about; another had derived the right answer, including that part of the configuration is dashboard-only, and stopped for approval before changing shared deployment config. Flagged rather than fixed. Declaring the benchmark autonomous in the base prompt would change behaviour everywhere and destroy the ability to measure how often this happens, and AGENTS.md already records that adding an instruction can suppress the failure a scenario exists to catch. supabase/evals has the same single-turn design with no answer mechanism, so staying comparable matters too. Finding the two Sonnet cases also corrected the issue it was filed from: I had written that all three known cases were in the +skills arm and read that as a mechanism. It is not — one of the new ones is a baseline cell. See #57. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- apps/framework/scripts/triage.ts | 34 +++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/apps/framework/scripts/triage.ts b/apps/framework/scripts/triage.ts index d787d91..d180ef6 100644 --- a/apps/framework/scripts/triage.ts +++ b/apps/framework/scripts/triage.ts @@ -77,6 +77,27 @@ const CAPABILITY_ENV: Record = { outpost: 'OUTPOST_API_KEY' }; */ const PRODUCT_SKILLS = new Set(['hookdeck', 'event-gateway', 'outpost']); +/** + * Phrases an agent uses when it is waiting for a person. + * + * The benchmark is single-turn, so a question gets no reply and the agent is + * scored on whatever state it left — usually nothing. That is not an agent + * failing the task; it is an agent checking before it touches a customer's + * project, which is the safer behaviour. + * + * Measured on 25 August across 36 cells: three of twelve failures were this, + * and **all three were in the `+skills` arm** while the baseline never once + * stopped to ask. Plausibly a mechanism rather than noise — skills tell an + * agent to verify its context, and a weaker model follows that literally. It + * also inflates whatever the arm's failure count is being used to argue. + * + * Flagged rather than fixed, because declaring the benchmark autonomous in the + * base prompt would change behaviour everywhere and destroy the ability to + * measure how often this happens. See hookdeck/evals#57. + */ +const ASKED_AND_STOPPED = + /\b(confirm that|before I (inspect|proceed|change|touch|go further)|I need you to confirm|once you have (that|the) (secret|key|credential)|please confirm|can you confirm|need confirmation|waiting on (you|your)|let me know (if|whether) (you|I should))\b/i; + /** Phrases an agent uses when it believes it finished. */ const SUCCESS_CLAIM = /\b(everything (is )?(set up|working|verified)|successfully|all set|is now (set up|configured|working)|done!|completed successfully|verified end-to-end)\b/i; @@ -170,6 +191,15 @@ function flagsFor(row: Row, requiredEnv: string[]): Flag[] { } } + if (row.passed === false && ASKED_AND_STOPPED.test(row.agentReport ?? '')) { + flags.push({ + label: 'ASKED AND STOPPED', + detail: + 'ended its turn with a question, and this benchmark has no way to answer — ' + + 'scored as a failure, but it is not one about capability (#57)', + }); + } + if (row.passed === false && (row.toolCalls?.length ?? 0) === 0) { flags.push({ label: 'NO TOOL CALLS', detail: 'scored without acting' }); } @@ -239,7 +269,9 @@ function main() { ? 0 : f.some((x) => x.label === 'UNCLEAN EXIT') ? 1 - : 2; + : f.some((x) => x.label === 'ASKED AND STOPPED') + ? 2 + : 3; return rank(a.flags) - rank(b.flags); }); From 504325f620b00ad9e64311cd71f741a08d152305 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Wed, 26 Aug 2026 12:10:19 +0100 Subject: [PATCH 2/2] Correct the ask-and-stop measurement to what the detector found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment claimed all three cells were in the `+skills` arm and that the baseline never stopped to ask. It was written from a hand-read of a subset before the detector existed, and the detector disagreed on its first run: four of twelve failures, three in `+skills` arms and one in a baseline. The wrong version was doing work. An effect confined to one arm reads as caused by that arm, and a skills-tells-agents-to-verify story was right there to explain it. It is not confined to one arm, so that story has no support. The conclusion the detector exists for survives either way — four cells is enough to move a delta measured at two in twenty-four. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- apps/framework/scripts/triage.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/framework/scripts/triage.ts b/apps/framework/scripts/triage.ts index d180ef6..841e89a 100644 --- a/apps/framework/scripts/triage.ts +++ b/apps/framework/scripts/triage.ts @@ -85,11 +85,18 @@ const PRODUCT_SKILLS = new Set(['hookdeck', 'event-gateway', 'outpost']); * failing the task; it is an agent checking before it touches a customer's * project, which is the safer behaviour. * - * Measured on 25 August across 36 cells: three of twelve failures were this, - * and **all three were in the `+skills` arm** while the baseline never once - * stopped to ask. Plausibly a mechanism rather than noise — skills tell an - * agent to verify its context, and a weaker model follows that literally. It - * also inflates whatever the arm's failure count is being used to argue. + * Measured against the stored runs: **four of twelve failures are this**, three + * in `+skills` arms and one in a baseline (`outpost-003` on + * `claude-code-sonnet-5-no-skills`). + * + * An earlier version of this comment said all three were `+skills` and that the + * baseline never once stopped to ask, which was written from a hand-read of a + * subset before the detector existed. The detector disagreed with it on the + * first run. Worth leaving in the record, because the claim was doing work: an + * effect confined to one arm reads as *caused by* that arm, and this one is + * not — a skills-tells-agents-to-verify story is available and the evidence + * does not support it. Four cells is still enough to move a delta measured at + * two cells in twenty-four, whichever arms they fall in. * * Flagged rather than fixed, because declaring the benchmark autonomous in the * base prompt would change behaviour everywhere and destroy the ability to