Let a scenario declare that one attempt is not enough for it

AGENTS.md

@@ -677,7 +677,17 @@ Claiming the instruction bought three scenarios would be the same mistake as the
 **Prefer more attempts to more instruction.** `--runs 3` stops at the first pass,
 so it costs about 1.11× and it removes a stopped run's power to decide a cell
 without touching the prompt at all. It is the cheapest correction available and
-should be reached for before anything is added to the base prompt. What cannot be
+should be reached for before anything is added to the base prompt.
+
+A scenario that knows its own outcome varies can say so, with `min_attempts` in
+its frontmatter. It is a floor on `--runs` for that scenario alone, capped at
+five, and with stop-on-pass it is paid for only on the cells that were already
+failing. `outpost-003` carries `min_attempts: 3`: across three full passes it
+was stable for both frontier models on all six of their observations and split
+2-1 for the weak model in both arms, because passing turns on which undocumented
+route the agent happens to guess. Use it where variance is measured rather than
+suspected, and remember what it buys — the published number becomes best-of-N,
+which the row's `attempts` field states rather than hides. What cannot be
 copied is upstream's other route: Vercel and Convex avoid clarifying questions by
 writing prompts as requirement lists, and this repo has measured that adding an
 instruction to build *suppresses the very failure a scenario exists to catch*.

apps/framework/harness/run-eval.ts

@@ -90,6 +90,19 @@ const SELECTED_EXPERIMENT_SUITE =
     ? EXPERIMENT_SUITE_FILTERS[0]
     : undefined;
 const RUNS = Number(readFlag('runs') ?? 1);
+
+/**
+ * How many attempts this scenario gets: `--runs`, raised to the scenario's own
+ * floor if it declares one.
+ *
+ * A floor and never a ceiling. A scenario that declares `min_attempts: 3`
+ * knows something the schedule does not — that its outcome varies at the
+ * bottom of the model range — and the schedule running `--runs 1` should not
+ * be able to publish a coin flip because of it. `--runs 5` still runs five.
+ */
+function runsFor(ev: EvalManifest): number {
+  return Math.max(RUNS, ev.metadata.minAttempts ?? 1);
+}
 const TIMEOUT_SEC = Number(readFlag('timeout-sec') ?? 720);
 const CONCURRENCY = Number(readFlag('concurrency') ?? 1);
 const STOP_ON_PASS = !args.has('--run-all-attempts');
@@ -448,7 +461,9 @@ async function runOne(
   let lastStoppedReason = 'not_started';
   let lastUsage: AgentRunResult['usage'];
 
-  for (let attempt = 1; attempt <= RUNS; attempt += 1) {
+  const runs = runsFor(ev);
+
+  for (let attempt = 1; attempt <= runs; attempt += 1) {
     // Tools mode: the eval's tool surface is MCP (platform-lite). A CLI agent
     // gets the same sandbox as local-stack minus the running stack — with its
     // skills installed — and reaches the in-container MCP servers' host-side
@@ -578,7 +593,7 @@ async function runOne(
 
   return {
     ...last,
-    attempts: RUNS,
+    attempts: runs,
     skills: buildSkillResult(availableSkills, lastToolCalls),
     docs: buildDocsResult(lastToolCalls),
     toolCalls: lastToolCalls,
@@ -621,10 +636,12 @@ function logRetryAttempt(
   attempt: number,
   result: ScoreResult
 ) {
-  if (!STOP_ON_PASS || result.passed || attempt >= RUNS) return;
+  const runs = runsFor(ev);
+  if (!STOP_ON_PASS || result.passed || attempt >= runs) return;
   const summary = formatRunSummary({ ...result, attempts: attempt });
+  const floor = runs > RUNS ? `, scenario floor ${runs}` : '';
   console.log(
-    `🔁 RETRY ${expName} x ${ev.id} (attempt ${attempt}/${RUNS} failed, ${summary})`
+    `🔁 RETRY ${expName} x ${ev.id} (attempt ${attempt}/${runs} failed${floor}, ${summary})`
   );
 }
 

evals/benchmark-outpost-003-operator-events/PROMPT.md

@@ -10,6 +10,7 @@ requires:
   - outpost
 extra_skills:
   - outpost
+min_attempts: 3
 motivation: Follows the incident in benchmark-outpost-002. A destination was auto-disabled, the customer's events were held, and nobody found out until the customer emailed. Outpost emits `alert.destination.disabled` for exactly this, but it is delivered only to a configured operator events destination, and this project has none. The routes that configure it are absent from the published OpenAPI spec and from the API reference, so this measures whether an agent can set up alerting it cannot read about.
 ---
 

packages/core/src/eval-metadata.ts

@@ -217,6 +217,29 @@ export type EvalMetadata = {
    * with `projectRunning: false`.
    */
   skipCliInstall?: boolean;
+  /**
+   * The fewest attempts this scenario may be run with, overriding a lower
+   * `--runs` for this scenario alone.
+   *
+   * For scenarios whose outcome is known to vary at the floor. `outpost-003`
+   * is the case that produced this: across three full passes it was stable for
+   * both frontier models on all six of their observations and split 2-1 for the
+   * weak model in both arms, because passing turns on which undocumented route
+   * the agent happens to guess (hookdeck/evals#34). A single attempt there
+   * publishes a coin flip against a named vendor.
+   *
+   * With stop-on-pass this costs nothing where the scenario passes first time,
+   * so the spend lands only on the cells that were already unreliable. It also
+   * makes the scenario's own variance the author's decision rather than a
+   * property of whichever `--runs` the schedule happens to use, and `attempts`
+   * on the published row records what was actually spent.
+   *
+   * A floor, never a ceiling: `--runs 3` against `min_attempts: 2` still runs
+   * three. Raising this does not make a flaky scenario sound — it makes the
+   * published number best-of-N, which is a different claim and one the row
+   * says out loud.
+   */
+  minAttempts?: number;
 };
 
 export type ParsedEvalMarkdown = {
@@ -242,6 +265,10 @@ export const evalMetadataSchema = z.object({
   skills: z.array(z.string().min(1)).optional(),
   extraSkills: z.array(z.string().min(1)).optional(),
   skipCliInstall: z.union([z.boolean(), z.stringbool()]).optional(),
+  // Capped rather than open-ended: this multiplies spend on the scenarios
+  // least likely to pass, and a typo of 30 in a file nobody re-reads would be
+  // discovered by the bill.
+  minAttempts: z.coerce.number().int().min(1).max(5).optional(),
 });
 
 // Collapse a YAML scalar into a comparable token: trim, lowercase, and fold
@@ -333,6 +360,7 @@ export const evalFrontmatterSchema = z.preprocess((raw) => {
       ? toIdentifierList((data.extraSkills ?? data.extra_skills) as unknown[])
       : undefined,
     skipCliInstall: data.skipCliInstall,
+    minAttempts: data.minAttempts ?? data.min_attempts,
   };
 }, evalMetadataSchema);
 

packages/sandbox/test/unit.test.ts

@@ -137,6 +137,45 @@ describe('cliVersion frontmatter', () => {
   });
 });
 
+describe('min_attempts frontmatter', () => {
+  const front = (extra: string[]) =>
+    [
+      '---',
+      'stage: build',
+      'suite: benchmark',
+      'product: outpost',
+      'topic: alerting',
+      ...extra,
+      '---',
+      'Set up alerting.',
+    ].join('\n');
+
+  it('reads the snake_case key the scenario files use', () => {
+    expect(
+      parseEvalMarkdown(front(['min_attempts: 3'])).metadata.minAttempts
+    ).toBe(3);
+  });
+
+  it('reads the camelCase key too, like every other key here', () => {
+    expect(
+      parseEvalMarkdown(front(['minAttempts: 2'])).metadata.minAttempts
+    ).toBe(2);
+  });
+
+  it('is absent when unset, so the scenario takes --runs as given', () => {
+    expect(parseEvalMarkdown(front([])).metadata.minAttempts).toBeUndefined();
+  });
+
+  it('rejects a floor above the cap, which would only be found on the bill', () => {
+    expect(() => parseEvalMarkdown(front(['min_attempts: 30']))).toThrow();
+  });
+
+  it('rejects zero and fractions rather than silently flooring them', () => {
+    expect(() => parseEvalMarkdown(front(['min_attempts: 0']))).toThrow();
+    expect(() => parseEvalMarkdown(front(['min_attempts: 1.5']))).toThrow();
+  });
+});
+
 describe('skills frontmatter', () => {
   const buildMarkdown = (extra: string) =>
     [