diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 392920d..cff84ad 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "skillhook", "displayName": "skillhook", - "version": "0.10.0", + "version": "0.10.1", "description": "Turn this machine into a permanent, secure webhook endpoint that runs Agent Skills with Claude Code or Codex, and run your whole Skillhook Cloud organisation from the agent. Three skills teach the agent to install and expose skillhook, to write good webhook skills and to operate the fleet; two MCP servers manage this machine (skills, secrets, jobs, exposure) and, with an organisation API key, every machine as the Skillhook Cloud dashboard does (what needs a person, failures, stats, answers, replays, runs, hosted URLs).", "author": { "name": "Meter", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 2046cc6..21cf425 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "skillhook", - "version": "0.10.0", + "version": "0.10.1", "description": "Turn this machine into a permanent, secure webhook endpoint that runs Agent Skills with Claude Code or Codex, and run your whole Skillhook Cloud organisation from the agent. Three skills teach the agent to install and expose skillhook, to write good webhook skills and to operate the fleet; two MCP servers manage this machine (skills, secrets, jobs, exposure) and, with an organisation API key, every machine as the Skillhook Cloud dashboard does (what needs a person, failures, stats, answers, replays, runs, hosted URLs).", "author": { "name": "Meter", diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index 0740ef3..1db8d9a 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "skillhook", - "version": "0.10.0", + "version": "0.10.1", "description": "Turn this machine into a permanent, secure webhook endpoint that runs Agent Skills with Claude Code or Codex, and run your whole Skillhook Cloud organisation from the agent. Three skills teach the agent to install and expose skillhook, to write good webhook skills and to operate the fleet; two MCP servers manage this machine (skills, secrets, jobs, exposure) and, with an organisation API key, every machine as the Skillhook Cloud dashboard does (what needs a person, failures, stats, answers, replays, runs, hosted URLs).", "author": { "name": "Meter", diff --git a/CHANGELOG.md b/CHANGELOG.md index a97b50f..3768f80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to skillhook, newest first. The format follows [Keep a Chang ## Unreleased +## 0.10.1 (2026-10-13) + +- Fallback targets accept a runner, model and effort. Recognise Claude session/weekly limits and exhausted credits, ignore synthetic API error notices when checking whether fallback is safe, and never revisit a failed runner in the fallback chain. + ## 0.10.0 (2026-10-09) - `skillhook cloud login` signs in with the browser: it shows a code, opens the cloud's sign-in page, and the person diff --git a/docs/operations.md b/docs/operations.md index 48c514d..c69f42d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -414,3 +414,5 @@ The agent asked a question (`skillhook jobs show ` prints it) or finished wi ### The MCP server sees a different home than the CLI The MCP process reads `SKILLHOOK_HOME` or `--dir` from its own configuration. `skillhook mcp --print-config` adds `--dir` when either is set; make the MCP client and your shell agree. + +Fallback runners also accept targets such as `{ runner: codex, model: gpt-6.1-sol, effort: medium }` (JSON uses quoted keys and values). A target overrides the original model and effort when selected; plain runner names keep the original settings. diff --git a/docs/runners.md b/docs/runners.md index 3bf4cbe..8ca1b2e 100644 --- a/docs/runners.md +++ b/docs/runners.md @@ -180,7 +180,7 @@ Every job that ends `failed` or `timed_out` carries `failure: {kind, code?, retr ```yaml skillhook: fallback: - runners: [codex] # in order of preference; shell only for a skill with shell.command + runners: [{runner: codex, model: gpt-6.1-sol, effort: medium}] # in order of preference; shell only for a skill with shell.command on: [not_ready] # default; add auth, usage_limit, rate_limit, crash to re-run a failed job on the next runner retry: attempts: 1 # more runs on the same runner (1–3) @@ -190,6 +190,8 @@ skillhook: `fallback` names other runners for when this one cannot run. With the default `on: [not_ready]` it acts only before the run: the readiness check fails, the first ready runner of the list takes over, and the job records `runner` (the one that ran), `runner_requested` (the one asked for) and `runner_reason` (`fallback: claude not logged in`). `defaults.fallback` in `skillhook.json` applies to every skill that sets none. +Retries on the current runner happen first; once exhausted, fallbacks are considered in list order. Each runner is visited at most once per job (same-runner retries are still allowed). Object targets set their own `model` and `effort`; plain names retain the original settings. Claude synthetic API error messages do not count as agent work, but real assistant messages and tool activity prevent fallback. + The other triggers (`auth`, `usage_limit`, `rate_limit`, `crash`) and `retry` act after a run failed that way, **only when the agent had not produced anything yet** (no assistant message): the failed run becomes an entry of `attempts` (`{runner, started_at, finished_at, status, error, failure}`), the next run starts on the same runner (`retry`, after `backoff_seconds`) or on the next ready fallback runner, and the job record is the last attempt. A run that had already started acting is never repeated, because a repeat could redo its side effects; both features are off by default and belong on idempotent skills only. Each attempt gets the full `timeout_seconds`; `stdout.log` / `stderr.log` keep every attempt, separated by `--- attempt N () ---`. ## Environment diff --git a/docs/skills.md b/docs/skills.md index 754ef7c..fcaa209 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -554,3 +554,5 @@ skillhook: Not used by the shell runner; document what handle.py does here. ``` + +Fallback runners also accept targets such as `{ runner: codex, model: gpt-6.1-sol, effort: medium }` (JSON uses quoted keys and values). A target overrides the original model and effort when selected; plain runner names keep the original settings. diff --git a/package-lock.json b/package-lock.json index a1b3c4b..5269415 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@meterapp/skillhook", - "version": "0.10.0", + "version": "0.10.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@meterapp/skillhook", - "version": "0.10.0", + "version": "0.10.1", "license": "MIT", "dependencies": { "@modelcontextprotocol/server": "^2.2.0", diff --git a/package.json b/package.json index 15160d7..930f445 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@meterapp/skillhook", - "version": "0.10.0", + "version": "0.10.1", "description": "Make your Agent Skills reactive. A permanent, secure webhook endpoint on your own machine that runs SKILL.md files with Claude Code or Codex the moment something happens, plus version-controlled schedules for the work that has no trigger. Webhook in, agent out.", "license": "MIT", "author": "Meter (https://meterapp.co)", diff --git a/schema/skillhook.schema.json b/schema/skillhook.schema.json index 5a905ff..9fbf2ba 100644 --- a/schema/skillhook.schema.json +++ b/schema/skillhook.schema.json @@ -61,11 +61,38 @@ "minItems": 1, "type": "array", "items": { - "type": "string", - "enum": [ - "claude", - "codex", - "shell" + "anyOf": [ + { + "type": "string", + "enum": [ + "claude", + "codex", + "shell" + ] + }, + { + "type": "object", + "properties": { + "runner": { + "type": "string", + "enum": [ + "claude", + "codex", + "shell" + ] + }, + "model": { + "type": "string" + }, + "effort": { + "type": "string" + } + }, + "required": [ + "runner" + ], + "additionalProperties": false + } ] } }, diff --git a/schema/skillhook.yaml.schema.json b/schema/skillhook.yaml.schema.json index 7a2a1d1..16327af 100644 --- a/schema/skillhook.yaml.schema.json +++ b/schema/skillhook.yaml.schema.json @@ -551,11 +551,38 @@ "minItems": 1, "type": "array", "items": { - "type": "string", - "enum": [ - "claude", - "codex", - "shell" + "anyOf": [ + { + "type": "string", + "enum": [ + "claude", + "codex", + "shell" + ] + }, + { + "type": "object", + "properties": { + "runner": { + "type": "string", + "enum": [ + "claude", + "codex", + "shell" + ] + }, + "model": { + "type": "string" + }, + "effort": { + "type": "string" + } + }, + "required": [ + "runner" + ], + "additionalProperties": false + } ] } }, diff --git a/skills/skillhook-authoring/SKILL.md b/skills/skillhook-authoring/SKILL.md index 692534e..b56df38 100644 --- a/skills/skillhook-authoring/SKILL.md +++ b/skills/skillhook-authoring/SKILL.md @@ -181,3 +181,5 @@ To iterate on a draft before installing it, or to try a change without touching 6. Configure the sender (`skillhook url ` plus the secret), trigger one real event, and watch `skillhook jobs list`. Read `result.md` of the first few jobs and tighten the body wherever the agent guessed. Every delivery the server received, including the ones it rejected or skipped, is in `skillhook deliveries list`, and `skillhook deliveries replay ` (MCP: `replay_delivery`) runs it again through the skill as it is now, so a real event is the test payload of the next iteration. On a machine paired with Skillhook Cloud the same history and replay cover every machine, from the dashboard, its API, the `skillhook-cloud` MCP server (`replay_delivery`) or `skillhook cloud replay_delivery `, on the machine it arrived at, on another machine or through another skill: https://skillhook.dev/docs/deliveries. Keep SKILL.md under about 150 lines; move API shapes, field lists and long procedures into `references/*.md` and tell the agent when to read them. + +Fallback runners also accept targets such as `{ runner: codex, model: gpt-6.1-sol, effort: medium }` (JSON uses quoted keys and values). A target overrides the original model and effort when selected; plain runner names keep the original settings. diff --git a/src/queue.ts b/src/queue.ts index 90ae2d9..79f9a8c 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -10,7 +10,7 @@ import { answerQuestion, readTimeline, type JobAnswer, type JobQuestion, type Pr import { deriveOutcome, resolveJobResponse } from "./response.js"; import type { ReadinessCache, RunnerReadiness } from "./readiness.js"; import { prepareRun } from "./run.js"; -import { classifyFailure, type FailureKind, type FallbackTrigger, type JobFailure } from "./runners/failure.js"; +import { classifyFailure, type FailureKind, type FallbackConfig, type FallbackTrigger, type JobFailure } from "./runners/failure.js"; import type { RunnerOutcome, StreamState } from "./runners/index.js"; import { lastLines } from "./runners/types.js"; import type { SkillRegistry } from "./registry.js"; @@ -46,7 +46,7 @@ interface AttemptResult { } /** The skill's `fallback:` (else `defaults.fallback`), with the default trigger. */ -function fallbackPolicy(skill: Skill, config: Config): { runners: RunnerName[]; on: FallbackTrigger[] } { +function fallbackPolicy(skill: Skill, config: Config): { runners: FallbackConfig["runners"]; on: FallbackTrigger[] } { const spec = skill.config.fallback ?? config.defaults.fallback; return { runners: spec?.runners ?? [], on: spec?.on ?? ["not_ready"] }; } @@ -300,13 +300,15 @@ export class JobQueue extends EventEmitter { } /** The first usable runner of `candidates` other than `exclude` (`shell` only for a skill that has a command). */ - private async firstReady(candidates: RunnerName[], exclude: RunnerName, skill: Skill): Promise { - for (const candidate of candidates) { - if (candidate === exclude) continue; + private async firstReady(candidates: FallbackConfig["runners"], exclude: Set, skill: Skill): Promise<{ runner: RunnerName; model?: string; effort?: string } | undefined> { + for (const entry of candidates) { + const target = typeof entry === "string" ? { runner: entry } : entry; + const candidate = target.runner; + if (exclude.has(candidate)) continue; if (candidate === "shell" && !skill.config.shell?.command) continue; try { const readiness = await this.deps.readiness?.get(candidate); - if (readiness?.ready) return readiness; + if (readiness?.ready) return target; } catch { /* a probe that fails is not a ready runner */ } @@ -338,6 +340,7 @@ export class JobQueue extends EventEmitter { // Pre-flight: a runner that is not installed or not logged in never spawns; a fallback takes over or the job fails fast. const policy = fallbackPolicy(skill, config); + const tried = new Set([running.job.runner]); if (this.deps.readiness && running.job.runner !== "shell") { let readiness: RunnerReadiness | undefined; try { @@ -346,14 +349,15 @@ export class JobQueue extends EventEmitter { logger.warn("readiness check failed; running anyway", { job: job.id, runner: running.job.runner, error: errorMessage(error) }); } if (readiness && !readiness.ready) { - const alternative = policy.on.includes("not_ready") ? await this.firstReady(policy.runners, running.job.runner, skill) : undefined; + const alternative = policy.on.includes("not_ready") ? await this.firstReady(policy.runners, tried, skill) : undefined; if (!alternative) { this.finish(running.job, { status: "failed", started_at: nowIso(), error: `${running.job.runner} is not ready: ${readiness.detail}${readiness.hint ? ` (${readiness.hint})` : ""}`, failure: { kind: readiness.found ? "auth" : "not_found", retryable: false, message: readiness.detail } }); return; } logger.warn("runner not ready; using the fallback", { job: job.id, skill: job.skill, runner: running.job.runner, fallback: alternative.runner, detail: readiness.detail }); - running.job = store.update(job.id, { runner: alternative.runner, runner_requested: running.job.runner, runner_reason: `fallback: ${running.job.runner} ${readiness.detail}` }); - this.events.emit("job.updated", { job: running.job, fields: ["runner", "runner_requested", "runner_reason"] }); + tried.add(alternative.runner); + running.job = store.update(job.id, { runner: alternative.runner, ...(alternative.model ? { model: alternative.model } : {}), ...(alternative.effort ? { effort: alternative.effort } : {}), runner_requested: running.job.runner, runner_reason: `fallback: ${running.job.runner} ${readiness.detail}` }); + this.events.emit("job.updated", { job: running.job, fields: ["runner", "model", "effort", "runner_requested", "runner_reason"] }); } } @@ -367,16 +371,19 @@ export class JobQueue extends EventEmitter { if (kind && (attempt.status === "failed" || attempt.status === "timed_out") && !attempt.producedOutput && !running.cancelled && !this.stopping) { if (kind === "auth") this.deps.readiness?.invalidate(running.job.runner); const sameRunner = retriesLeft > 0 && retryOn.includes(kind); - const alternative = !sameRunner && (policy.on as string[]).includes(kind) ? await this.firstReady(policy.runners, running.job.runner, skill) : undefined; + const alternative = !sameRunner && (policy.on as string[]).includes(kind) ? await this.firstReady(policy.runners, tried, skill) : undefined; if (sameRunner || alternative) { attempts.push({ runner: running.job.runner, started_at: attempt.startedAt, finished_at: nowIso(), status: attempt.status, ...(attempt.error ? { error: attempt.error } : {}), failure: attempt.failure }); const fields: (keyof JobRecord)[] = ["attempts"]; const patch: Partial = { attempts: [...attempts] }; if (alternative) { + tried.add(alternative.runner); patch.runner = alternative.runner; + if (alternative.model) patch.model = alternative.model; + if (alternative.effort) patch.effort = alternative.effort; patch.runner_requested = running.job.runner_requested ?? running.job.runner; patch.runner_reason = `fallback: ${running.job.runner} failed (${kind})`; - fields.push("runner", "runner_requested", "runner_reason"); + fields.push("runner", "model", "effort", "runner_requested", "runner_reason"); } running.job = store.update(job.id, patch); this.events.emit("job.updated", { job: running.job, fields }); @@ -567,7 +574,7 @@ export class JobQueue extends EventEmitter { status, failure, error, - producedOutput: Boolean(state.lastMessage), + producedOutput: Boolean(state.producedOutput || state.lastMessage), startedAt, patch: { ...(outcomeOverride ? { outcome: outcomeOverride } : {}), diff --git a/src/runners/claude.ts b/src/runners/claude.ts index 6ad21ec..41be5a6 100644 --- a/src/runners/claude.ts +++ b/src/runners/claude.ts @@ -70,7 +70,8 @@ export const claudeRunner: Runner = { return; } if (typeof event.session_id === "string" && !state.sessionId) state.sessionId = event.session_id; - if (event.type === "assistant") { + if (event.type === "assistant" && event.is_api_error_message !== true) { + state.producedOutput = true; const text = extractText(event.message); if (text) state.lastMessage = text; } diff --git a/src/runners/codex.ts b/src/runners/codex.ts index 870db4d..8cb7772 100644 --- a/src/runners/codex.ts +++ b/src/runners/codex.ts @@ -65,7 +65,11 @@ export const codexRunner: Runner = { case "thread.started": if (typeof event.thread_id === "string") state.sessionId = event.thread_id; break; + case "item.started": + state.producedOutput = true; + break; case "item.completed": { + state.producedOutput = true; const item = event.item as { type?: string; text?: string } | undefined; if (item?.type === "agent_message" && typeof item.text === "string") state.lastMessage = item.text; break; diff --git a/src/runners/failure.test.ts b/src/runners/failure.test.ts index 44ee0f2..6c2198b 100644 --- a/src/runners/failure.test.ts +++ b/src/runners/failure.test.ts @@ -21,6 +21,9 @@ describe("classifyFailure", () => { expect(classifyFailure({ status: "failed", error: "simulated failure", exitCode: 1, signal: null, resultEvent: { subtype: "error", is_error: true }, reported: true })).toEqual({ kind: "unknown", code: "error", retryable: false, message: "simulated failure" }); // The last stderr lines count when the error text says nothing. expect(classifyFailure({ status: "failed", error: "claude exited with code 1", exitCode: 1, signal: null, stderr: "Error: Invalid API key · Please run /login" })).toMatchObject({ kind: "auth" }); + for (const error of ["You've hit your session limit · resets 6pm (America/New_York)", "You've hit your weekly limit", "Credit balance is too low", "Out of credits"]) { + expect(classifyFailure({ status: "failed", error, exitCode: 1, signal: null, reported: true })).toMatchObject({ kind: "usage_limit", retryable: false }); + } expect(FAILURE_KINDS).toHaveLength(9); }); diff --git a/src/runners/failure.ts b/src/runners/failure.ts index 62d5992..0ef7d73 100644 --- a/src/runners/failure.ts +++ b/src/runners/failure.ts @@ -22,8 +22,8 @@ export type FallbackTrigger = "not_ready" | "auth" | "usage_limit" | "rate_limit export const FALLBACK_TRIGGERS: FallbackTrigger[] = ["not_ready", "auth", "usage_limit", "rate_limit", "crash"]; export const FallbackSchema = z .object({ - /** Runners to use instead, in order of preference; `shell` only for a skill that has a `shell.command`. */ - runners: z.array(z.enum(["claude", "codex", "shell"])).min(1), + /** Runners in order of preference, or targets with their own model and effort; `shell` requires `shell.command`. */ + runners: z.array(z.union([z.enum(["claude", "codex", "shell"]), z.object({ runner: z.enum(["claude", "codex", "shell"]), model: z.string().optional(), effort: z.string().optional() }).strict()])).min(1), /** When to fall back. Default `[not_ready]`: only before the run, when the runner is not installed or not logged in. The others re-run a failed job on the next runner and are safe only for idempotent skills. */ on: z.array(z.enum(FALLBACK_TRIGGERS as [FallbackTrigger, ...FallbackTrigger[]])).optional(), }) @@ -46,7 +46,7 @@ const RETRYABLE: FailureKind[] = ["rate_limit", "crash"]; const PATTERNS: { kind: FailureKind; needles: string[] }[] = [ { kind: "rate_limit", needles: ["rate limit", "rate_limit", "too many requests", "overloaded", " 429", "429 "] }, - { kind: "usage_limit", needles: ["usage limit", "usage_limit", "hit your limit", "out of extra usage", "quota", "insufficient_quota", "limit will reset"] }, + { kind: "usage_limit", needles: ["usage limit", "usage_limit", "hit your limit", "hit your session limit", "hit your weekly limit", "credit balance is too low", "out of credits", "out of extra usage", "quota", "insufficient_quota", "limit will reset"] }, { kind: "auth", needles: ["failed to authenticate", "not logged in", "please run /login", "claude login", "codex login", "oauth", "invalid api key", "invalid x-api-key", "authentication", "unauthorized", "unauthenticated", " 401", "401 ", "credentials", "login required", "please log in"] }, { kind: "budget", needles: ["max_budget", "max budget", "budget"] }, { kind: "max_turns", needles: ["max_turns", "max turns", "maximum turns", "maximum number of turns"] }, diff --git a/src/runners/types.ts b/src/runners/types.ts index 29682b6..37f551c 100644 --- a/src/runners/types.ts +++ b/src/runners/types.ts @@ -49,6 +49,8 @@ export interface RunnerInvocation { /** Facts extracted from the runner's live output stream. */ export interface StreamState { + /** Actual agent output or tool activity; excludes synthetic API error notices. */ + producedOutput?: boolean; sessionId?: string; lastMessage?: string; failed?: string; diff --git a/src/server.test.ts b/src/server.test.ts index d1c19d3..8aab28b 100644 --- a/src/server.test.ts +++ b/src/server.test.ts @@ -85,6 +85,7 @@ beforeAll(async () => { FAKE_CLAUDE_ASK: "Deploy A or B?", FAKE_CLAUDE_ASK_WAIT_MS: "4000", SKILLHOOK_SECRET_FLAKY: "fl", + SKILLHOOK_SECRET_SESSIONFALLBACK: "sf", SKILLHOOK_SECRET_RETRIER: "re", SKILLHOOK_SECRET_FALLBACKY: "fb", FAKE_CLAUDE_FAIL_KIND: "rate_limit", @@ -94,6 +95,7 @@ beforeAll(async () => { writeSkill(paths, "asker", "description: ask\nskillhook:\n timeout_seconds: 2\n human_wait_seconds: 20\n env: [FAKE_CLAUDE_ASK, FAKE_CLAUDE_ASK_WAIT_MS]"); writeSkill(paths, "askalone", "description: alone\nskillhook:\n timeout_seconds: 30\n env: [FAKE_CLAUDE_ASK, FAKE_CLAUDE_ASK_WAIT_MS]"); writeSkill(paths, "flaky", "description: fl\nskillhook:\n env: [FAKE_CLAUDE_FAIL_KIND]\n fallback:\n runners: [codex]\n on: [not_ready, rate_limit]"); + writeSkill(paths, "sessionfallback", "description: sf\nskillhook:\n model: opus\n effort: high\n env: [FAKE_CLAUDE_FAIL_KIND, FAKE_CODEX_FAIL_KIND]\n fallback:\n runners: [{runner: codex, model: gpt-6.1-sol, effort: medium}, claude]\n on: [not_ready, usage_limit] "); writeSkill(paths, "retrier", "description: re\nskillhook:\n env: [FAKE_CLAUDE_FAIL_KIND]\n retry:\n attempts: 1\n on: [rate_limit]\n backoff_seconds: 0"); writeSkill(paths, "fallbacky", "description: fb\nskillhook:\n fallback:\n runners: [codex]"); writeSkill(paths, "structured", "description: st\nskillhook:\n response:\n mode: structured\n env: [FAKE_CLAUDE_OUTCOME]"); @@ -773,6 +775,29 @@ describe("HTTP surface", () => { expect(first.data.changed.find((c) => c.name === "node")).toEqual({ name: "node", from: null, to: "ok" }); }); + it("falls back on a synthetic Claude session limit with target settings, but never after tool activity or in a cycle", async () => { + writeEnv(paths, { ...ENV_BASE, FAKE_CLAUDE_FAIL_KIND: "session_limit" }); + try { + const send = async (body = "{}") => json(await fetch(`${base}/hooks/sessionfallback?wait=20`, { method: "POST", body, headers: { authorization: "Bearer sf" } })); + const fell = await send(); + const job = store.get(String(fell.job_id))!; + expect(job).toMatchObject({ status: "succeeded", runner: "codex", model: "gpt-6.1-sol", effort: "medium", runner_requested: "claude" }); + expect(job.command).toContain("gpt-6.1-sol"); + expect(job.command).toContain('model_reasoning_effort="medium"'); + expect(job.attempts).toHaveLength(1); + expect(job.attempts?.[0]?.failure?.kind).toBe("usage_limit"); + const acted = await send('{"text":"already acting"}'); + expect(store.get(String(acted.job_id))).toMatchObject({ status: "failed", runner: "claude", failure: { kind: "usage_limit" } }); + expect(store.get(String(acted.job_id))?.attempts).toBeUndefined(); + writeEnv(paths, { ...ENV_BASE, FAKE_CLAUDE_FAIL_KIND: "session_limit", FAKE_CODEX_FAIL_KIND: "usage_limit" }); + const exhausted = await send(); + expect(store.get(String(exhausted.job_id))).toMatchObject({ status: "failed", runner: "codex", failure: { kind: "usage_limit" } }); + expect(store.get(String(exhausted.job_id))?.attempts).toHaveLength(1); + } finally { + writeEnv(paths, ENV_BASE); + } + }); + it("classifies failures, retries on the same runner and falls back to another after a failed run", async () => { const auth = { authorization: `Bearer ${ADMIN}` }; const retried = await json(await fetch(`${base}/hooks/retrier?wait=20`, { method: "POST", body: "{}", headers: { authorization: "Bearer re" } })); diff --git a/src/skills.test.ts b/src/skills.test.ts index 2522016..106e586 100644 --- a/src/skills.test.ts +++ b/src/skills.test.ts @@ -109,6 +109,7 @@ describe("fallback and retry options", () => { it("accepts runner lists and failure kinds and rejects anything else", () => { const doc = (block: string) => `---\nname: f\ndescription: f\nskillhook:\n${block}\n---\nBody\n`; expect(parseSkillDocument(doc(" fallback:\n runners: [codex, shell]\n on: [not_ready, rate_limit]"), "/tmp/f").config.fallback).toEqual({ runners: ["codex", "shell"], on: ["not_ready", "rate_limit"] }); + expect(parseSkillDocument(doc(" fallback:\n runners: [{runner: codex, model: gpt-6.1-sol, effort: medium}]"), "/tmp/f").config.fallback?.runners).toEqual([{ runner: "codex", model: "gpt-6.1-sol", effort: "medium" }]); expect(parseSkillDocument(doc(" retry:\n attempts: 2\n on: [crash]\n backoff_seconds: 5"), "/tmp/f").config.retry).toEqual({ attempts: 2, on: ["crash"], backoff_seconds: 5 }); expect(() => parseSkillDocument(doc(" fallback:\n runners: [gemini]"), "/tmp/f")).toThrow(/fallback/); expect(() => parseSkillDocument(doc(" fallback:\n runners: [codex]\n on: [timeout]"), "/tmp/f")).toThrow(/fallback/); diff --git a/test/fixtures/fake-claude.mjs b/test/fixtures/fake-claude.mjs index 05456a0..1ed811f 100644 --- a/test/fixtures/fake-claude.mjs +++ b/test/fixtures/fake-claude.mjs @@ -2,7 +2,7 @@ // Emulates `claude -p --output-format stream-json`: reads the prompt from stdin and prints // stream-json events. Controlled through env vars the test passes via env_passthrough: // FAKE_CLAUDE_FAIL= -> emit an is_error result and exit 1 -// FAKE_CLAUDE_FAIL_KIND= -> the failure the real CLI prints for it (also `auth` +// FAKE_CLAUDE_FAIL_KIND= -> the failure the real CLI prints for it (also `auth` // when CLAUDE_CONFIG_DIR/logged-out exists, like a logged-out install) // FAKE_CLAUDE_SLEEP_MS= -> delay before answering (timeout/cancel tests) // FAKE_CLAUDE_RECORD= -> write argv, prompt, env and cwd as JSON for assertions @@ -126,11 +126,16 @@ if (failKind) { // Captured shapes: the real CLI reports an auth failure as `subtype: success` with `is_error: true`. const canned = { auth: { subtype: "success", result: "Failed to authenticate: OAuth session expired and could not be refreshed" }, + session_limit: { subtype: "success", result: "You've hit your session limit · resets 6pm (America/New_York)" }, usage_limit: { subtype: "success", result: "You've hit your usage limit. Your limit will reset at 3pm (UTC)." }, rate_limit: { subtype: "error_during_execution", result: 'API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"This request would exceed your account\'s rate limit."}}' }, max_turns: { subtype: "error_max_turns", result: "Reached max turns (1)" }, budget: { subtype: "error_max_budget_usd", result: "Reached max budget ($0.01)" }, }[failKind] ?? { subtype: "success", result: `simulated ${failKind} failure` }; + if (failKind === "session_limit") { + if (prompt.includes("already acting")) out({ type: "assistant", message: { content: [{ type: "tool_use", name: "Bash", input: {} }] } }); + out({ type: "assistant", is_api_error_message: true, error: "rate_limit", message: { model: "", content: [{ type: "text", text: canned.result }] } }); + } out({ type: "result", is_error: true, ...canned, session_id: sessionId, total_cost_usd: 0, num_turns: 1, duration_ms: 5 }); process.exit(1); }