From f585cbb4f7306777b504b80dc7aeacfa42fffbbe Mon Sep 17 00:00:00 2001 From: vinlee19 <1401597760@qq.com> Date: Fri, 17 Jul 2026 22:50:24 +0800 Subject: [PATCH 1/3] fix(kosong): fail fast on quota-exhausted 429 instead of retrying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 429 caused by an exhausted account quota or insufficient balance (Moonshot error.type "exceeded_current_quota_error", OpenAI "insufficient_quota") can never succeed on retry, yet it was classified as APIProviderRateLimitError and silently retried for the whole budget (10 attempts, ~3 minutes of backoff) with no UI feedback — the session appeared frozen on every request. Introduce APIProviderQuotaExhaustedError, minted in normalizeAPIStatusError from the structured body error.type/error.code forwarded by convertOpenAIError, with billing-anchored message patterns as a fallback for gateways that flatten the body to text. The new class is excluded from isRetryableGenerateError (fail fast, even when a retry-after header is present) and from isProviderRateLimitError (no swarm requeue/suspend). toKimiErrorPayload and translateProviderError map it to provider.api_error (retryable: false) instead of provider.rate_limit, and classifyApiError reports it as quota_exhausted in telemetry. agent-core-v2 mirrors the same fix. Transient rate-limit 429s keep the existing retry, backoff, and Retry-After behavior (verified end-to-end against a mock provider: quota body fails after attempt 1/10; rate-limit body still walks the full 10-attempt ladder). Behavior changes to note: quota-failed swarm subagents now fail instead of suspending indefinitely as "Rate limited...", and quota errors cross the wire as provider.api_error rather than provider.rate_limit. --- .changeset/quota-exhausted-fail-fast.md | 6 ++ docs/en/configuration/config-files.md | 2 + docs/zh/configuration/config-files.md | 2 + .../src/app/llmProtocol/errors.ts | 62 +++++++++++++ .../llmProtocol/providers/openai-common.ts | 6 ++ .../agent-core-v2/src/app/protocol/errors.ts | 16 ++-- .../test/app/llmProtocol/errors.test.ts | 54 ++++++++++++ .../providers/provider-errors.test.ts | 35 +++++++- .../test/app/protocol/errors.test.ts | 17 ++++ packages/agent-core/src/agent/turn/index.ts | 6 ++ packages/agent-core/src/errors/serialize.ts | 17 ++-- .../agent-core/test/errors/serialize.test.ts | 20 ++++- packages/agent-core/test/loop/retry.test.ts | 38 ++++++++ packages/kosong/src/errors.ts | 86 ++++++++++++++++++ packages/kosong/src/index.ts | 2 + .../kosong/src/providers/openai-common.ts | 7 ++ packages/kosong/test/errors.test.ts | 88 +++++++++++++++++++ .../kosong/test/openai-common-errors.test.ts | 41 +++++++++ 18 files changed, 493 insertions(+), 12 deletions(-) create mode 100644 .changeset/quota-exhausted-fail-fast.md diff --git a/.changeset/quota-exhausted-fail-fast.md b/.changeset/quota-exhausted-fail-fast.md new file mode 100644 index 0000000000..b2405dc210 --- /dev/null +++ b/.changeset/quota-exhausted-fail-fast.md @@ -0,0 +1,6 @@ +--- +"@moonshot-ai/kimi-code": patch +"@moonshot-ai/kimi-code-sdk": patch +--- + +Fail fast on quota/balance-exhausted HTTP 429 errors (e.g. Moonshot `exceeded_current_quota_error`, OpenAI `insufficient_quota`) instead of silently retrying for ~3 minutes. Transient rate-limit 429s keep the existing retry, backoff, and Retry-After behavior. diff --git a/docs/en/configuration/config-files.md b/docs/en/configuration/config-files.md index 59504ec0b4..5195d49847 100644 --- a/docs/en/configuration/config-files.md +++ b/docs/en/configuration/config-files.md @@ -210,6 +210,8 @@ You can also switch models temporarily without touching the config file — by s | `max_retries_per_step` | `integer` | `10` | Maximum retries after a step failure | | `reserved_context_size` | `integer` | — | Number of tokens reserved for model output; automatic compaction is triggered when the remaining context window falls below this value | +Retries only apply to transient failures — connection errors, timeouts, HTTP 429 rate limits, and 5xx server errors. A 429 caused by an exhausted quota or insufficient account balance is not retried and fails immediately, since it cannot succeed until the account is recharged. + ## `background` `background` controls the concurrency behavior of background tasks (launched via the `Bash` tool or the `Agent` tool's `run_in_background=true` parameter). diff --git a/docs/zh/configuration/config-files.md b/docs/zh/configuration/config-files.md index 07ccfcf1be..f27e951608 100644 --- a/docs/zh/configuration/config-files.md +++ b/docs/zh/configuration/config-files.md @@ -210,6 +210,8 @@ display_name = "Kimi for Coding (custom)" | `max_retries_per_step` | `integer` | `10` | 单步失败后的最大重试次数 | | `reserved_context_size` | `integer` | — | 预留给模型输出的 token 数;上下文窗口剩余量低于此值时触发自动压缩 | +重试仅针对瞬时故障——连接错误、超时、HTTP 429 限流和 5xx 服务端错误。账户额度耗尽或余额不足导致的 429 不会重试,会立即失败:在充值之前重试不可能成功。 + ## `background` `background` 控制后台任务(通过 `Bash` 工具或 `Agent` 工具的 `run_in_background=true` 参数启动)的并发数。 diff --git a/packages/agent-core-v2/src/app/llmProtocol/errors.ts b/packages/agent-core-v2/src/app/llmProtocol/errors.ts index 7a4d5d70cd..5fd039aaa2 100644 --- a/packages/agent-core-v2/src/app/llmProtocol/errors.ts +++ b/packages/agent-core-v2/src/app/llmProtocol/errors.ts @@ -82,6 +82,21 @@ export class APIProviderRateLimitError extends APIStatusError { } } +// HTTP 429 caused by an exhausted account quota/balance — deterministic until +// the account is recharged, so unlike a rate limit it is neither retried nor +// requeued. Deliberately not a subclass of APIProviderRateLimitError. +export class APIProviderQuotaExhaustedError extends APIStatusError { + constructor( + message: string, + requestId?: string | null, + retryAfterMs?: number | null, + traceId?: string | null, + ) { + super(429, message, requestId, retryAfterMs, traceId); + this.name = 'APIProviderQuotaExhaustedError'; + } +} + export class APIProviderOverloadedError extends APIStatusError { constructor( statusCode: number, @@ -158,6 +173,11 @@ export function isRetryableGenerateError(error: unknown): boolean { return true; } if (error instanceof APIStatusError) { + // Quota/balance exhaustion is a 429 but deterministic until the account + // is recharged — retrying can never succeed. + if (error instanceof APIProviderQuotaExhaustedError) { + return false; + } return [408, 409, 429, 500, 502, 503, 504, 529].includes(error.statusCode); } return error instanceof ChatProviderError && !isImageFormatError(error); @@ -199,6 +219,24 @@ const PROVIDER_RATE_LIMIT_MESSAGE_PATTERNS = [ const PROVIDER_OVERLOAD_MESSAGE_PATTERNS = [/overload/] as const; +// Structured error `type`/`code` values that mean the account's quota or +// balance is exhausted: Moonshot `exceeded_current_quota_error` (body +// `error.type`), OpenAI `insufficient_quota` (both `type` and `code`). +const QUOTA_EXHAUSTED_ERROR_CODES = new Set(['exceeded_current_quota_error', 'insufficient_quota']); + +// Message fallback for gateways that flatten the body to text, matched against +// the lowercased message of a 429. Anchored to billing wording — deliberately +// no bare /quota/ or /balance/, which would also match transient throttle +// messages like "token quota per minute". +const QUOTA_EXHAUSTED_MESSAGE_PATTERNS = [ + /exceeded your current (?:token )?quota/, + /check your account balance/, + /insufficient balance/, + /recharge your account|please recharge/, + /account (?:is )?in arrears/, + /insufficient_quota/, +] as const; + const REQUEST_TOO_LARGE_MESSAGE_PATTERNS = [ /request exceeds the maximum size/, /request entity too large/, @@ -242,8 +280,12 @@ export function normalizeAPIStatusError( requestId?: string | null, retryAfterMs?: number | null, traceId?: string | null, + options?: { readonly errorCode?: string | null; readonly errorType?: string | null }, ): APIStatusError { if (statusCode === 429) { + if (isQuotaExhaustedStatusError(statusCode, message, options)) { + return new APIProviderQuotaExhaustedError(message, requestId, retryAfterMs, traceId); + } return new APIProviderRateLimitError(message, requestId, retryAfterMs, traceId); } if (isContextOverflowStatusError(statusCode, message)) { @@ -307,6 +349,23 @@ export function isRequestTooLargeStatusError(statusCode: number, message: string return REQUEST_TOO_LARGE_MESSAGE_PATTERNS.some((pattern) => pattern.test(lowerMessage)); } +// Whether a 429 means the account's quota/balance is exhausted rather than a +// transient rate limit. Structured `type`/`code` is authoritative when +// forwarded; message patterns only backstop text-flattening gateways. +export function isQuotaExhaustedStatusError( + statusCode: number, + message: string, + options?: { readonly errorCode?: string | null; readonly errorType?: string | null }, +): boolean { + if (statusCode !== 429) return false; + const errorCode = options?.errorCode; + if (typeof errorCode === 'string' && QUOTA_EXHAUSTED_ERROR_CODES.has(errorCode)) return true; + const errorType = options?.errorType; + if (typeof errorType === 'string' && QUOTA_EXHAUSTED_ERROR_CODES.has(errorType)) return true; + const lowerMessage = message.toLowerCase(); + return QUOTA_EXHAUSTED_MESSAGE_PATTERNS.some((pattern) => pattern.test(lowerMessage)); +} + const TOOL_EXCHANGE_ADJACENCY_MESSAGE_PATTERNS = [ /tool_use[\s\S]*tool_result/, /tool_result[\s\S]*tool_use/, @@ -345,6 +404,9 @@ export function isRecoverableRequestStructureError(error: unknown): boolean { } export function isProviderRateLimitError(error: unknown): boolean { + // Quota exhaustion is a 429 but not a rate limit: the rate-limit reactions + // (retry, requeue, suspend) cannot help until the account is recharged. + if (error instanceof APIProviderQuotaExhaustedError) return false; if (error instanceof APIProviderRateLimitError) return true; const statusCode = getStatusCode(error); diff --git a/packages/agent-core-v2/src/app/llmProtocol/providers/openai-common.ts b/packages/agent-core-v2/src/app/llmProtocol/providers/openai-common.ts index 9322a02a49..e3cf546c65 100644 --- a/packages/agent-core-v2/src/app/llmProtocol/providers/openai-common.ts +++ b/packages/agent-core-v2/src/app/llmProtocol/providers/openai-common.ts @@ -98,6 +98,12 @@ export function convertOpenAIError(error: unknown): ChatProviderError { reqId, parseRetryAfterMs(error.headers), parseTraceId(error.headers), + // Forward the SDK-parsed body `error.code`/`error.type` so a + // quota-exhausted 429 classifies structurally, not by wording. + { + errorCode: typeof error.code === 'string' ? error.code : null, + errorType: typeof error.type === 'string' ? error.type : null, + }, ); } if ( diff --git a/packages/agent-core-v2/src/app/protocol/errors.ts b/packages/agent-core-v2/src/app/protocol/errors.ts index 86a4186781..694dc00152 100644 --- a/packages/agent-core-v2/src/app/protocol/errors.ts +++ b/packages/agent-core-v2/src/app/protocol/errors.ts @@ -15,6 +15,7 @@ import { APIContextOverflowError, APIEmptyResponseError, APIProviderOverloadedError, + APIProviderQuotaExhaustedError, APIStatusError, APITimeoutError, ChatProviderError, @@ -84,11 +85,16 @@ export function translateProviderError(error: unknown): Error2 { ? ProtocolErrors.codes.CONTEXT_OVERFLOW : error instanceof APIProviderOverloadedError || error.statusCode === 529 ? ProtocolErrors.codes.PROVIDER_OVERLOADED - : error.statusCode === 429 - ? ProtocolErrors.codes.PROVIDER_RATE_LIMIT - : error.statusCode === 401 || error.statusCode === 403 - ? ProtocolErrors.codes.PROVIDER_AUTH_ERROR - : ProtocolErrors.codes.PROVIDER_API_ERROR; + : // Quota exhaustion shares status 429 but must not carry the + // rate-limit code — that code drives retry/requeue reactions, + // which cannot help until the account is recharged. + error instanceof APIProviderQuotaExhaustedError + ? ProtocolErrors.codes.PROVIDER_API_ERROR + : error.statusCode === 429 + ? ProtocolErrors.codes.PROVIDER_RATE_LIMIT + : error.statusCode === 401 || error.statusCode === 403 + ? ProtocolErrors.codes.PROVIDER_AUTH_ERROR + : ProtocolErrors.codes.PROVIDER_API_ERROR; return new Error2(code, sanitizeStatusErrorMessage(error.message), { name: error.name, cause: error, diff --git a/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts b/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts index 5de9f5b1aa..2ecc8e6c5d 100644 --- a/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts +++ b/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts @@ -8,6 +8,7 @@ import { APIContextOverflowError, APIEmptyResponseError, APIProviderOverloadedError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIRequestTooLargeError, APIStatusError, @@ -15,6 +16,7 @@ import { ChatProviderError, isImageFormatError, isProviderRateLimitError, + isQuotaExhaustedStatusError, isRecoverableRequestStructureError, isRetryableGenerateError, isToolExchangeAdjacencyError, @@ -696,3 +698,55 @@ describe('isProviderRateLimitError', () => { expect(isProviderRateLimitError(new Error('context length exceeded'))).toBe(false); }); }); + +describe('quota-exhausted 429 classification', () => { + it.each([ + 'You exceeded your current token quota: 31275, please check your account balance', + 'Your account org-0123456789abcdef is suspended due to insufficient balance, please recharge your account or check your plan and billing details', + 'You exceeded your current quota, please check your plan and billing details.', + 'Your account is in arrears, please top up', + ])('normalizes 429 "%s" to APIProviderQuotaExhaustedError by message', (message) => { + const error = normalizeAPIStatusError(429, message, 'req-quota'); + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(error).not.toBeInstanceOf(APIProviderRateLimitError); + expect(error.statusCode).toBe(429); + }); + + it('classifies a neutral message by structured errorType/errorCode', () => { + expect( + normalizeAPIStatusError(429, 'Too many requests', null, null, null, { + errorType: 'exceeded_current_quota_error', + }), + ).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect( + normalizeAPIStatusError(429, 'Too many requests', null, null, null, { + errorCode: 'insufficient_quota', + }), + ).toBeInstanceOf(APIProviderQuotaExhaustedError); + }); + + it.each([ + 'Too many requests', + 'request reached user+model max RPM: 50', + 'your token quota per minute was exceeded', + ])('keeps transient 429 "%s" an APIProviderRateLimitError', (message) => { + const error = normalizeAPIStatusError(429, message); + expect(error).toBeInstanceOf(APIProviderRateLimitError); + expect(error).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + }); + + it('is gated on status 429', () => { + expect(isQuotaExhaustedStatusError(403, 'insufficient balance')).toBe(false); + expect(normalizeAPIStatusError(403, 'insufficient balance')).not.toBeInstanceOf( + APIProviderQuotaExhaustedError, + ); + }); + + it('is neither retryable nor a provider rate limit', () => { + const quota = new APIProviderQuotaExhaustedError('quota exhausted', 'req-quota', 1); + expect(isRetryableGenerateError(quota)).toBe(false); + expect(isProviderRateLimitError(quota)).toBe(false); + expect(isRetryableGenerateError(new APIProviderRateLimitError('rate limited'))).toBe(true); + expect(isProviderRateLimitError(new APIProviderRateLimitError('rate limited'))).toBe(true); + }); +}); diff --git a/packages/agent-core-v2/test/app/llmProtocol/providers/provider-errors.test.ts b/packages/agent-core-v2/test/app/llmProtocol/providers/provider-errors.test.ts index 6615783cfe..5e1ae4c7ea 100644 --- a/packages/agent-core-v2/test/app/llmProtocol/providers/provider-errors.test.ts +++ b/packages/agent-core-v2/test/app/llmProtocol/providers/provider-errors.test.ts @@ -7,7 +7,12 @@ import { APIError as AnthropicAPIError } from '@anthropic-ai/sdk'; import { APIError as OpenAIAPIError } from 'openai'; import { describe, expect, it } from 'vitest'; -import { APIProviderRateLimitError, APIStatusError } from '#/app/llmProtocol/errors'; +import { + APIProviderQuotaExhaustedError, + APIProviderRateLimitError, + APIStatusError, + isRetryableGenerateError, +} from '#/app/llmProtocol/errors'; import { convertAnthropicError } from '#/app/llmProtocol/providers/anthropic'; import { convertOpenAIError } from '#/app/llmProtocol/providers/openai-common'; import { OpenAIResponsesStreamedMessage } from '#/app/llmProtocol/providers/openai-responses'; @@ -92,3 +97,31 @@ describe('OpenAI Responses rate-limit conversion', () => { await expect(consume(stream)).rejects.toBeInstanceOf(APIProviderRateLimitError); }); }); + +describe('OpenAI quota-exhausted 429 conversion', () => { + const QUOTA_MESSAGE = + 'Your account org-0123456789abcdef is suspended due to insufficient balance, please recharge your account or check your plan and billing details'; + + it('classifies a structured exceeded_current_quota_error body as quota-exhausted', () => { + const source = new OpenAIAPIError( + 429, + { message: QUOTA_MESSAGE, type: 'exceeded_current_quota_error' }, + `429 ${QUOTA_MESSAGE}`, + new Headers(), + ); + + const error = convertOpenAIError(source); + + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(error)).toBe(false); + }); + + it('falls back to message wording when no structured body is present', () => { + const source = new OpenAIAPIError(429, undefined, QUOTA_MESSAGE, new Headers()); + + const error = convertOpenAIError(source); + + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(error)).toBe(false); + }); +}); diff --git a/packages/agent-core-v2/test/app/protocol/errors.test.ts b/packages/agent-core-v2/test/app/protocol/errors.test.ts index 0327351f3c..3e15b1a0fa 100644 --- a/packages/agent-core-v2/test/app/protocol/errors.test.ts +++ b/packages/agent-core-v2/test/app/protocol/errors.test.ts @@ -6,6 +6,7 @@ import { APIContextOverflowError, APIEmptyResponseError, APIProviderOverloadedError, + APIProviderQuotaExhaustedError, APIStatusError, APITimeoutError, ChatProviderError, @@ -152,4 +153,20 @@ describe('translateProviderError', () => { ); }); }); + + describe('quota-exhausted 429', () => { + it('maps to provider.api_error, not provider.rate_limit', () => { + // provider.rate_limit drives retry/requeue reactions, which cannot help + // until the account is recharged. + const translated = translateProviderError( + new APIProviderQuotaExhaustedError( + 'Your account is suspended due to insufficient balance, please recharge your account', + 'req-quota', + ), + ); + expect(translated.code).toBe('provider.api_error'); + expect(translated.message).toContain('recharge'); + expect(translated.details).toMatchObject({ statusCode: 429, requestId: 'req-quota' }); + }); + }); }); diff --git a/packages/agent-core/src/agent/turn/index.ts b/packages/agent-core/src/agent/turn/index.ts index 3136aa6889..f4d6258f4b 100644 --- a/packages/agent-core/src/agent/turn/index.ts +++ b/packages/agent-core/src/agent/turn/index.ts @@ -5,6 +5,7 @@ import { APIConnectionError, APIContextOverflowError, APIEmptyResponseError, + APIProviderQuotaExhaustedError, APIStatusError, APITimeoutError, inputTotal, @@ -1437,6 +1438,11 @@ interface ApiErrorClassification { } function classifyApiError(error: unknown, summary: KimiErrorPayload): ApiErrorClassification { + // Quota/balance exhaustion shares status 429 with rate limits but fails + // fast instead of retrying — keep the two apart in telemetry. + if (error instanceof APIProviderQuotaExhaustedError) { + return { errorType: 'quota_exhausted', statusCode: error.statusCode }; + } const statusCode = apiStatusCode(error) ?? summaryStatusCode(summary); if (statusCode !== undefined) { if (statusCode === 429) return { errorType: 'rate_limit', statusCode }; diff --git a/packages/agent-core/src/errors/serialize.ts b/packages/agent-core/src/errors/serialize.ts index 17ba995343..632334c8d7 100644 --- a/packages/agent-core/src/errors/serialize.ts +++ b/packages/agent-core/src/errors/serialize.ts @@ -1,6 +1,7 @@ import { APIConnectionError, APIEmptyResponseError, + APIProviderQuotaExhaustedError, APIStatusError, APITimeoutError, ChatProviderError, @@ -58,6 +59,10 @@ export function makeErrorPayload( * Recognized errors: * - `KimiError`: passthrough. * - `APIStatusError`: 429 -> rate_limit, 401 -> auth_error, otherwise -> api_error. + * Exception: a quota-exhausted 429 maps to api_error (retryable: false) — + * the rate_limit code would re-mint a rate-limit error across the wire + * boundary and drive the swarm requeue/suspend loop, which cannot help + * until the account is recharged. * - `APIConnectionError` / `APITimeoutError`: connection_error. * - `ChatProviderError`: api_error. * @@ -77,11 +82,13 @@ export function toKimiErrorPayload(error: unknown): KimiErrorPayload { if (error instanceof APIStatusError) { const code: KimiErrorCode = - error.statusCode === 429 - ? ErrorCodes.PROVIDER_RATE_LIMIT - : error.statusCode === 401 - ? ErrorCodes.PROVIDER_AUTH_ERROR - : ErrorCodes.PROVIDER_API_ERROR; + error instanceof APIProviderQuotaExhaustedError + ? ErrorCodes.PROVIDER_API_ERROR + : error.statusCode === 429 + ? ErrorCodes.PROVIDER_RATE_LIMIT + : error.statusCode === 401 + ? ErrorCodes.PROVIDER_AUTH_ERROR + : ErrorCodes.PROVIDER_API_ERROR; return { code, message: sanitizeStatusErrorMessage(error.message), diff --git a/packages/agent-core/test/errors/serialize.test.ts b/packages/agent-core/test/errors/serialize.test.ts index 43e56ff717..db095b0613 100644 --- a/packages/agent-core/test/errors/serialize.test.ts +++ b/packages/agent-core/test/errors/serialize.test.ts @@ -1,4 +1,4 @@ -import { APIStatusError } from '@moonshot-ai/kosong'; +import { APIProviderQuotaExhaustedError, APIStatusError } from '@moonshot-ai/kosong'; import { describe, expect, it } from 'vitest'; import { toKimiErrorPayload } from '#/errors/serialize'; @@ -48,3 +48,21 @@ describe('toKimiErrorPayload — APIStatusError message sanitization', () => { ); }); }); + +describe('toKimiErrorPayload — quota-exhausted 429', () => { + it('maps a quota-exhausted 429 to provider.api_error, not provider.rate_limit', () => { + // provider.rate_limit is retryable and re-minted as a rate-limit error + // across the wire boundary, which drives the swarm requeue/suspend loop; + // quota exhaustion must carry the non-retryable generic code instead. + const payload = toKimiErrorPayload( + new APIProviderQuotaExhaustedError( + 'Your account is suspended due to insufficient balance, please recharge your account', + 'req-quota', + ), + ); + expect(payload.code).toBe('provider.api_error'); + expect(payload.retryable).toBe(false); + expect(payload.message).toContain('recharge'); + expect(payload.details).toMatchObject({ statusCode: 429, requestId: 'req-quota' }); + }); +}); diff --git a/packages/agent-core/test/loop/retry.test.ts b/packages/agent-core/test/loop/retry.test.ts index 9c38ed7811..8166cfc116 100644 --- a/packages/agent-core/test/loop/retry.test.ts +++ b/packages/agent-core/test/loop/retry.test.ts @@ -1,5 +1,6 @@ import { APIConnectionError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, emptyUsage, isRetryableGenerateError, @@ -210,6 +211,43 @@ describe('chatWithRetry: default retry budget', () => { }); }); +describe('chatWithRetry: quota-exhausted 429 fails fast', () => { + it('does not retry a quota-exhausted 429 even when it carries retry-after', async () => { + // Same status as a rate limit, but exhausted quota/balance never clears + // on its own — the error must surface after a single attempt instead of + // burning the whole default budget. The 1ms retry-after proves a server + // backoff hint does not re-enable retries either. + let calls = 0; + const captured: Array<{ type: string }> = []; + const llm: LLM = { + systemPrompt: '', + modelName: 'mock', + isRetryableError: (e) => isRetryableGenerateError(e), + async chat(): Promise { + calls += 1; + throw new APIProviderQuotaExhaustedError( + 'Your account is suspended due to insufficient balance, please recharge your account', + null, + 1, + ); + }, + }; + const input = makeInput(llm, new AbortController().signal); + + await expect( + chatWithRetry({ + ...input, + dispatchEvent: async (event) => { + captured.push(event as { type: string }); + }, + }), + ).rejects.toMatchObject({ name: 'APIProviderQuotaExhaustedError' }); + + expect(calls).toBe(1); + expect(captured.filter((e) => e.type === 'step.retrying')).toHaveLength(0); + }); +}); + describe('chatWithRetry: honors server retry-after', () => { it('uses the error retryAfterMs as the retry delay instead of the backoff', async () => { let calls = 0; diff --git a/packages/kosong/src/errors.ts b/packages/kosong/src/errors.ts index 9f0862e753..1d937ea762 100644 --- a/packages/kosong/src/errors.ts +++ b/packages/kosong/src/errors.ts @@ -119,6 +119,33 @@ export class APIProviderRateLimitError extends APIStatusError { } } +/** + * HTTP 429 that specifically means the account's quota or balance is + * exhausted, as opposed to a transient rate limit. Deliberately NOT a + * subclass of `APIProviderRateLimitError`: a rate limit clears on its own + * (retry/requeue helps), while quota exhaustion is deterministic until the + * account is recharged — so this class is excluded from retry and from the + * rate-limit requeue/suspend paths. + * + * Observed shapes: Moonshot returns `error.type = + * "exceeded_current_quota_error"` with wording that varies by account state + * ("You exceeded your current token quota: ... please check your account + * balance" vs "Your account ... is suspended due to insufficient balance, + * please recharge your account ..."); OpenAI uses `insufficient_quota` as + * both `error.type` and `error.code`. + */ +export class APIProviderQuotaExhaustedError extends APIStatusError { + constructor( + message: string, + requestId?: string | null, + retryAfterMs?: number | null, + traceId?: string | null, + ) { + super(429, message, requestId, retryAfterMs, traceId); + this.name = 'APIProviderQuotaExhaustedError'; + } +} + /** * The API returned an empty response (no content, no tool calls). */ @@ -148,6 +175,12 @@ export function isRetryableGenerateError(error: unknown): boolean { return true; } if (error instanceof APIStatusError) { + // Quota/balance exhaustion is a 429 but deterministic until the account + // is recharged — retrying can never succeed, so it fails fast instead of + // burning the whole retry budget (~2-3 minutes of backoff). + if (error instanceof APIProviderQuotaExhaustedError) { + return false; + } // Transient statuses worth retrying: 408 (request timeout), 409 // (lock/conflict timeout), 429 (rate limit), 5xx (server errors) and 529 // (provider overloaded — the "engine is currently overloaded" case). @@ -290,6 +323,30 @@ const PROVIDER_RATE_LIMIT_MESSAGE_PATTERNS = [ /rate-limited/, ] as const; +// Structured error `type`/`code` values that mean the account's quota or +// balance is exhausted (as opposed to a transient rate limit). Moonshot sets +// `exceeded_current_quota_error` as the body `error.type`; OpenAI uses +// `insufficient_quota` as both `type` and `code`. +const QUOTA_EXHAUSTED_ERROR_CODES = new Set(['exceeded_current_quota_error', 'insufficient_quota']); + +// Message fallback for providers/gateways that do not forward a structured +// type/code, matched against the lowercased message of a 429. Every pattern +// is anchored to billing wording — deliberately no bare /quota/ or /balance/, +// which would also match transient throttle messages like "token quota per +// minute". Grounded in observed bodies: Moonshot "You exceeded your current +// token quota: ... please check your account balance" and "Your account ... +// is suspended due to insufficient balance, please recharge your account or +// check your plan and billing details"; OpenAI "You exceeded your current +// quota, please check your plan and billing details". +const QUOTA_EXHAUSTED_MESSAGE_PATTERNS = [ + /exceeded your current (?:token )?quota/, + /check your account balance/, + /insufficient balance/, + /recharge your account|please recharge/, + /account (?:is )?in arrears/, + /insufficient_quota/, +] as const; + // Wordings that mean the serialized request BODY was too big, matched against // the lowercased message of a 413. Kept separate from the context-overflow // patterns above: those describe token counts, these describe bytes. A 413 @@ -346,8 +403,14 @@ export function normalizeAPIStatusError( requestId?: string | null, retryAfterMs?: number | null, traceId?: string | null, + options?: { readonly errorCode?: string | null; readonly errorType?: string | null }, ): APIStatusError { if (statusCode === 429) { + // Quota/balance exhaustion first: same status as a rate limit, but it + // never clears on its own, so it must not classify as retryable. + if (isQuotaExhaustedStatusError(statusCode, message, options)) { + return new APIProviderQuotaExhaustedError(message, requestId, retryAfterMs, traceId); + } return new APIProviderRateLimitError(message, requestId, retryAfterMs, traceId); } // Context overflow first: Vertex returns prompt-too-long as a 413, and a @@ -418,6 +481,26 @@ export function isRequestTooLargeStatusError(statusCode: number, message: string return REQUEST_TOO_LARGE_MESSAGE_PATTERNS.some((pattern) => pattern.test(lowerMessage)); } +/** + * Whether a 429 means the account's quota/balance is exhausted rather than a + * transient rate limit. The structured body `error.type`/`error.code` is + * authoritative when the provider forwards one; the message patterns only + * backstop gateways that flatten the body to text. + */ +export function isQuotaExhaustedStatusError( + statusCode: number, + message: string, + options?: { readonly errorCode?: string | null; readonly errorType?: string | null }, +): boolean { + if (statusCode !== 429) return false; + const errorCode = options?.errorCode; + if (typeof errorCode === 'string' && QUOTA_EXHAUSTED_ERROR_CODES.has(errorCode)) return true; + const errorType = options?.errorType; + if (typeof errorType === 'string' && QUOTA_EXHAUSTED_ERROR_CODES.has(errorType)) return true; + const lowerMessage = message.toLowerCase(); + return QUOTA_EXHAUSTED_MESSAGE_PATTERNS.some((pattern) => pattern.test(lowerMessage)); +} + // Strict providers reject a request whose assistant `tool_use`/`tool_calls` and // `tool_result`/`tool` blocks are not correctly paired and adjacent — a missing // result, a stray result with no matching call, or a result that does not @@ -502,6 +585,9 @@ export function isRecoverableRequestStructureError(error: unknown): boolean { } export function isProviderRateLimitError(error: unknown): boolean { + // Quota exhaustion is a 429 but not a rate limit: the rate-limit reactions + // (retry, requeue, suspend) cannot help until the account is recharged. + if (error instanceof APIProviderQuotaExhaustedError) return false; if (error instanceof APIProviderRateLimitError) return true; const statusCode = getStatusCode(error); diff --git a/packages/kosong/src/index.ts b/packages/kosong/src/index.ts index 10a7af7304..04a30b83a8 100644 --- a/packages/kosong/src/index.ts +++ b/packages/kosong/src/index.ts @@ -62,6 +62,7 @@ export { APIConnectionError, APIContextOverflowError, APIEmptyResponseError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIRequestTooLargeError, APIStatusError, @@ -70,6 +71,7 @@ export { isContextOverflowStatusError, isImageFormatError, isProviderRateLimitError, + isQuotaExhaustedStatusError, isRecoverableRequestStructureError, isRequestTooLargeStatusError, isRetryableGenerateError, diff --git a/packages/kosong/src/providers/openai-common.ts b/packages/kosong/src/providers/openai-common.ts index bc3e18cddc..5d051ea842 100644 --- a/packages/kosong/src/providers/openai-common.ts +++ b/packages/kosong/src/providers/openai-common.ts @@ -111,6 +111,13 @@ export function convertOpenAIError(error: unknown): ChatProviderError { reqId, parseRetryAfterMs(error.headers), parseTraceId(error.headers), + // The SDK parses the body's `error.code`/`error.type` onto the error; + // forward them so a quota-exhausted 429 classifies structurally rather + // than by message wording. + { + errorCode: typeof error.code === 'string' ? error.code : null, + errorType: typeof error.type === 'string' ? error.type : null, + }, ); } // Base APIError with no status and no body => transport-layer failure. diff --git a/packages/kosong/test/errors.test.ts b/packages/kosong/test/errors.test.ts index 1eb62efdbf..eb235866f4 100644 --- a/packages/kosong/test/errors.test.ts +++ b/packages/kosong/test/errors.test.ts @@ -2,6 +2,7 @@ import { APIConnectionError, APIContextOverflowError, APIEmptyResponseError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIRequestTooLargeError, APIStatusError, @@ -9,6 +10,7 @@ import { ChatProviderError, isImageFormatError, isProviderRateLimitError, + isQuotaExhaustedStatusError, isRecoverableRequestStructureError, isRetryableGenerateError, isToolExchangeAdjacencyError, @@ -667,3 +669,89 @@ describe('isImageFormatError', () => { ).toBe(false); }); }); + +describe('APIProviderQuotaExhaustedError', () => { + it('extends APIStatusError and preserves HTTP details', () => { + const err = new APIProviderQuotaExhaustedError('quota exhausted', 'req-quota', 12_500); + expect(err).toBeInstanceOf(APIStatusError); + expect(err).toBeInstanceOf(ChatProviderError); + expect(err).not.toBeInstanceOf(APIProviderRateLimitError); + expect(err.name).toBe('APIProviderQuotaExhaustedError'); + expect(err.statusCode).toBe(429); + expect(err.requestId).toBe('req-quota'); + expect(err.retryAfterMs).toBe(12_500); + }); +}); + +describe('normalizeAPIStatusError: quota-exhausted 429', () => { + // Both Moonshot wordings observed live from the same account (`error.type` + // "exceeded_current_quota_error"), plus OpenAI's insufficient_quota wording + // and the arrears synonym. + it.each([ + 'You exceeded your current token quota: 31275, please check your account balance', + 'Your account org-0123456789abcdef is suspended due to insufficient balance, please recharge your account or check your plan and billing details', + 'You exceeded your current quota, please check your plan and billing details.', + 'Your account is in arrears, please top up', + ])('classifies 429 "%s" as quota-exhausted by message', (message) => { + const error = normalizeAPIStatusError(429, message, 'req-quota'); + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(error.statusCode).toBe(429); + expect(error.requestId).toBe('req-quota'); + }); + + it('classifies a neutral message as quota-exhausted by structured errorType', () => { + const error = normalizeAPIStatusError(429, 'Too many requests', null, null, null, { + errorType: 'exceeded_current_quota_error', + }); + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + }); + + it('classifies a neutral message as quota-exhausted by structured errorCode', () => { + const error = normalizeAPIStatusError(429, 'Too many requests', null, null, null, { + errorCode: 'insufficient_quota', + }); + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + }); + + it.each([ + 'Too many requests', + 'request reached user+model max RPM: 50', + 'your token quota per minute was exceeded', + ])('keeps transient 429 "%s" an APIProviderRateLimitError', (message) => { + const error = normalizeAPIStatusError(429, message); + expect(error).toBeInstanceOf(APIProviderRateLimitError); + expect(error).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + }); + + it('keeps a 429 with a transient structured type an APIProviderRateLimitError', () => { + const error = normalizeAPIStatusError(429, 'Too many requests', null, null, null, { + errorType: 'rate_limit_reached_error', + }); + expect(error).toBeInstanceOf(APIProviderRateLimitError); + expect(error).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + }); + + it('is gated on status 429 — billing wording on other statuses stays generic', () => { + expect(isQuotaExhaustedStatusError(403, 'insufficient balance')).toBe(false); + const error = normalizeAPIStatusError(403, 'insufficient balance'); + expect(error).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(error.constructor).toBe(APIStatusError); + }); +}); + +describe('quota-exhausted retry and rate-limit semantics', () => { + it('is not retryable while a plain rate limit stays retryable', () => { + expect(isRetryableGenerateError(new APIProviderQuotaExhaustedError('quota exhausted'))).toBe( + false, + ); + expect(isRetryableGenerateError(new APIProviderRateLimitError('rate limited'))).toBe(true); + expect(isRetryableGenerateError(new APIStatusError(429, 'rate limited'))).toBe(true); + }); + + it('is not a provider rate limit despite carrying status 429', () => { + expect(isProviderRateLimitError(new APIProviderQuotaExhaustedError('quota exhausted'))).toBe( + false, + ); + expect(isProviderRateLimitError(new APIProviderRateLimitError('rate limited'))).toBe(true); + }); +}); diff --git a/packages/kosong/test/openai-common-errors.test.ts b/packages/kosong/test/openai-common-errors.test.ts index d2b1bf388e..264fb60f59 100644 --- a/packages/kosong/test/openai-common-errors.test.ts +++ b/packages/kosong/test/openai-common-errors.test.ts @@ -1,6 +1,7 @@ import { APIConnectionError, APIContextOverflowError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIStatusError, APITimeoutError, @@ -393,3 +394,43 @@ describe('convertOpenAIError: non-Error values', () => { expect(result.message).toContain('plain error'); }); }); + +describe('convertOpenAIError: quota-exhausted 429', () => { + const QUOTA_MESSAGE = + 'Your account org-0123456789abcdef is suspended due to insufficient balance, please recharge your account or check your plan and billing details'; + + it('classifies a structured exceeded_current_quota_error body as quota-exhausted', () => { + // The SDK parses the body's inner error object onto the APIError, exposing + // `type` — the structured path must win regardless of message wording. + const err = new OpenAIAPIError( + 429, + { message: QUOTA_MESSAGE, type: 'exceeded_current_quota_error' }, + `429 ${QUOTA_MESSAGE}`, + new Headers(), + ); + const result = convertOpenAIError(err); + expect(result).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect((result as APIProviderQuotaExhaustedError).statusCode).toBe(429); + expect(isRetryableGenerateError(result)).toBe(false); + }); + + it('falls back to message wording when no structured body is present', () => { + const err = new OpenAIAPIError(429, undefined, QUOTA_MESSAGE, new Headers()); + const result = convertOpenAIError(err); + expect(result).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(result)).toBe(false); + }); + + it('keeps a transient structured 429 an APIProviderRateLimitError', () => { + const err = new OpenAIAPIError( + 429, + { message: 'Too many requests', type: 'rate_limit_reached_error' }, + 'Too many requests', + new Headers(), + ); + const result = convertOpenAIError(err); + expect(result).toBeInstanceOf(APIProviderRateLimitError); + expect(result).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(result)).toBe(true); + }); +}); From 5c60c91604b92c97cc6490ebb414afc6e3a3320a Mon Sep 17 00:00:00 2001 From: vinlee19 <1401597760@qq.com> Date: Fri, 17 Jul 2026 23:28:30 +0800 Subject: [PATCH 2/3] fix(kosong): classify quota exhaustion in OpenAI Responses stream errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Responses response.failed / error SSE events carry no HTTP status and were minted by errorFromOpenAIResponsesEvent as either a rate-limit error (rate_limit_exceeded / embedded status_code=429) or a base ChatProviderError — and the base class falls into the retryable unclassified-failure fallback, so an insufficient_quota event still burned the whole retry budget on the openai_responses path. Route the event code and message through the same quota-exhausted check before the rate-limit branch, in kosong and the agent-core-v2 mirror. Covers all three entry paths (error events, response.failed, nested gateway frames) since they share the single converter. --- .../llmProtocol/providers/openai-responses.ts | 5 ++ .../providers/provider-errors.test.ts | 21 +++++++ .../kosong/src/providers/openai-responses.ts | 11 ++++ packages/kosong/test/openai-responses.test.ts | 58 +++++++++++++++++++ 4 files changed, 95 insertions(+) diff --git a/packages/agent-core-v2/src/app/llmProtocol/providers/openai-responses.ts b/packages/agent-core-v2/src/app/llmProtocol/providers/openai-responses.ts index 9953a0d148..efac57f455 100644 --- a/packages/agent-core-v2/src/app/llmProtocol/providers/openai-responses.ts +++ b/packages/agent-core-v2/src/app/llmProtocol/providers/openai-responses.ts @@ -1,8 +1,10 @@ import { APIContextOverflowError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, ChatProviderError, isContextOverflowErrorCode, + isQuotaExhaustedStatusError, } from '../errors'; import type { ContentPart, Message, StreamedMessagePart, ToolCall } from '../message'; import { extractText, isToolDeclarationOnlyMessage } from '../message'; @@ -242,6 +244,9 @@ function errorFromOpenAIResponsesEvent( if (isContextOverflowErrorCode(code)) { return new APIContextOverflowError(400, fullMessage); } + if (isQuotaExhaustedStatusError(429, fullMessage, { errorCode: code })) { + return new APIProviderQuotaExhaustedError(fullMessage); + } if (code === 'rate_limit_exceeded' || readEmbeddedStatusCode(message) === 429) { return new APIProviderRateLimitError(fullMessage); } diff --git a/packages/agent-core-v2/test/app/llmProtocol/providers/provider-errors.test.ts b/packages/agent-core-v2/test/app/llmProtocol/providers/provider-errors.test.ts index 5e1ae4c7ea..987f737a32 100644 --- a/packages/agent-core-v2/test/app/llmProtocol/providers/provider-errors.test.ts +++ b/packages/agent-core-v2/test/app/llmProtocol/providers/provider-errors.test.ts @@ -96,6 +96,27 @@ describe('OpenAI Responses rate-limit conversion', () => { await expect(consume(stream)).rejects.toBeInstanceOf(APIProviderRateLimitError); }); + + it('fails fast on a streamed insufficient_quota error event', async () => { + const stream = new OpenAIResponsesStreamedMessage( + streamEvents([ + { + type: 'error', + code: 'insufficient_quota', + message: 'You exceeded your current quota, please check your plan and billing details.', + param: null, + }, + ]), + true, + ); + + const caught = await consume(stream).then( + () => undefined, + (error: unknown) => error, + ); + expect(caught).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(caught)).toBe(false); + }); }); describe('OpenAI quota-exhausted 429 conversion', () => { diff --git a/packages/kosong/src/providers/openai-responses.ts b/packages/kosong/src/providers/openai-responses.ts index baf8666abe..9aaccc2ff2 100644 --- a/packages/kosong/src/providers/openai-responses.ts +++ b/packages/kosong/src/providers/openai-responses.ts @@ -1,8 +1,10 @@ import { APIContextOverflowError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, ChatProviderError, isContextOverflowErrorCode, + isQuotaExhaustedStatusError, } from '#/errors'; import type { ContentPart, Message, StreamedMessagePart, ToolCall } from '#/message'; import { extractText, isToolDeclarationOnlyMessage } from '#/message'; @@ -251,6 +253,15 @@ function errorFromOpenAIResponsesEvent( if (isContextOverflowErrorCode(code)) { return new APIContextOverflowError(400, fullMessage); } + // Quota/balance exhaustion first — otherwise an `insufficient_quota` event + // falls through to the base ChatProviderError (whose unclassified fallback + // is retryable), and a quota message with an embedded status_code=429 would + // classify as a retryable rate limit. Responses stream events carry no HTTP + // status, so the 429 passed here only satisfies the predicate's status gate + // while the event code / billing wording carries the actual evidence. + if (isQuotaExhaustedStatusError(429, fullMessage, { errorCode: code })) { + return new APIProviderQuotaExhaustedError(fullMessage); + } if (code === 'rate_limit_exceeded' || readEmbeddedStatusCode(message) === 429) { return new APIProviderRateLimitError(fullMessage); } diff --git a/packages/kosong/test/openai-responses.test.ts b/packages/kosong/test/openai-responses.test.ts index 1a8d4c752d..faf6df3140 100644 --- a/packages/kosong/test/openai-responses.test.ts +++ b/packages/kosong/test/openai-responses.test.ts @@ -1,8 +1,10 @@ import { APIContextOverflowError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIStatusError, ChatProviderError, + isRetryableGenerateError, } from '#/errors'; import { generate } from '#/generate'; import type { ContentPart, Message, StreamedMessagePart, ToolCall } from '#/message'; @@ -2008,6 +2010,62 @@ describe('OpenAIResponsesChatProvider', () => { expect((caughtError as Error).message).toContain('status_code=429'); }); + it('fails fast on response.failed with an insufficient_quota code', async () => { + // Quota exhaustion arriving as a Responses stream event carries no HTTP + // status; without the structured-code check it would fall through to the + // base ChatProviderError, whose unclassified fallback is retryable — and + // burn the whole retry budget on an error that cannot succeed. + const events = [ + { + type: 'response.failed', + response: { + id: 'resp_quota', + status: 'failed', + error: { + code: 'insufficient_quota', + message: 'You exceeded your current quota, please check your plan and billing details.', + }, + }, + }, + ]; + const stream = new OpenAIResponsesStreamedMessage(makeAsyncIterable(events), true); + + let caughtError: unknown; + try { + await collectStreamParts(stream); + } catch (error) { + caughtError = error; + } + + expect(caughtError).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect((caughtError as APIProviderQuotaExhaustedError).statusCode).toBe(429); + expect(isRetryableGenerateError(caughtError)).toBe(false); + }); + + it('classifies an embedded status_code=429 with billing wording as quota exhausted', async () => { + const events = [ + { + type: 'error', + code: 'upstream_error', + message: + 'llmproxy/openai/responses/resp_q.json status_code=429 Your account is suspended due to insufficient balance, please recharge your account', + param: null, + }, + ]; + const stream = new OpenAIResponsesStreamedMessage(makeAsyncIterable(events), true); + + let caughtError: unknown; + try { + await collectStreamParts(stream); + } catch (error) { + caughtError = error; + } + + expect(caughtError).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(caughtError).not.toBeInstanceOf(APIProviderRateLimitError); + expect(isRetryableGenerateError(caughtError)).toBe(false); + }); + it('rejects malformed stream events with a non-string type even when message is present', async () => { const events = [ { From 7ceabcc41e7fe6fe0d32be3f17fc306756e463ac Mon Sep 17 00:00:00 2001 From: vinlee19 <1401597760@qq.com> Date: Fri, 17 Jul 2026 23:28:31 +0800 Subject: [PATCH 3/3] style(agent-core-v2): drop inline comments per AGENTS.md header-only rule agent-core-v2 comments live solely in the top-of-file block, never beside functions or statements; the kosong twins keep the full rationale. --- .../agent-core-v2/src/app/llmProtocol/errors.ts | 17 ----------------- .../app/llmProtocol/providers/openai-common.ts | 2 -- .../agent-core-v2/src/app/protocol/errors.ts | 5 +---- .../test/app/protocol/errors.test.ts | 2 -- 4 files changed, 1 insertion(+), 25 deletions(-) diff --git a/packages/agent-core-v2/src/app/llmProtocol/errors.ts b/packages/agent-core-v2/src/app/llmProtocol/errors.ts index 5fd039aaa2..6ba54a0e55 100644 --- a/packages/agent-core-v2/src/app/llmProtocol/errors.ts +++ b/packages/agent-core-v2/src/app/llmProtocol/errors.ts @@ -82,9 +82,6 @@ export class APIProviderRateLimitError extends APIStatusError { } } -// HTTP 429 caused by an exhausted account quota/balance — deterministic until -// the account is recharged, so unlike a rate limit it is neither retried nor -// requeued. Deliberately not a subclass of APIProviderRateLimitError. export class APIProviderQuotaExhaustedError extends APIStatusError { constructor( message: string, @@ -173,8 +170,6 @@ export function isRetryableGenerateError(error: unknown): boolean { return true; } if (error instanceof APIStatusError) { - // Quota/balance exhaustion is a 429 but deterministic until the account - // is recharged — retrying can never succeed. if (error instanceof APIProviderQuotaExhaustedError) { return false; } @@ -219,15 +214,8 @@ const PROVIDER_RATE_LIMIT_MESSAGE_PATTERNS = [ const PROVIDER_OVERLOAD_MESSAGE_PATTERNS = [/overload/] as const; -// Structured error `type`/`code` values that mean the account's quota or -// balance is exhausted: Moonshot `exceeded_current_quota_error` (body -// `error.type`), OpenAI `insufficient_quota` (both `type` and `code`). const QUOTA_EXHAUSTED_ERROR_CODES = new Set(['exceeded_current_quota_error', 'insufficient_quota']); -// Message fallback for gateways that flatten the body to text, matched against -// the lowercased message of a 429. Anchored to billing wording — deliberately -// no bare /quota/ or /balance/, which would also match transient throttle -// messages like "token quota per minute". const QUOTA_EXHAUSTED_MESSAGE_PATTERNS = [ /exceeded your current (?:token )?quota/, /check your account balance/, @@ -349,9 +337,6 @@ export function isRequestTooLargeStatusError(statusCode: number, message: string return REQUEST_TOO_LARGE_MESSAGE_PATTERNS.some((pattern) => pattern.test(lowerMessage)); } -// Whether a 429 means the account's quota/balance is exhausted rather than a -// transient rate limit. Structured `type`/`code` is authoritative when -// forwarded; message patterns only backstop text-flattening gateways. export function isQuotaExhaustedStatusError( statusCode: number, message: string, @@ -404,8 +389,6 @@ export function isRecoverableRequestStructureError(error: unknown): boolean { } export function isProviderRateLimitError(error: unknown): boolean { - // Quota exhaustion is a 429 but not a rate limit: the rate-limit reactions - // (retry, requeue, suspend) cannot help until the account is recharged. if (error instanceof APIProviderQuotaExhaustedError) return false; if (error instanceof APIProviderRateLimitError) return true; diff --git a/packages/agent-core-v2/src/app/llmProtocol/providers/openai-common.ts b/packages/agent-core-v2/src/app/llmProtocol/providers/openai-common.ts index e3cf546c65..845275090b 100644 --- a/packages/agent-core-v2/src/app/llmProtocol/providers/openai-common.ts +++ b/packages/agent-core-v2/src/app/llmProtocol/providers/openai-common.ts @@ -98,8 +98,6 @@ export function convertOpenAIError(error: unknown): ChatProviderError { reqId, parseRetryAfterMs(error.headers), parseTraceId(error.headers), - // Forward the SDK-parsed body `error.code`/`error.type` so a - // quota-exhausted 429 classifies structurally, not by wording. { errorCode: typeof error.code === 'string' ? error.code : null, errorType: typeof error.type === 'string' ? error.type : null, diff --git a/packages/agent-core-v2/src/app/protocol/errors.ts b/packages/agent-core-v2/src/app/protocol/errors.ts index 694dc00152..f8617494c1 100644 --- a/packages/agent-core-v2/src/app/protocol/errors.ts +++ b/packages/agent-core-v2/src/app/protocol/errors.ts @@ -85,10 +85,7 @@ export function translateProviderError(error: unknown): Error2 { ? ProtocolErrors.codes.CONTEXT_OVERFLOW : error instanceof APIProviderOverloadedError || error.statusCode === 529 ? ProtocolErrors.codes.PROVIDER_OVERLOADED - : // Quota exhaustion shares status 429 but must not carry the - // rate-limit code — that code drives retry/requeue reactions, - // which cannot help until the account is recharged. - error instanceof APIProviderQuotaExhaustedError + : error instanceof APIProviderQuotaExhaustedError ? ProtocolErrors.codes.PROVIDER_API_ERROR : error.statusCode === 429 ? ProtocolErrors.codes.PROVIDER_RATE_LIMIT diff --git a/packages/agent-core-v2/test/app/protocol/errors.test.ts b/packages/agent-core-v2/test/app/protocol/errors.test.ts index 3e15b1a0fa..a99e0b7ebf 100644 --- a/packages/agent-core-v2/test/app/protocol/errors.test.ts +++ b/packages/agent-core-v2/test/app/protocol/errors.test.ts @@ -156,8 +156,6 @@ describe('translateProviderError', () => { describe('quota-exhausted 429', () => { it('maps to provider.api_error, not provider.rate_limit', () => { - // provider.rate_limit drives retry/requeue reactions, which cannot help - // until the account is recharged. const translated = translateProviderError( new APIProviderQuotaExhaustedError( 'Your account is suspended due to insufficient balance, please recharge your account',