From a87b0d938811d2e54b4c9bbf10a56ef7e6db004b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=94=A1=E5=8F=8A?= <522caiji@gmail.com> Date: Sun, 11 Oct 2026 11:55:29 +0800 Subject: [PATCH] fix(devin): honor rate-limit reset times and scope quota cooldowns Devin reports its model rate-limit reset in the response body ("reset in 13 minutes" / "(at 18:13 UTC)") and sends no Retry-After. Parse the body hint, and prefer a Retry-After header when present. Classify usage-quota exhaustion as an account-wide quota instead of a model rate limit, and let the executor replace its local-midnight fallback with the parsed reset so the account cools for exactly as long as the upstream says. --- .../unreleased/devin-rate-limit-reset.md | 7 ++ internal/executor/classify_test.go | 16 ++- internal/executor/error.go | 9 +- internal/providers/devin/chat.go | 4 +- internal/providers/devin/devin_test.go | 101 ++++++++++++++++++ internal/providers/devin/errors.go | 97 ++++++++++++++++- 6 files changed, 224 insertions(+), 10 deletions(-) create mode 100644 changelog/unreleased/devin-rate-limit-reset.md diff --git a/changelog/unreleased/devin-rate-limit-reset.md b/changelog/unreleased/devin-rate-limit-reset.md new file mode 100644 index 00000000..044e367e --- /dev/null +++ b/changelog/unreleased/devin-rate-limit-reset.md @@ -0,0 +1,7 @@ +### English + +- Honor Devin rate-limit reset times from the response body, and cool an exhausted usage quota for the whole account instead of until local midnight. + +### 中文 + +- 采用 Devin 响应正文里的限流重置时间，并把用尽的用量配额冷却到整个账号，而不是冷却到本地午夜。 diff --git a/internal/executor/classify_test.go b/internal/executor/classify_test.go index d68478df..c2ab5ad6 100644 --- a/internal/executor/classify_test.go +++ b/internal/executor/classify_test.go @@ -82,12 +82,24 @@ func TestClassifyErrorKeepsJSONRetryAfterWhenCodeIsSet(t *testing.T) { } } +func TestClassifyErrorHonorsProviderQuotaReset(t *testing.T) { + got := ClassifyError(&providers.Error{ + Kind: KindQuota, Status: 429, Message: "usage quota exhausted", RetryAfter: 13 * time.Minute, + }) + if got.Kind != KindQuota || got.Failover || got.Cooldown != 13*time.Minute || got.RetryAfter != 13*time.Minute { + t.Fatalf("quota reset hint must replace local midnight without failing over, got %+v", got) + } + if got.Model != "" { + t.Fatalf("quota cooldown must stay account-wide, got model %q", got.Model) + } +} + func TestClassifyErrorIgnoresProviderFailoverAndAuthCooldown(t *testing.T) { got := ClassifyError(&providers.Error{ - Kind: KindQuota, Status: 429, Message: "plan exhausted", RetryAfter: time.Minute, + Kind: KindQuota, Status: 429, Message: "plan exhausted", }) if got.Kind != KindQuota || got.Failover || got.Cooldown <= 0 || got.Cooldown > 24*time.Hour { - t.Fatalf("quota must not fail over and must use local midnight, got %+v", got) + t.Fatalf("quota without a reset hint must use local midnight, got %+v", got) } got = ClassifyError(&providers.Error{ diff --git a/internal/executor/error.go b/internal/executor/error.go index 5e6cb2d3..66f49f26 100644 --- a/internal/executor/error.go +++ b/internal/executor/error.go @@ -120,7 +120,8 @@ func ClassifyError(err error) Classified { classified.Failover = true classified.Cooldown = 0 } - if classified.Kind == accounts.KindRateLimit { + switch classified.Kind { + case accounts.KindRateLimit: switch { case providerErr.Code == "4011" && providerErr.RetryAfter <= 0: classified.Cooldown = 5 * time.Minute @@ -130,6 +131,12 @@ func ClassifyError(err error) Classified { classified.Cooldown = 30 * time.Second } } + case accounts.KindQuota: + // A parsed upstream reset replaces the local-midnight fallback. + // Quota stays account-wide and does not fail over mid-request. + if providerErr.RetryAfter > 0 { + classified.Cooldown = providerErr.RetryAfter + } } classified.RetryAfter = classified.Cooldown return classified diff --git a/internal/providers/devin/chat.go b/internal/providers/devin/chat.go index bb389e34..1f5327e8 100644 --- a/internal/providers/devin/chat.go +++ b/internal/providers/devin/chat.go @@ -107,7 +107,7 @@ func (c *Client) chatNonStreamOnce(client *http.Client, built chatRequestBuild, defer resp.Body.Close() if resp.StatusCode >= 300 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) - err := classifiedErrorWithToolsDiag(resp.StatusCode, string(body), built.toolsDiag) + err := classifiedErrorWithHeaders(resp.StatusCode, resp.Header, string(body), built.toolsDiag) return providers.ChatOutcome{}, isMCPConfigDenialError(err), err } aggregate, err := aggregateConnectStream(resp.Body, built.originalByAlias, built.toolsDiag) @@ -125,7 +125,7 @@ func (c *Client) chatStreamOnce(client *http.Client, built chatRequestBuild) (*h if resp.StatusCode >= 300 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) resp.Body.Close() - err := classifiedErrorWithToolsDiag(resp.StatusCode, string(body), built.toolsDiag) + err := classifiedErrorWithHeaders(resp.StatusCode, resp.Header, string(body), built.toolsDiag) return nil, isMCPConfigDenialError(err), err } // MCP configuration denials often arrive as an end-stream trailer on HTTP diff --git a/internal/providers/devin/devin_test.go b/internal/providers/devin/devin_test.go index 78c93f7c..62970d9e 100644 --- a/internal/providers/devin/devin_test.go +++ b/internal/providers/devin/devin_test.go @@ -165,6 +165,107 @@ func TestClassify401And429(t *testing.T) { } } +func TestDevinResetHintFromMessage(t *testing.T) { + now := time.Date(2026, 10, 8, 18, 0, 0, 0, time.UTC) + cases := []struct { + name string + msg string + want time.Duration + }{ + { + name: "minutes", + msg: "Reached free model rate limit. Your limit will reset in 13 minutes (at 18:13 UTC).", + want: 13 * time.Minute, + }, + { + name: "hours and minutes", + msg: "Reached free model rate limit. Your limit will reset in 5 hours 30 minutes (at 23:30 UTC).", + want: 5*time.Hour + 30*time.Minute, + }, + { + name: "seconds", + msg: "Your limit will reset in 52 seconds (at 18:01 UTC).", + want: 52 * time.Second, + }, + { + name: "single hour", + msg: "Your limit will reset in 1 hour (at 19:00 UTC).", + want: time.Hour, + }, + { + name: "no hint", + msg: "Your weekly usage quota has been exhausted.", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := resetHintFromMessage(tc.msg, now) + if got != tc.want { + t.Fatalf("reset hint = %v, want %v", got, tc.want) + } + }) + } +} + +func TestDevinResetHintAbsoluteRollsToNextDay(t *testing.T) { + now := time.Date(2026, 10, 8, 17, 55, 0, 0, time.UTC) + got := resetHintFromMessage("reset at unknown (at 18:01 UTC)", now) + if got != 6*time.Minute { + t.Fatalf("absolute hint = %v, want 6m", got) + } + later := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC) + got = resetHintFromMessage("(at 18:01 UTC)", later) + if got <= 23*time.Hour || got > 24*time.Hour { + t.Fatalf("rollover hint = %v, want just under 24h", got) + } +} + +func TestClassifyDevinRateLimitCarriesResetHint(t *testing.T) { + body := "devin upstream error (resource_exhausted): Reached free model rate limit. Your limit will reset in 13 minutes (at 18:13 UTC)." + err := classifiedErrorWithHeaders(http.StatusTooManyRequests, http.Header{}, body, "") + var providerErr *providers.Error + if !errors.As(err, &providerErr) { + t.Fatalf("error type = %T", err) + } + if providerErr.Kind != accounts.KindRateLimit || providerErr.RetryAfter != 13*time.Minute { + t.Fatalf("model rate limit = %+v, want rate_limit 13m", providerErr) + } + + header := http.Header{} + header.Set("Retry-After", "90") + err = classifiedErrorWithHeaders(http.StatusTooManyRequests, header, body, "") + if !errors.As(err, &providerErr) { + t.Fatalf("error type = %T", err) + } + if providerErr.RetryAfter != 90*time.Second { + t.Fatalf("Retry-After = %v, want 90s", providerErr.RetryAfter) + } +} + +func TestClassifyDevinUsageQuotaIsAccountScoped(t *testing.T) { + body := "devin upstream error (failed_precondition): Your weekly usage quota has been exhausted." + err := classifiedErrorWithToolsDiag(http.StatusTooManyRequests, body, "") + var providerErr *providers.Error + if !errors.As(err, &providerErr) { + t.Fatalf("error type = %T", err) + } + if providerErr.Kind != accounts.KindQuota { + t.Fatalf("usage quota kind = %s, want quota", providerErr.Kind) + } + if providerErr.RetryAfter != 0 { + t.Fatalf("quota without a reset hint must leave retry_after empty, got %v", providerErr.RetryAfter) + } + + hinted := "Your weekly usage quota has been exhausted. Your limit will reset in 9 minutes (at 18:09 UTC)." + err = classifiedErrorWithToolsDiag(http.StatusTooManyRequests, hinted, "") + if !errors.As(err, &providerErr) { + t.Fatalf("error type = %T", err) + } + if providerErr.Kind != accounts.KindQuota || providerErr.RetryAfter != 9*time.Minute { + t.Fatalf("quota reset = %+v, want quota 9m", providerErr) + } +} + type memStore struct { accounts map[string]accounts.Account creds map[string][]byte diff --git a/internal/providers/devin/errors.go b/internal/providers/devin/errors.go index 07483f6f..b3a1357b 100644 --- a/internal/providers/devin/errors.go +++ b/internal/providers/devin/errors.go @@ -2,8 +2,11 @@ package devin import ( "log" + "net/http" "regexp" + "strconv" "strings" + "time" "github.com/caigee-cmd/cli2api/internal/accounts" "github.com/caigee-cmd/cli2api/internal/providers" @@ -12,6 +15,11 @@ import ( var ( sessionTokenPattern = regexp.MustCompile(`(?i)devin-session-token\$[A-Za-z0-9._\-+/=]+`) jwtLikePattern = regexp.MustCompile(`\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b`) + // resetInPattern matches relative hints such as "reset in 13 minutes" or + // "reset in 5 hours 30 minutes". Devin puts the quota reset in the body and + // does not send Retry-After. + resetInPattern = regexp.MustCompile(`(?i)reset in\s+(?:(\d+)\s*hours?)?\s*(?:(\d+)\s*minutes?)?\s*(?:(\d+)\s*seconds?)?`) + resetAtPattern = regexp.MustCompile(`(?i)\(at\s+(\d{1,2}):(\d{2})\s*UTC\)`) ) func redactSecrets(text string) string { @@ -54,10 +62,7 @@ func Classify(status int, body string) providers.ClassifiedError { strings.Contains(text, "rate limit") || strings.Contains(text, "too many requests"): kind := accounts.KindRateLimit - if strings.Contains(text, "quota") || strings.Contains(text, "credit") || strings.Contains(text, "exhausted") && !strings.Contains(text, "resource_exhausted") { - kind = accounts.KindQuota - } - if strings.Contains(text, "resource_exhausted") && (strings.Contains(text, "quota") || strings.Contains(text, "credit") || strings.Contains(text, "acu")) { + if devinQuotaScoped(text) { kind = accounts.KindQuota } return providers.ClassifiedError{ @@ -94,6 +99,10 @@ func Classify(status int, body string) providers.ClassifiedError { } func classifiedErrorWithToolsDiag(status int, body, toolsDiag string) error { + return classifiedErrorWithHeaders(status, nil, body, toolsDiag) +} + +func classifiedErrorWithHeaders(status int, headers http.Header, body, toolsDiag string) error { classified := Classify(status, body) if classified.Kind == "" { classified = providers.ClassifiedError{ @@ -107,11 +116,89 @@ func classifiedErrorWithToolsDiag(status int, body, toolsDiag string) error { log.Printf("devin mcp configuration denial tools_diag=%s", toolsDiag) message = appendToolsDiag(message, toolsDiag) } - return &providers.Error{ + err := &providers.Error{ Kind: classified.Kind, Status: classified.Status, Message: message, } + if classified.Kind == accounts.KindRateLimit || classified.Kind == accounts.KindQuota { + err.RetryAfter = devinRetryAfter(headers, body, time.Now()) + } + return err +} + +// devinQuotaScoped reports an account-level quota exhaustion. A per-model rate +// limit stays model-scoped so the rest of the account keeps serving. +func devinQuotaScoped(text string) bool { + if strings.Contains(text, "usage quota") { + return true + } + if strings.Contains(text, "quota") || strings.Contains(text, "credit") || strings.Contains(text, "acu") { + return true + } + return strings.Contains(text, "exhausted") && !strings.Contains(text, "resource_exhausted") +} + +// devinRetryAfter prefers Retry-After, then the reset hint embedded in the +// body. Returns 0 when neither is present so the executor keeps its fallback. +func devinRetryAfter(headers http.Header, body string, now time.Time) time.Duration { + if delay := retryAfterHeader(headers, now); delay > 0 { + return delay + } + return resetHintFromMessage(body, now) +} + +func retryAfterHeader(headers http.Header, now time.Time) time.Duration { + if headers == nil { + return 0 + } + raw := strings.TrimSpace(headers.Get("Retry-After")) + if raw == "" { + return 0 + } + if seconds, err := strconv.ParseInt(raw, 10, 64); err == nil && seconds > 0 { + return time.Duration(seconds) * time.Second + } + if deadline, err := http.ParseTime(raw); err == nil { + if delay := deadline.Sub(now); delay > 0 { + return delay + } + } + return 0 +} + +func resetHintFromMessage(msg string, now time.Time) time.Duration { + if match := resetInPattern.FindStringSubmatch(msg); match != nil { + seconds := int64(0) + if value, err := strconv.ParseInt(match[1], 10, 64); err == nil { + seconds += value * 3600 + } + if value, err := strconv.ParseInt(match[2], 10, 64); err == nil { + seconds += value * 60 + } + if value, err := strconv.ParseInt(match[3], 10, 64); err == nil { + seconds += value + } + if seconds > 0 { + return time.Duration(seconds) * time.Second + } + } + match := resetAtPattern.FindStringSubmatch(msg) + if match == nil { + return 0 + } + hour, errH := strconv.Atoi(match[1]) + minute, errM := strconv.Atoi(match[2]) + if errH != nil || errM != nil || hour > 23 || minute > 59 { + return 0 + } + utc := now.UTC() + reset := time.Date(utc.Year(), utc.Month(), utc.Day(), hour, minute, 0, 0, time.UTC) + delay := reset.Sub(utc) + if delay <= 0 { + delay += 24 * time.Hour + } + return delay } func appendToolsDiag(message, toolsDiag string) string {