diff --git a/packages/adapter-utils/src/session-compaction.ts b/packages/adapter-utils/src/session-compaction.ts index 1de7f3d662c..af539820896 100644 --- a/packages/adapter-utils/src/session-compaction.ts +++ b/packages/adapter-utils/src/session-compaction.ts @@ -3,6 +3,7 @@ export interface SessionCompactionPolicy { maxSessionRuns: number; maxRawInputTokens: number; maxSessionAgeHours: number; + maxConsecutiveAdapterFailed: number; } export type NativeContextManagement = "confirmed" | "likely" | "unknown" | "none"; @@ -25,6 +26,7 @@ const DEFAULT_SESSION_COMPACTION_POLICY: SessionCompactionPolicy = { maxSessionRuns: 200, maxRawInputTokens: 2_000_000, maxSessionAgeHours: 72, + maxConsecutiveAdapterFailed: 0, }; // Adapters with native context management still participate in session resume, @@ -34,6 +36,7 @@ const ADAPTER_MANAGED_SESSION_POLICY: SessionCompactionPolicy = { maxSessionRuns: 0, maxRawInputTokens: 0, maxSessionAgeHours: 0, + maxConsecutiveAdapterFailed: 0, }; export const LEGACY_SESSIONED_ADAPTER_TYPES = new Set([ @@ -57,7 +60,10 @@ export const ADAPTER_SESSION_MANAGEMENT: Record) { - return policy.maxSessionRuns > 0 || policy.maxRawInputTokens > 0 || policy.maxSessionAgeHours > 0; + return ( + policy.maxSessionRuns > 0 || + policy.maxRawInputTokens > 0 || + policy.maxSessionAgeHours > 0 || + policy.maxConsecutiveAdapterFailed > 0 + ); } diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.ts index 067f68cdbce..ecb658a645d 100644 --- a/packages/adapters/claude-local/src/server/execute.ts +++ b/packages/adapters/claude-local/src/server/execute.ts @@ -958,6 +958,28 @@ export async function execute(ctx: AdapterExecutionContext): Promise { ).toBe(false); }); + it("classifies connection refused / ECONNREFUSED as transient", () => { + expect( + isClaudeTransientUpstreamError({ + stderr: "Error: connect ECONNREFUSED 127.0.0.1:3456", + }), + ).toBe(true); + expect( + isClaudeTransientUpstreamError({ + stderr: "Connection refused while trying to reach Claude Code server", + }), + ).toBe(true); + expect( + isClaudeTransientUpstreamError({ + errorMessage: "Claude Code CLI connection refused", + }), + ).toBe(true); + }); + it("does not classify deterministic validation errors as transient", () => { expect( isClaudeTransientUpstreamError({ diff --git a/packages/adapters/claude-local/src/server/parse.ts b/packages/adapters/claude-local/src/server/parse.ts index f645c4f27aa..d8aa7adb587 100644 --- a/packages/adapters/claude-local/src/server/parse.ts +++ b/packages/adapters/claude-local/src/server/parse.ts @@ -10,7 +10,7 @@ const CLAUDE_AUTH_REQUIRED_RE = /(?:not\s+logged\s+in|please\s+log\s+in|please\s const URL_RE = /(https?:\/\/[^\s'"`<>()[\]{};,!?]+[^\s'"`<>()[\]{};,!.?:]+)/gi; const CLAUDE_TRANSIENT_UPSTREAM_RE = - /(?:rate[-\s]?limit(?:ed)?|rate_limit_error|too\s+many\s+requests|\b429\b|overloaded(?:_error)?|server\s+overloaded|service\s+unavailable|\b503\b|\b529\b|high\s+demand|try\s+again\s+later|temporarily\s+unavailable|throttl(?:ed|ing)|throttlingexception|servicequotaexceededexception|out\s+of\s+extra\s+usage|extra\s+usage\b|claude\s+usage\s+limit\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|usage\s+limit\s+reached|usage\s+cap\s+reached)/i; + /(?:rate[-\s]?limit(?:ed)?|rate_limit_error|too\s+many\s+requests|\b429\b|overloaded(?:_error)?|server\s+overloaded|service\s+unavailable|\b503\b|\b529\b|high\s+demand|try\s+again\s+later|temporarily\s+unavailable|throttl(?:ed|ing)|throttlingexception|servicequotaexceededexception|out\s+of\s+extra\s+usage|extra\s+usage\b|claude\s+usage\s+limit\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|usage\s+limit\s+reached|usage\s+cap\s+reached|connection\s+refused|econnrefused|connect\s+error)/i; const CLAUDE_EXTRA_USAGE_RESET_RE = /(?:out\s+of\s+extra\s+usage|extra\s+usage|usage\s+limit\s+reached|usage\s+cap\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|claude\s+usage\s+limit\s+reached)[\s\S]{0,80}?\bresets?\s+(?:at\s+)?([^\n()]+?)(?:\s*\(([^)]+)\))?(?:[.!]|\n|$)/i; diff --git a/server/src/__tests__/heartbeat-session-compaction.test.ts b/server/src/__tests__/heartbeat-session-compaction.test.ts new file mode 100644 index 00000000000..c390cdb07ea --- /dev/null +++ b/server/src/__tests__/heartbeat-session-compaction.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it } from "vitest"; +import type { agents } from "@paperclipai/db"; +import { + parseSessionCompactionPolicy, + evaluateSessionCompactionFromRuns, + type SessionCompactionDecision, +} from "../services/heartbeat.ts"; + +function buildAgent(adapterType: string, runtimeConfig: Record = {}) { + return { + id: "agent-1", + companyId: "company-1", + projectId: null, + goalId: null, + name: "Agent", + role: "engineer", + title: null, + icon: null, + status: "running", + reportsTo: null, + capabilities: null, + adapterType, + adapterConfig: {}, + runtimeConfig, + budgetMonthlyCents: 0, + spentMonthlyCents: 0, + permissions: {}, + lastHeartbeatAt: null, + metadata: null, + createdAt: new Date(), + updatedAt: new Date(), + } as unknown as typeof agents.$inferSelect; +} + +function buildRun(overrides: { + id?: string; + createdAt?: Date; + resultJson?: Record | null; + errorCode?: string | null; + error?: string | null; +} = {}) { + return { + id: overrides.id ?? "run-1", + createdAt: overrides.createdAt ?? new Date("2026-06-30T12:00:00.000Z"), + usageJson: null, + error: overrides.error ?? null, + resultSummary: null, + resultResult: null, + resultMessage: null, + resultError: null, + resultTotalCostUsd: null, + resultCostUsd: null, + resultCostUsdCamel: null, + resultJson: overrides.resultJson ?? null, + errorCode: overrides.errorCode ?? null, + }; +} + +describe("parseSessionCompactionPolicy maxConsecutiveAdapterFailed", () => { + it("defaults claude_local to 2 consecutive adapter_failed rotations", () => { + expect(parseSessionCompactionPolicy(buildAgent("claude_local"))).toMatchObject({ + maxConsecutiveAdapterFailed: 2, + }); + }); + + it("defaults other adapters to 0", () => { + expect(parseSessionCompactionPolicy(buildAgent("codex_local"))).toMatchObject({ + maxConsecutiveAdapterFailed: 0, + }); + expect(parseSessionCompactionPolicy(buildAgent("cursor"))).toMatchObject({ + maxConsecutiveAdapterFailed: 0, + }); + }); + + it("allows agent runtime config override", () => { + expect( + parseSessionCompactionPolicy( + buildAgent("claude_local", { + heartbeat: { + sessionCompaction: { + maxConsecutiveAdapterFailed: 5, + }, + }, + }), + ), + ).toMatchObject({ + maxConsecutiveAdapterFailed: 5, + }); + }); +}); + +function evaluateFromRuns( + agent: ReturnType, + runs: ReturnType[], +): SessionCompactionDecision { + const policy = parseSessionCompactionPolicy(agent); + return evaluateSessionCompactionFromRuns({ + agent, + sessionId: "session-1", + issueId: "issue-1", + policy, + runs, + oldestRun: runs[runs.length - 1] ?? null, + }); +} + +describe("evaluateSessionCompaction consecutive adapter_failed rule", () => { + it("does not rotate when threshold is 0", () => { + const agent = buildAgent("codex_local"); + const decision = evaluateFromRuns(agent, [ + buildRun({ resultJson: { stopReason: "adapter_failed" }, errorCode: "adapter_failed" }), + buildRun({ resultJson: { stopReason: "adapter_failed" }, errorCode: "adapter_failed" }), + buildRun({ resultJson: { stopReason: "adapter_failed" }, errorCode: "adapter_failed" }), + ]); + expect(decision.rotate).toBe(false); + }); + + it("does not rotate when there are fewer consecutive adapter_failed runs than threshold", () => { + const agent = buildAgent("claude_local"); + const decision = evaluateFromRuns(agent, [ + buildRun({ resultJson: { stopReason: "adapter_failed" }, errorCode: "adapter_failed" }), + ]); + expect(decision.rotate).toBe(false); + }); + + it("rotates when 2 consecutive adapter_failed runs occur for claude_local", () => { + const agent = buildAgent("claude_local"); + const decision = evaluateFromRuns(agent, [ + buildRun({ + id: "run-2", + createdAt: new Date("2026-06-30T12:01:00.000Z"), + resultJson: { stopReason: "adapter_failed" }, + errorCode: "adapter_failed", + }), + buildRun({ + id: "run-1", + createdAt: new Date("2026-06-30T12:00:00.000Z"), + resultJson: { stopReason: "adapter_failed" }, + errorCode: "adapter_failed", + }), + ]); + expect(decision.rotate).toBe(true); + expect(decision.reason).toContain("2 consecutive adapter_failed"); + expect(decision.previousRunId).toBe("run-2"); + }); + + it("rotates when consecutive runs include transient_upstream error family", () => { + const agent = buildAgent("claude_local"); + const decision = evaluateFromRuns(agent, [ + buildRun({ + resultJson: { stopReason: "adapter_failed", errorFamily: "transient_upstream" }, + errorCode: "claude_transient_upstream", + }), + buildRun({ + resultJson: { stopReason: "adapter_failed", errorFamily: "transient_upstream" }, + errorCode: "claude_transient_upstream", + }), + ]); + expect(decision.rotate).toBe(true); + }); + + it("stops counting at the first non-adapter-failed run", () => { + const agent = buildAgent("claude_local"); + const decision = evaluateFromRuns(agent, [ + buildRun({ resultJson: { stopReason: "adapter_failed" }, errorCode: "adapter_failed" }), + buildRun({ resultJson: { stopReason: "completed" }, errorCode: null }), + buildRun({ resultJson: { stopReason: "adapter_failed" }, errorCode: "adapter_failed" }), + ]); + expect(decision.rotate).toBe(false); + }); + + it("does not rotate when the latest run succeeded", () => { + const agent = buildAgent("claude_local"); + const decision = evaluateFromRuns(agent, [ + buildRun({ resultJson: { stopReason: "completed" }, errorCode: null }), + buildRun({ resultJson: { stopReason: "adapter_failed" }, errorCode: "adapter_failed" }), + buildRun({ resultJson: { stopReason: "adapter_failed" }, errorCode: "adapter_failed" }), + ]); + expect(decision.rotate).toBe(false); + }); + + it("falls back to errorCode when stopReason is missing", () => { + const agent = buildAgent("claude_local"); + const decision = evaluateFromRuns(agent, [ + buildRun({ resultJson: {}, errorCode: "adapter_failed" }), + buildRun({ resultJson: {}, errorCode: "adapter_failed" }), + ]); + expect(decision.rotate).toBe(true); + }); +}); diff --git a/server/src/__tests__/heartbeat-workspace-session.test.ts b/server/src/__tests__/heartbeat-workspace-session.test.ts index 3c74475ec95..897f61c4c0f 100644 --- a/server/src/__tests__/heartbeat-workspace-session.test.ts +++ b/server/src/__tests__/heartbeat-workspace-session.test.ts @@ -527,12 +527,14 @@ describe("parseSessionCompactionPolicy", () => { maxSessionRuns: 0, maxRawInputTokens: 0, maxSessionAgeHours: 0, + maxConsecutiveAdapterFailed: 0, }); expect(parseSessionCompactionPolicy(buildAgent("claude_local"))).toEqual({ enabled: true, maxSessionRuns: 0, maxRawInputTokens: 0, maxSessionAgeHours: 0, + maxConsecutiveAdapterFailed: 2, }); }); @@ -542,12 +544,14 @@ describe("parseSessionCompactionPolicy", () => { maxSessionRuns: 200, maxRawInputTokens: 2_000_000, maxSessionAgeHours: 72, + maxConsecutiveAdapterFailed: 0, }); expect(parseSessionCompactionPolicy(buildAgent("opencode_local"))).toEqual({ enabled: true, maxSessionRuns: 200, maxRawInputTokens: 2_000_000, maxSessionAgeHours: 72, + maxConsecutiveAdapterFailed: 0, }); }); @@ -559,6 +563,7 @@ describe("parseSessionCompactionPolicy", () => { sessionCompaction: { maxSessionRuns: 25, maxRawInputTokens: 500_000, + maxConsecutiveAdapterFailed: 3, }, }, }), @@ -568,6 +573,7 @@ describe("parseSessionCompactionPolicy", () => { maxSessionRuns: 25, maxRawInputTokens: 500_000, maxSessionAgeHours: 0, + maxConsecutiveAdapterFailed: 3, }); }); }); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 8c34e99233a..7af85485ee7 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1504,6 +1504,146 @@ export function parseSessionCompactionPolicy(agent: typeof agents.$inferSelect): return resolveSessionCompactionPolicy(agent.adapterType, agent.runtimeConfig).policy; } + +function countConsecutiveAdapterFailedRuns( + runs: Array<{ + resultJson?: Record | null; + errorCode?: string | null; + }>, +): number { + let count = 0; + for (const run of runs) { + const resultJson = run.resultJson ?? {}; + const stopReason = String(resultJson.stopReason ?? "").toLowerCase(); + const errorFamily = String(resultJson.errorFamily ?? "").toLowerCase(); + const errorCode = (run.errorCode ?? "").toLowerCase(); + const isAdapterFailed = + stopReason === "adapter_failed" || + errorCode === "adapter_failed" || + errorFamily === "transient_upstream"; + if (isAdapterFailed) { + count += 1; + } else { + break; + } + } + return count; +} + +export function evaluateSessionCompactionFromRuns(input: { + agent: typeof agents.$inferSelect; + sessionId: string; + issueId: string | null; + policy: SessionCompactionPolicy; + runs: Array<{ + id: string; + createdAt: Date; + usageJson: Record | null; + error: string | null; + errorCode?: string | null; + resultJson?: Record | null; + resultSummary?: string | null; + resultResult?: string | null; + resultMessage?: string | null; + resultError?: string | null; + resultTotalCostUsd?: string | null; + resultCostUsd?: string | null; + resultCostUsdCamel?: string | null; + }>; + oldestRun: { + id: string; + createdAt: Date; + usageJson: Record | null; + error: string | null; + } | null; + continuationSummaryBody?: string | null; +}): SessionCompactionDecision { + const { agent, sessionId, issueId, policy, runs, oldestRun } = input; + const latestRun = runs[0] ?? null; + if (!latestRun) { + return { + rotate: false, + reason: null, + handoffMarkdown: null, + previousRunId: null, + }; + } + + const latestRawUsage = readRawUsageTotals(latestRun.usageJson); + const sessionAgeHours = + oldestRun + ? Math.max( + 0, + (new Date(latestRun.createdAt).getTime() - new Date(oldestRun.createdAt).getTime()) / (1000 * 60 * 60), + ) + : 0; + + let reason: string | null = null; + if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) { + reason = `session exceeded ${policy.maxSessionRuns} runs`; + } else if ( + policy.maxRawInputTokens > 0 && + latestRawUsage && + latestRawUsage.inputTokens >= policy.maxRawInputTokens + ) { + reason = + `session raw input reached ${formatCount(latestRawUsage.inputTokens)} tokens ` + + `(threshold ${formatCount(policy.maxRawInputTokens)})`; + } else if (policy.maxSessionAgeHours > 0 && sessionAgeHours >= policy.maxSessionAgeHours) { + reason = `session age reached ${Math.floor(sessionAgeHours)} hours`; + } else if (policy.maxConsecutiveAdapterFailed > 0) { + const consecutive = countConsecutiveAdapterFailedRuns(runs); + if (consecutive >= policy.maxConsecutiveAdapterFailed) { + reason = `session reached ${consecutive} consecutive adapter_failed runs`; + } + } + + if (!reason) { + return { + rotate: false, + reason: null, + handoffMarkdown: null, + previousRunId: latestRun.id, + }; + } + + const latestSummary = summarizeHeartbeatRunListResultJson({ + summary: latestRun.resultSummary, + result: latestRun.resultResult, + message: latestRun.resultMessage, + error: latestRun.resultError, + totalCostUsd: latestRun.resultTotalCostUsd, + costUsd: latestRun.resultCostUsd, + costUsdCamel: latestRun.resultCostUsdCamel, + }); + const latestTextSummary = + readNonEmptyString(latestSummary?.summary) ?? + readNonEmptyString(latestSummary?.result) ?? + readNonEmptyString(latestSummary?.message) ?? + readNonEmptyString(latestRun.error); + + const handoffMarkdown = [ + "Paperclip session handoff:", + `- Previous session: ${sessionId}`, + issueId ? `- Issue: ${issueId}` : "", + `- Rotation reason: ${reason}`, + latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "", + input.continuationSummaryBody + ? `- Issue continuation summary: ${input.continuationSummaryBody.slice(0, 1_500)}` + : "", + "Continue from the current task state. Rebuild only the minimum context you need.", + ] + .filter(Boolean) + .join("\n"); + + return { + rotate: true, + reason, + handoffMarkdown, + previousRunId: latestRun.id, + }; +} + export function resolveRuntimeSessionParamsForWorkspace(input: { agentId: string; previousSessionParams: Record | null; @@ -3293,6 +3433,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .select({ id: heartbeatRuns.id, createdAt: heartbeatRuns.createdAt, + usageJson: heartbeatRuns.usageJson, + error: heartbeatRuns.error, }) .from(heartbeatRuns) .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.sessionIdAfter, sessionId))) @@ -3358,6 +3500,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) createdAt: heartbeatRuns.createdAt, usageJson: heartbeatRuns.usageJson, error: heartbeatRuns.error, + errorCode: heartbeatRuns.errorCode, ...heartbeatRunListResultColumns, }) .from(heartbeatRuns) @@ -3374,79 +3517,20 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; } - const latestRun = runs[0] ?? null; const oldestRun = policy.maxSessionAgeHours > 0 ? await getOldestRunForSession(agent.id, sessionId) - : runs[runs.length - 1] ?? latestRun; - const latestRawUsage = readRawUsageTotals(latestRun?.usageJson); - const sessionAgeHours = - latestRun && oldestRun - ? Math.max( - 0, - (new Date(latestRun.createdAt).getTime() - new Date(oldestRun.createdAt).getTime()) / (1000 * 60 * 60), - ) - : 0; - - let reason: string | null = null; - if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) { - reason = `session exceeded ${policy.maxSessionRuns} runs`; - } else if ( - policy.maxRawInputTokens > 0 && - latestRawUsage && - latestRawUsage.inputTokens >= policy.maxRawInputTokens - ) { - reason = - `session raw input reached ${formatCount(latestRawUsage.inputTokens)} tokens ` + - `(threshold ${formatCount(policy.maxRawInputTokens)})`; - } else if (policy.maxSessionAgeHours > 0 && sessionAgeHours >= policy.maxSessionAgeHours) { - reason = `session age reached ${Math.floor(sessionAgeHours)} hours`; - } + : runs[runs.length - 1] ?? runs[0]; - if (!reason || !latestRun) { - return { - rotate: false, - reason: null, - handoffMarkdown: null, - previousRunId: latestRun?.id ?? null, - }; - } - - const latestSummary = summarizeHeartbeatRunListResultJson({ - summary: latestRun?.resultSummary, - result: latestRun?.resultResult, - message: latestRun?.resultMessage, - error: latestRun?.resultError, - totalCostUsd: latestRun?.resultTotalCostUsd, - costUsd: latestRun?.resultCostUsd, - costUsdCamel: latestRun?.resultCostUsdCamel, + return evaluateSessionCompactionFromRuns({ + agent, + sessionId, + issueId, + policy, + runs, + oldestRun, + continuationSummaryBody: input.continuationSummaryBody, }); - const latestTextSummary = - readNonEmptyString(latestSummary?.summary) ?? - readNonEmptyString(latestSummary?.result) ?? - readNonEmptyString(latestSummary?.message) ?? - readNonEmptyString(latestRun.error); - - const handoffMarkdown = [ - "Paperclip session handoff:", - `- Previous session: ${sessionId}`, - issueId ? `- Issue: ${issueId}` : "", - `- Rotation reason: ${reason}`, - latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "", - input.continuationSummaryBody - ? `- Issue continuation summary: ${input.continuationSummaryBody.slice(0, 1_500)}` - : "", - "Continue from the current task state. Rebuild only the minimum context you need.", - ] - .filter(Boolean) - .join("\n"); - - return { - rotate: true, - reason, - handoffMarkdown, - previousRunId: latestRun.id, - }; } async function resolveSessionBeforeForWakeup(