diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 58a4e092b6a..efc4f2e267c 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -32,6 +32,8 @@ import { issueTreeHolds, issueWorkProducts, issues, + projects, + projectWorkspaces, workspaceOperations, } from "@paperclipai/db"; import { @@ -369,6 +371,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } await db.delete(agentWakeupRequests); await db.delete(budgetPolicies); + await db.delete(projectWorkspaces); + await db.delete(projects); for (let attempt = 0; attempt < 5; attempt += 1) { await db.delete(agentRuntimeState); try { @@ -924,6 +928,118 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { return { companyId, agentId, runId, wakeupRequestId, issueId }; } + it("blocks git-backed project workspace routing failures without invoking the adapter", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + const projectId = randomUUID(); + const projectWorkspaceId = randomUUID(); + const issueId = randomUUID(); + const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "CodexCoder", + role: "engineer", + status: "idle", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "paperclip", + status: "in_progress", + }); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "primary", + sourceType: "git_repo", + cwd: null, + repoUrl: `file:///tmp/paperclip-missing-${randomUUID()}`, + repoRef: "master", + isPrimary: true, + }); + await db.insert(issues).values({ + id: issueId, + companyId, + projectId, + projectWorkspaceId, + title: "Project-bound implementation", + status: "todo", + priority: "medium", + assigneeAgentId: agentId, + assigneeUserId: null, + issueNumber: 1, + identifier: `${issuePrefix}-1`, + }); + + const heartbeat = heartbeatService(db); + const run = await heartbeat.invoke(agentId, "assignment", { + issueId, + taskId: issueId, + wakeReason: "issue_assigned", + }, "system", { actorType: "system", actorId: "test" }); + expect(run).toBeTruthy(); + + await waitForHeartbeatIdle(db, 10_000); + + expect(mockAdapterExecute).not.toHaveBeenCalled(); + const failedRun = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, run!.id)) + .then((rows) => rows[0] ?? null); + expect(failedRun).toMatchObject({ + status: "failed", + errorCode: "workspace_routing_failed", + }); + + const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(sourceIssue).toMatchObject({ + status: "blocked", + executionRunId: null, + assigneeAgentId: agentId, + }); + + const [action] = await db + .select() + .from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(action).toMatchObject({ + kind: "stranded_assigned_issue", + status: "active", + ownerAgentId: agentId, + cause: "workspace_routing_failed", + attemptCount: 1, + wakePolicy: { + type: "manual_unblock", + reason: "workspace_routing_failed", + }, + }); + expect(action?.evidence).toMatchObject({ + latestRunId: run!.id, + latestRunErrorCode: "workspace_routing_failed", + workspaceId: projectWorkspaceId, + }); + + const recoveryWakeups = await db + .select() + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.reason, "source_scoped_recovery_action")); + expect(recoveryWakeups).toHaveLength(0); + }); + it("keeps a local run active when the recorded pid is still alive", async () => { const child = spawnAliveProcess(); childProcesses.add(child); diff --git a/server/src/__tests__/heartbeat-workspace-session.test.ts b/server/src/__tests__/heartbeat-workspace-session.test.ts index 47282db0d1d..089c72d48eb 100644 --- a/server/src/__tests__/heartbeat-workspace-session.test.ts +++ b/server/src/__tests__/heartbeat-workspace-session.test.ts @@ -1,8 +1,14 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; import { describe, expect, it } from "vitest"; import type { agents } from "@paperclipai/db"; import { sessionCodec as codexSessionCodec } from "@paperclipai/adapter-codex-local/server"; import { resolveDefaultAgentWorkspaceDir } from "../home-paths.js"; import { + assertGitBackedProjectWorkspace, applyPersistedExecutionWorkspaceConfig, buildRealizedExecutionWorkspaceFromPersisted, buildExplicitResumeSessionOverride, @@ -16,9 +22,12 @@ import { resolveRuntimeSessionParamsForWorkspace, stripWorkspaceRuntimeFromExecutionRunConfig, shouldResetTaskSessionForWake, + WorkspaceRoutingError, type ResolvedWorkspaceForRun, } from "../services/heartbeat.ts"; +const execFile = promisify(execFileCallback); + function buildResolvedWorkspace(overrides: Partial = {}): ResolvedWorkspaceForRun { return { cwd: "/tmp/project", @@ -59,6 +68,79 @@ function buildAgent(adapterType: string, runtimeConfig: Record } as unknown as typeof agents.$inferSelect; } +describe("assertGitBackedProjectWorkspace", () => { + it("accepts a git-backed project workspace when cwd is a git checkout", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-git-workspace-")); + await execFile("git", ["init"], { cwd: dir }); + + await expect(assertGitBackedProjectWorkspace({ + workspace: { + id: "workspace-1", + name: "Paperclip", + sourceType: "git_repo", + repoUrl: "https://github.com/paperclipai/paperclip", + }, + cwd: dir, + })).resolves.toBeUndefined(); + }); + + it("rejects a git-backed project workspace when cwd is missing", async () => { + await expect(assertGitBackedProjectWorkspace({ + workspace: { + id: "workspace-1", + sourceType: "git_repo", + repoUrl: "https://github.com/paperclipai/paperclip", + }, + cwd: path.join(os.tmpdir(), "paperclip-missing-workspace"), + })).rejects.toThrow(/checkout path is not available/); + await expect(assertGitBackedProjectWorkspace({ + workspace: { + id: "workspace-1", + sourceType: "git_repo", + repoUrl: "https://github.com/paperclipai/paperclip", + }, + cwd: path.join(os.tmpdir(), "paperclip-missing-workspace"), + })).rejects.toMatchObject({ + code: "workspace_routing_failed", + workspaceId: "workspace-1", + }); + }); + + it("rejects a repoUrl-backed project workspace when cwd is not a git checkout", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-non-git-workspace-")); + + await expect(assertGitBackedProjectWorkspace({ + workspace: { + id: "workspace-1", + sourceType: "local_path", + repoUrl: "https://github.com/paperclipai/paperclip", + }, + cwd: dir, + })).rejects.toThrow(/not a git repository/); + await expect(assertGitBackedProjectWorkspace({ + workspace: { + id: "workspace-1", + sourceType: "local_path", + repoUrl: "https://github.com/paperclipai/paperclip", + }, + cwd: dir, + })).rejects.toBeInstanceOf(WorkspaceRoutingError); + }); + + it("allows non-git local_path project workspaces", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-local-workspace-")); + + await expect(assertGitBackedProjectWorkspace({ + workspace: { + id: "workspace-1", + sourceType: "local_path", + repoUrl: null, + }, + cwd: dir, + })).resolves.toBeUndefined(); + }); +}); + describe("resolveRuntimeSessionParamsForWorkspace", () => { it("migrates fallback workspace sessions to project workspace when project cwd becomes available", () => { const agentId = "agent-123"; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 09a1e5fe7cb..56ddfbc83a2 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -150,6 +150,7 @@ import { withRecoveryModelProfileHint, } from "./recovery/model-profile-hint.js"; import { recoveryService } from "./recovery/service.js"; +import { issueRecoveryActionService } from "./issue-recovery-actions.js"; import { productivityReviewService } from "./productivity-review.js"; import { withAgentStartLock } from "./agent-start-lock.js"; import { @@ -228,6 +229,7 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0.25; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; +const WORKSPACE_ROUTING_ERROR_CODE = "workspace_routing_failed"; export const MAX_TURN_CONTINUATION_RETRY_REASON = "max_turns_continuation"; export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry"; const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2; @@ -1112,6 +1114,85 @@ function readNonEmptyString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value : null; } +export class WorkspaceRoutingError extends Error { + readonly code = WORKSPACE_ROUTING_ERROR_CODE; + readonly workspaceId: string; + readonly cwd: string | null; + + constructor(message: string, input: { workspaceId: string; cwd?: string | null; cause?: unknown }) { + super(message, input.cause === undefined ? undefined : { cause: input.cause }); + this.name = "WorkspaceRoutingError"; + this.workspaceId = input.workspaceId; + this.cwd = readNonEmptyString(input.cwd); + } +} + +export function isGitBackedProjectWorkspace(input: { + sourceType?: string | null; + repoUrl?: string | null; +}): boolean { + return input.sourceType === "git_repo" || Boolean(readNonEmptyString(input.repoUrl)); +} + +function describeProjectWorkspaceForError(input: { + id: string; + name?: string | null; + cwd?: string | null; +}): string { + const name = readNonEmptyString(input.name); + const cwd = readNonEmptyString(input.cwd); + const parts = [`"${input.id}"`]; + if (name) parts.push(`(${name})`); + if (cwd) parts.push(`at "${cwd}"`); + return parts.join(" "); +} + +export async function assertGitBackedProjectWorkspace(input: { + workspace: { + id: string; + name?: string | null; + sourceType?: string | null; + repoUrl?: string | null; + }; + cwd: string | null; +}): Promise { + if (!isGitBackedProjectWorkspace(input.workspace)) return; + + const cwd = readNonEmptyString(input.cwd); + const workspaceLabel = describeProjectWorkspaceForError({ + id: input.workspace.id, + name: input.workspace.name, + cwd, + }); + + if (!cwd) { + throw new WorkspaceRoutingError( + `Project workspace ${workspaceLabel} is git-backed but has no local checkout path. Restore or configure the project workspace checkout before running this issue.`, + { workspaceId: input.workspace.id, cwd }, + ); + } + + const stats = await fs.stat(cwd).catch(() => null); + if (!stats?.isDirectory()) { + throw new WorkspaceRoutingError( + `Project workspace ${workspaceLabel} is git-backed but its checkout path is not available. Restore or clone the project workspace before running this issue.`, + { workspaceId: input.workspace.id, cwd }, + ); + } + + const gitCheck = await execFile("git", ["rev-parse", "--is-inside-work-tree"], { + cwd, + env: sanitizeRuntimeServiceBaseEnv(process.env), + timeout: 10_000, + }).catch(() => null); + if (gitCheck?.stdout.trim() !== "true") { + throw new WorkspaceRoutingError( + `Project workspace ${workspaceLabel} is git-backed but its checkout path is not a git repository. Restore or clone the project workspace before running this issue.`, + { workspaceId: input.workspace.id, cwd }, + ); + } +} + function readModelProfileKey(value: unknown): ModelProfileKey | null { return MODEL_PROFILE_KEYS.includes(value as ModelProfileKey) ? (value as ModelProfileKey) @@ -2526,6 +2607,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) environmentRuntime, }); const workspaceOperationsSvc = workspaceOperationService(db); + const recoveryActionsSvc = issueRecoveryActionService(db); const activeRunExecutions = new Set(); const budgetHooks = { cancelWorkForScope: cancelBudgetScopeWork, @@ -3736,6 +3818,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) projectCwd = managedWorkspace.cwd; managedWorkspaceWarning = managedWorkspace.warning; } catch (error) { + if (isGitBackedProjectWorkspace(workspace)) { + throw new WorkspaceRoutingError( + `Project workspace ${describeProjectWorkspaceForError({ + id: workspace.id, + name: workspace.name, + cwd: projectCwd, + })} is git-backed but Paperclip could not prepare its checkout. Restore or clone the project workspace before running this issue. ${ + error instanceof Error ? error.message : String(error) + }`, + { workspaceId: workspace.id, cwd: projectCwd, cause: error }, + ); + } if (preferredWorkspace?.id === workspace.id) { preferredWorkspaceWarning = error instanceof Error ? error.message : String(error); } @@ -3743,6 +3837,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } } hasConfiguredProjectCwd = true; + await assertGitBackedProjectWorkspace({ + workspace, + cwd: projectCwd, + }); const projectCwdExists = await fs .stat(projectCwd) .then((stats) => stats.isDirectory()) @@ -8484,15 +8582,116 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun). // The inner catch did not fire, so we must record the failure here. const message = outerErr instanceof Error ? outerErr.message : "Unknown setup failure"; + const workspaceRoutingError = outerErr instanceof WorkspaceRoutingError ? outerErr : null; + const errorCode = workspaceRoutingError?.code ?? "adapter_failed"; logger.error({ err: outerErr, runId }, "heartbeat execution setup failed"); const setupFailureAgent = await getAgent(run.agentId).catch(() => null); + let workspaceRoutingHandled = false; + if (workspaceRoutingError) { + const runContext = parseObject(run.contextSnapshot); + const contextIssueId = readNonEmptyString(runContext.issueId) ?? readNonEmptyString(runContext.taskId); + const sourceIssue = contextIssueId + ? await db + .select() + .from(issues) + .where(and(eq(issues.id, contextIssueId), eq(issues.companyId, run.companyId))) + .then((rows) => rows[0] ?? null) + .catch(() => null) + : null; + if (sourceIssue) { + const recoveryAction = await recoveryActionsSvc.upsertSourceScoped({ + companyId: sourceIssue.companyId, + sourceIssueId: sourceIssue.id, + kind: "stranded_assigned_issue", + ownerType: sourceIssue.assigneeAgentId ? "agent" : "board", + ownerAgentId: sourceIssue.assigneeAgentId, + previousOwnerAgentId: sourceIssue.assigneeAgentId, + returnOwnerAgentId: sourceIssue.assigneeAgentId, + cause: WORKSPACE_ROUTING_ERROR_CODE, + fingerprint: [ + "workspace_routing", + sourceIssue.companyId, + sourceIssue.id, + workspaceRoutingError.workspaceId, + ].join(":"), + evidence: { + sourceIssueId: sourceIssue.id, + sourceIdentifier: sourceIssue.identifier, + latestRunId: run.id, + latestRunStatus: "failed", + latestRunErrorCode: errorCode, + workspaceId: workspaceRoutingError.workspaceId, + cwd: workspaceRoutingError.cwd, + }, + nextAction: "Restore or clone the selected git-backed project workspace checkout, then return the source issue to todo.", + wakePolicy: { + type: "manual_unblock", + reason: WORKSPACE_ROUTING_ERROR_CODE, + }, + monitorPolicy: null, + maxAttempts: null, + lastAttemptAt: new Date(), + }); + const updated = await db + .update(issues) + .set({ + status: "blocked", + assigneeAgentId: sourceIssue.assigneeAgentId, + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where(and(eq(issues.id, sourceIssue.id), eq(issues.companyId, sourceIssue.companyId))) + .returning() + .then((rows) => rows[0] ?? null); + workspaceRoutingHandled = Boolean(updated); + if (updated) { + await logActivity(db, { + companyId: sourceIssue.companyId, + actorType: "system", + actorId: "system", + agentId: run.agentId, + runId, + action: "issue.updated", + entityType: "issue", + entityId: sourceIssue.id, + details: { + identifier: sourceIssue.identifier, + status: "blocked", + previousStatus: sourceIssue.status, + source: "workspace.routing_failure", + recoveryActionId: recoveryAction.id, + workspaceId: workspaceRoutingError.workspaceId, + }, + }); + } + if (updated && recoveryAction.attemptCount === 1) { + await issuesSvc.addComment(sourceIssue.id, [ + "Paperclip stopped this run before adapter execution because the selected project workspace is git-backed but not usable.", + "", + `- Failure: ${message}`, + `- Recovery action: \`${recoveryAction.id}\``, + `- Workspace id: \`${workspaceRoutingError.workspaceId}\``, + `- Workspace cwd: \`${workspaceRoutingError.cwd ?? "none"}\``, + "- Next action: restore or clone the project workspace checkout, then return this issue to `todo`.", + ].join("\n"), {}, { authorType: "system" }).catch((commentErr) => { + logger.warn( + { err: commentErr, runId, issueId: sourceIssue.id, recoveryActionId: recoveryAction.id }, + "failed to write workspace routing recovery comment", + ); + }); + } + } + } await setRunStatus(runId, "failed", { error: message, - errorCode: "adapter_failed", + errorCode, finishedAt: new Date(), ...(setupFailureAgent ? { resultJson: mergeRunStopMetadataForAgent(setupFailureAgent, "failed", { - errorCode: "adapter_failed", + errorCode, errorMessage: message, }), } : {}), @@ -8515,9 +8714,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const failedAgent = setupFailureAgent ?? await getAgent(run.agentId).catch(() => null); if (failedAgent) { await refreshContinuationSummaryForRun(livenessRun, failedAgent).catch(() => undefined); - await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(() => undefined); } - await releaseIssueExecutionAndPromote(livenessRun).catch(() => undefined); + if (workspaceRoutingError) { + if (!workspaceRoutingHandled) { + await releaseIssueExecutionAndPromote(livenessRun).catch(() => undefined); + } + } else { + if (failedAgent) { + await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(() => undefined); + } + await releaseIssueExecutionAndPromote(livenessRun).catch(() => undefined); + } } // Ensure the agent is not left stuck in "running" if the inner catch handler's // DB calls threw (e.g. a transient DB error in finalizeAgentStatus).