fix(mothership): re-send an undelivered workflow tool completion while another run owns the workflow

A run whose completion report failed keeps it in session storage for recovery to re-send.
Recovery refused to touch the call whenever another run tool owned the workflow, which the
queue now makes common, so the stored report was never delivered. Re-sending a stored
report needs nothing from the workflow; recovery now does it first, and only skips the
pointer lookup and cleanup that belong to the run that owns the workflow.

Also makes the abandoned-reconnect test track the running flag, so it fails without the
stale-state clearing it covers.

Author: waleedlatif1

apps/sim/lib/mothership/tools/client/run-tool-execution.test.ts

@@ -840,6 +840,18 @@ describe('run tool execution cancellation', () => {
       )
       let reconnectOpen = true
       isReconnectStreamOpen.mockImplementation(() => reconnectOpen)
+      const running = new Map<string, boolean>()
+      setIsExecuting.mockImplementation((workflowId: string, value: boolean) => {
+        running.set(workflowId, value)
+      })
+      getWorkflowExecution.mockImplementation((workflowId: string) => ({
+        isExecuting: running.get(workflowId) ?? false,
+      }))
+      onTestFinished(() => {
+        setIsExecuting.mockReset()
+        getWorkflowExecution.mockReset()
+        getWorkflowExecution.mockImplementation(() => ({ isExecuting: false }))
+      })
       executeWorkflowWithFullLogging.mockRejectedValueOnce(
         new MockSSEStreamInterruptedError('Execution stream interrupted', 'exec-abandoned')
       )
@@ -852,6 +864,7 @@ describe('run tool execution cancellation', () => {
       // The editor's reconnect re-attaches, then the user navigates away and it is
       // cancelled, leaving the execution marked current in the store.
       setCurrentExecutionId('wf-1', 'exec-abandoned')
+      setIsExecuting('wf-1', true)
       serverStatus = 'completed'
       await vi.waitFor(() =>
         expect(isReconnectStreamOpen).toHaveBeenCalledWith('wf-1', 'exec-abandoned')
@@ -868,6 +881,36 @@ describe('run tool execution cancellation', () => {
       ])
     })
 
+    it('re-sends an undelivered completion while a later call owns the workflow', async () => {
+      const { finish, launchedToolCallIds } = holdExecutions()
+      let confirmsFailFor: string | null = 'tool-a'
+      vi.stubGlobal(
+        'fetch',
+        vi.fn(async (_url: string, init?: RequestInit) =>
+          confirmsFailFor && String(init?.body).includes(`"toolCallId":"${confirmsFailFor}"`)
+            ? { ok: false, status: 500 }
+            : { ok: true }
+        )
+      )
+
+      executeRunToolOnClient('tool-a', 'run_block', { workflowId: 'wf-1', blockId: 'block-1' })
+      executeRunToolOnClient('tool-b', 'run_block', { workflowId: 'wf-1', blockId: 'block-2' })
+      await vi.waitFor(() => expect(launchedToolCallIds()).toEqual(['tool-a']))
+      const firstExecutionId = executeWorkflowWithFullLogging.mock.calls[0][0].executionId
+      finish('tool-a')
+      await vi.waitFor(() => expect(launchedToolCallIds()).toEqual(['tool-a', 'tool-b']))
+
+      confirmsFailFor = null
+      await expect(bindRunToolToExecution('tool-a', 'wf-1')).resolves.toBe(true)
+
+      const delivered = confirmBodies()
+        .filter((body) => body.toolCallId === 'tool-a')
+        .at(-1)
+      expect(delivered).toMatchObject({ status: 'success', executionId: firstExecutionId })
+      finish('tool-b')
+      await vi.waitFor(() => expect(isRunToolActiveForWorkflow('wf-1')).toBe(false))
+    })
+
     it('treats a waiting call as already owned by this tab when the chat recovers it', async () => {
       const { finish, launchedToolCallIds } = holdExecutions()
 

apps/sim/lib/mothership/tools/client/run-tool-execution.ts

@@ -423,18 +423,14 @@ export async function bindRunToolToExecution(
     })
     return true
   }
-  if (existingToolCallId && existingToolCallId !== toolCallId) {
-    logger.warn('[RunTool] Recovery skipped: another run tool is already active', {
-      workflowId,
-      toolCallId,
-      existingToolCallId,
-    })
-    return false
-  }
-
-  const pointer = await loadExecutionPointer(workflowId).catch(() => null)
+  // Another run owns the workflow's pointer, so it says nothing about this call.
+  const otherRunOwnsWorkflow =
+    existingToolCallId !== undefined || interruptedExecutionIdByWorkflowId.has(workflowId)
   const pendingCompletion = loadPendingCompletionReport(toolCallId)
   if (pendingCompletion) {
+    const pointer = otherRunOwnsWorkflow
+      ? null
+      : await loadExecutionPointer(workflowId).catch(() => null)
     const executionId = pendingCompletion.executionId ?? pointer?.executionId
     logger.info('[RunTool] Recovery re-sending pending completion report', {
       workflowId,
@@ -453,7 +449,7 @@ export async function bindRunToolToExecution(
         executionId
       )
       clearPendingCompletionReport(toolCallId)
-      if (pendingCompletion.clearExecutionPointerAfterReport) {
+      if (pendingCompletion.clearExecutionPointerAfterReport && !otherRunOwnsWorkflow) {
         await clearExecutionPointer(workflowId)
       }
     } catch (error) {
@@ -467,6 +463,16 @@ export async function bindRunToolToExecution(
     return true
   }
 
+  if (otherRunOwnsWorkflow) {
+    logger.warn('[RunTool] Recovery skipped: another run owns this workflow', {
+      workflowId,
+      toolCallId,
+      existingToolCallId,
+    })
+    return false
+  }
+
+  const pointer = await loadExecutionPointer(workflowId).catch(() => null)
   if (!pointer?.executionId) {
     logger.info('[RunTool] Recovery skipped: no tab-local execution pointer', {
       workflowId,