fix(execution): tag Sim invariant failures and keep network errors out of system faults

- Add SystemError and isSystemError to @sim/utils/errors; throw SystemError
  where an internal tool operation has no registered handler so a broken
  registry still faults the job after executeTool flattens the error.
- isProgrammingError excludes errors carrying a cause, so fetch's
  TypeError('fetch failed', { cause }) for an unreachable host is not treated
  as a defect in Sim code.
- Track the schedule job fault in a box so a thrown undefined still faults.
- Drop the partial executor-errors mock; the real hasExecutionResult suffices.

Author: waleedlatif1

apps/sim/background/async-preprocessing-correlation.test.ts

@@ -26,15 +26,13 @@ const {
   mockExecuteWorkflowCore,
   mockExecutionSnapshot,
   mockWasExecutionFinalizedByCore,
-  mockHasExecutionResult,
   mockIsWorkflowTimedOut,
   mockGetScheduleTimeValues,
   mockGetSubBlockValue,
 } = vi.hoisted(() => ({
   mockExecuteWorkflowCore: vi.fn(),
   mockExecutionSnapshot: vi.fn(),
   mockWasExecutionFinalizedByCore: vi.fn(),
-  mockHasExecutionResult: vi.fn(),
   mockIsWorkflowTimedOut: vi.fn(() => false),
   mockGetScheduleTimeValues: vi.fn(),
   mockGetSubBlockValue: vi.fn(),
@@ -74,11 +72,6 @@ vi.mock('@/executor/execution/snapshot', () => ({
   ExecutionSnapshot: mockExecutionSnapshot,
 }))
 
-vi.mock('@/executor/utils/errors', async (importOriginal) => ({
-  ...(await importOriginal<typeof import('@/executor/utils/errors')>()),
-  hasExecutionResult: mockHasExecutionResult,
-}))
-
 import { buildBlockExecutionError } from '@/executor/utils/errors'
 import type { SerializedBlock } from '@/serializer/types'
 import { executeScheduleJob } from './schedule-execution'
@@ -128,7 +121,6 @@ const principal = {
 describe('async preprocessing correlation threading', () => {
   beforeEach(() => {
     mockWasExecutionFinalizedByCore.mockReturnValue(false)
-    mockHasExecutionResult.mockReturnValue(false)
     mockIsWorkflowTimedOut.mockReturnValue(false)
     resetDbChainMock()
     dbChainMockFns.limit.mockResolvedValue([
@@ -382,7 +374,6 @@ describe('async preprocessing correlation threading', () => {
       executionTimeout: {},
     })
     mockExecuteWorkflowCore.mockRejectedValueOnce(rawError)
-    mockHasExecutionResult.mockImplementation((error) => error === rawError)
     mockWasExecutionFinalizedByCore.mockReturnValue(true)
 
     await expect(

apps/sim/background/schedule-execution.ts

@@ -861,7 +861,7 @@ export async function executeScheduleJob(
        * (failure count, next run, claim) has run, so faulting the job to alert
        * on it never leaves the schedule claimed or its cadence stalled.
        */
-      let jobFault: unknown
+      let jobFault: { error: unknown } | undefined
 
       try {
         const [scheduleRecord] = await db
@@ -1213,7 +1213,7 @@ export async function executeScheduleJob(
           )
 
           const failure = await classifyWorkflowJobFailure({ error, executionId, loggingSession })
-          if (failure === 'job_fault') jobFault = error
+          if (failure === 'job_fault') jobFault = { error }
         }
       } catch (error: unknown) {
         try {
@@ -1222,7 +1222,7 @@ export async function executeScheduleJob(
             return
           }
 
-          jobFault = error
+          jobFault = { error }
           logger.error(`[${requestId}] Error processing schedule ${payload.scheduleId}`, error, {
             cause: describeError(error),
           })
@@ -1243,7 +1243,7 @@ export async function executeScheduleJob(
         }
       }
 
-      if (jobFault !== undefined) throw jobFault
+      if (jobFault) throw jobFault.error
     })
   } finally {
     timeoutController.cleanup()

apps/sim/lib/workflows/executor/job-failure.ts

@@ -1,4 +1,4 @@
-import { findCause, getErrorMessage, isProgrammingError } from '@sim/utils/errors'
+import { findCause, getErrorMessage, isSystemError } from '@sim/utils/errors'
 import type { LoggingSession } from '@/lib/logs/execution/logging-session'
 import { wasExecutionFinalizedByCore } from '@/lib/workflows/executor/execution-core'
 import {
@@ -60,20 +60,12 @@ const UNATTRIBUTED_WORKFLOW_FAILURE_CODES: ReadonlySet<WorkflowExecutionErrorCod
 ])
 
 /**
- * Whether Sim's own code, not the workflow, caused the failure: a programming
- * error anywhere in the `.cause` chain, or a link marked `isSystemError` where
- * `executeTool` flattened such an error into a result. User code never produces
- * either, since sandboxes return its errors as data.
+ * Whether Sim's own code, not the workflow, caused the failure anywhere in the
+ * `.cause` chain. User code never produces a system error, since sandboxes
+ * return its errors as data.
  */
 function isSystemFailure(error: unknown): boolean {
-  return (
-    findCause(
-      error,
-      (value): value is Error =>
-        isProgrammingError(value) ||
-        (value instanceof Error && 'isSystemError' in value && value.isSystemError === true)
-    ) !== undefined
-  )
+  return findCause(error, isSystemError) !== undefined
 }
 
 /**

apps/sim/tools/index.test.ts

@@ -1191,6 +1191,41 @@ describe('executeTool Function', () => {
     }
   )
 
+  it('marks a declared operation with no registered handler as a system error', async () => {
+    const mockTool = {
+      id: 'test_unregistered_operation',
+      name: 'Test Unregistered Operation',
+      description: 'Declares an operation no handler implements',
+      version: '1.0.0',
+      params: {},
+      operation: { input: createInternalToolOperationInput },
+    } satisfies InternalToolConfig<Record<string, unknown>>
+    ;(tools as Record<string, unknown>).test_unregistered_operation = mockTool
+    mockGetInternalToolOperationHandler.mockResolvedValueOnce(undefined)
+
+    try {
+      const result = await executeTool(
+        'test_unregistered_operation',
+        {},
+        {
+          executionContext: createToolExecutionContext({
+            userId: 'user-1',
+            workspaceId: 'workspace-1',
+            workflowId: 'workflow-1',
+          }),
+        }
+      )
+
+      expect(result).toMatchObject({
+        success: false,
+        error: 'No internal operation registered for test_unregistered_operation',
+        isSystemError: true,
+      })
+    } finally {
+      Reflect.deleteProperty(tools, 'test_unregistered_operation')
+    }
+  })
+
   it('preserves actorless schedule authority for registered operations', async () => {
     const mockTool = {
       id: 'test_actorless_registered_operation',

apps/sim/tools/index.ts

@@ -4,7 +4,8 @@ import {
   describeError,
   findCause,
   getErrorMessage,
-  isProgrammingError,
+  isSystemError,
+  SystemError,
   toError,
 } from '@sim/utils/errors'
 import { sleep } from '@sim/utils/helpers'
@@ -2430,7 +2431,7 @@ async function executeToolImplementation(
       // thrown error into a result object; an upstream provider's status stays
       // on `output` where it cannot be mistaken for ours.
       ...(error instanceof HttpError ? { statusCode: error.statusCode } : {}),
-      ...(findCause(error, isProgrammingError) ? { isSystemError: true as const } : {}),
+      ...(findCause(error, isSystemError) ? { isSystemError: true as const } : {}),
       timing: {
         startTime: startTimeISO,
         endTime: endTimeISO,
@@ -2738,7 +2739,7 @@ async function executeDeclaredInternalOperation({
     })
   } else {
     const handler = await getInternalToolOperationHandler(toolId)
-    if (!handler) throw new Error(`No internal operation registered for ${toolId}`)
+    if (!handler) throw new SystemError(`No internal operation registered for ${toolId}`)
     const requestedTimeout = Number(params.timeout)
     const operationTimeout =
       Number.isFinite(requestedTimeout) && requestedTimeout > 0
@@ -3293,7 +3294,7 @@ async function executeMcpTool(
     logger.info(`[${actualRequestId}] Executing MCP tool: ${toolId}`)
     validateRequestBodySize(JSON.stringify(params), actualRequestId, `mcp:${toolId}`)
     const handler = await getInternalToolOperationHandler(toolId)
-    if (!handler) throw new Error(`No internal operation registered for ${toolId}`)
+    if (!handler) throw new SystemError(`No internal operation registered for ${toolId}`)
     const resultResponse = await handler({
       toolId,
       input: params,

packages/utils/src/errors.test.ts

@@ -4,6 +4,7 @@ import {
   getPostgresCancellationReason,
   getPostgresErrorCode,
   getTransientDatabaseFailure,
+  isProgrammingError,
 } from '@sim/utils/errors'
 import { describe, expect, it } from 'vitest'
 
@@ -236,3 +237,27 @@ describe('describeError', () => {
     expect(described?.causeChain?.length).toBeLessThanOrEqual(10)
   })
 })
+
+describe('isProgrammingError', () => {
+  it('recognizes a defect the running code raised itself', () => {
+    const thrown = (() => {
+      try {
+        JSON.parse('null').flatMap()
+      } catch (error) {
+        return error
+      }
+    })()
+
+    expect(isProgrammingError(thrown)).toBe(true)
+  })
+
+  it('does not treat a runtime API reporting an external failure as a defect', () => {
+    const networkFailure = new TypeError('fetch failed', {
+      cause: Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:443'), {
+        code: 'ECONNREFUSED',
+      }),
+    })
+
+    expect(isProgrammingError(networkFailure)).toBe(false)
+  })
+})

packages/utils/src/errors.ts

@@ -319,14 +319,45 @@ export function findCause<T>(
 
 /**
  * Whether `value` is a JavaScript runtime programming error: a type, reference
- * or range fault raised by the running code itself, as opposed to an `Error` a
- * caller throws on purpose to reject its input.
+ * or range fault raised by the running code itself. A runtime API reporting an
+ * external failure wraps what it observed as `cause` (`fetch` rejects a network
+ * failure as `TypeError('fetch failed', { cause })`), while a defect such as
+ * calling a missing method never carries one.
  */
 export function isProgrammingError(
   value: unknown
 ): value is TypeError | ReferenceError | RangeError {
   return (
-    value instanceof TypeError || value instanceof ReferenceError || value instanceof RangeError
+    (value instanceof TypeError ||
+      value instanceof ReferenceError ||
+      value instanceof RangeError) &&
+    value.cause === undefined
+  )
+}
+
+/**
+ * A broken invariant in Sim's own code, never a rejection of the caller's
+ * input. Throw it where an invariant is checked so the failure is recognised as
+ * a platform fault even after it is flattened into a plain error.
+ */
+export class SystemError extends Error {
+  readonly isSystemError = true
+
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options)
+    this.name = 'SystemError'
+  }
+}
+
+/**
+ * Whether Sim's own code caused `value`: a {@link SystemError}, an error that
+ * carries its `isSystemError` flag across a flattening boundary, or a
+ * programming error (see {@link isProgrammingError}).
+ */
+export function isSystemError(value: unknown): value is Error {
+  return (
+    isProgrammingError(value) ||
+    (value instanceof Error && 'isSystemError' in value && value.isSystemError === true)
   )
 }