fix(core): truncate large error stacks and messages to prevent OOM (#3405)

## Summary

Large error stacks and messages can OOM the worker process when
serialized into OTel spans or `TaskRunError` objects. This was reported
when throwing an error with a massive `.stack` property from a chat
agent hook.

This adds frame-based stack truncation (similar to Sentry's approach)
plus message length limits, applied consistently across all error
serialization paths.

### What changed

**`packages/core/src/v3/errors.ts`**
- `truncateStack()` — parses `error.stack` into message lines + frame
lines, caps at 50 frames (keep top 5 closest to throw + bottom 45 entry
points, with "... N frames omitted ..." in between). Individual lines
capped at 1024 chars.
- `truncateMessage()` — caps error messages at 1000 chars
- Applied in `parseError()` and `sanitizeError()`

**`packages/core/src/v3/otel/utils.ts`**
- `sanitizeSpanError()` now uses `truncateStack` and `truncateMessage`
from `errors.ts` instead of duplicating truncation logic
- Non-Error values (strings, JSON) capped at 5000 chars

**`packages/core/src/v3/tracer.ts`**
- `startActiveSpan` catch block now delegates to `recordSpanException()`
instead of calling `span.recordException()` directly

### Limits

| What | Limit | Rationale |
|------|-------|-----------|
| Stack frames | 50 | Matches Sentry's `STACKTRACE_FRAME_LIMIT` |
| Top frames kept | 5 | Closest to throw site |
| Bottom frames kept | 45 | Entry points / framework frames |
| Per-line length | 1024 | Matches Sentry, prevents regex DoS |
| Message length | 1000 | Bounded but generous |
| Generic string (non-Error) | 5000 | Fallback for JSON/string errors in
spans |

## Test plan

- [x] 17 unit tests in `packages/core/test/errors.test.ts`
- [x] E2E: threw a 300-frame / 5000-char-message error in the ai-chat
reference app, verified truncated stack and message in span via
`get_span_details`
- [x] Verified the run survived the error (no OOM, continued waiting for
next message)

Author: ericallam

.changeset/truncate-error-stacks.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Truncate large error stacks and messages to prevent OOM crashes. Stack traces are capped at 50 frames (keeping top 5 + bottom 45 with an omission notice), individual stack lines at 1024 chars, and error messages at 1000 chars. Applied in parseError, sanitizeError, and OTel span recording.

packages/core/src/v3/errors.ts

@@ -154,22 +154,74 @@ export function isCompleteTaskWithOutput(error: unknown): error is CompleteTaskW
   return error instanceof Error && error.name === "CompleteTaskWithOutput";
 }
 
+const MAX_STACK_FRAMES = 50;
+const KEEP_TOP_FRAMES = 5;
+const MAX_STACK_LINE_LENGTH = 1024;
+const MAX_MESSAGE_LENGTH = 1_000;
+
+/** Truncate a stack trace to at most MAX_STACK_FRAMES frames, keeping
+ *  the top (closest to throw) and bottom (entry points) frames.
+ *  Individual lines (including message lines) are capped at MAX_STACK_LINE_LENGTH
+ *  to prevent OOM from huge error messages embedded in the stack. */
+export function truncateStack(stack: string | undefined): string {
+  if (!stack) return "";
+
+  const lines = stack.split("\n");
+
+  // First line(s) before the first frame are the error message
+  const messageLines: string[] = [];
+  const frameLines: string[] = [];
+
+  for (const line of lines) {
+    const safe =
+      line.length > MAX_STACK_LINE_LENGTH
+        ? line.slice(0, MAX_STACK_LINE_LENGTH) + "...[truncated]"
+        : line;
+    if (frameLines.length === 0 && !line.trimStart().startsWith("at ")) {
+      messageLines.push(safe);
+    } else {
+      frameLines.push(safe);
+    }
+  }
+
+  if (frameLines.length <= MAX_STACK_FRAMES) {
+    return [...messageLines, ...frameLines].join("\n");
+  }
+
+  const keepBottom = MAX_STACK_FRAMES - KEEP_TOP_FRAMES;
+  const omitted = frameLines.length - MAX_STACK_FRAMES;
+
+  return [
+    ...messageLines,
+    ...frameLines.slice(0, KEEP_TOP_FRAMES),
+    `    ... ${omitted} frames omitted ...`,
+    ...frameLines.slice(-keepBottom),
+  ].join("\n");
+}
+
+export function truncateMessage(message: string | undefined): string {
+  if (!message) return "";
+  return message.length > MAX_MESSAGE_LENGTH
+    ? message.slice(0, MAX_MESSAGE_LENGTH) + "...[truncated]"
+    : message;
+}
+
 export function parseError(error: unknown): TaskRunError {
   if (isInternalError(error)) {
     return {
       type: "INTERNAL_ERROR",
       code: error.code,
-      message: error.message,
-      stackTrace: error.stack ?? "",
+      message: truncateMessage(error.message),
+      stackTrace: truncateStack(error.stack),
     };
   }
 
   if (error instanceof Error) {
     return {
       type: "BUILT_IN_ERROR",
       name: error.name,
-      message: error.message,
-      stackTrace: error.stack ?? "",
+      message: truncateMessage(error.message),
+      stackTrace: truncateStack(error.stack),
     };
   }
 
@@ -248,35 +300,52 @@ export function createJsonErrorObject(error: TaskRunError): SerializedError {
   }
 }
 
-// Removes any null characters from the error message
+// Removes null characters and truncates oversized fields to prevent OOM
 export function sanitizeError(error: TaskRunError): TaskRunError {
   switch (error.type) {
     case "BUILT_IN_ERROR": {
       return {
         type: "BUILT_IN_ERROR",
-        message: error.message?.replace(/\0/g, ""),
+        message: truncateMessage(error.message?.replace(/\0/g, "")),
         name: error.name?.replace(/\0/g, ""),
-        stackTrace: error.stackTrace?.replace(/\0/g, ""),
+        stackTrace: truncateStack(error.stackTrace?.replace(/\0/g, "")),
       };
     }
     case "STRING_ERROR": {
       return {
         type: "STRING_ERROR",
-        raw: error.raw.replace(/\0/g, ""),
+        raw: truncateMessage(error.raw.replace(/\0/g, "")),
       };
     }
     case "CUSTOM_ERROR": {
+      // CUSTOM_ERROR.raw holds JSON.stringify(error) which is later parsed by
+      // JSON.parse in createErrorTaskError. Naive truncation would cut mid-token
+      // and produce invalid JSON — wrap the preview in a valid JSON envelope.
+      const clean = error.raw.replace(/\0/g, "");
+      const safeRaw =
+        clean.length > MAX_MESSAGE_LENGTH
+          ? JSON.stringify({ truncated: true, preview: clean.slice(0, MAX_MESSAGE_LENGTH) })
+          : clean;
       return {
         type: "CUSTOM_ERROR",
-        raw: error.raw.replace(/\0/g, ""),
+        raw: safeRaw,
       };
     }
     case "INTERNAL_ERROR": {
+      // message and stackTrace are optional for INTERNAL_ERROR — preserve
+      // `undefined` so the `error.message ?? "Internal error (CODE)"` fallback
+      // in createErrorTaskError still kicks in (empty string is not nullish).
       return {
         type: "INTERNAL_ERROR",
         code: error.code,
-        message: error.message?.replace(/\0/g, ""),
-        stackTrace: error.stackTrace?.replace(/\0/g, ""),
+        message:
+          error.message != null
+            ? truncateMessage(error.message.replace(/\0/g, ""))
+            : undefined,
+        stackTrace:
+          error.stackTrace != null
+            ? truncateStack(error.stackTrace.replace(/\0/g, ""))
+            : undefined,
       };
     }
   }

packages/core/src/v3/otel/utils.ts

@@ -1,22 +1,47 @@
 import { type Span, SpanStatusCode, context, propagation } from "@opentelemetry/api";
+import { truncateStack, truncateMessage } from "../errors.js";
+
+const MAX_GENERIC_LENGTH = 5_000;
+
+function truncateGeneric(value: string): string {
+  return value.length > MAX_GENERIC_LENGTH
+    ? value.slice(0, MAX_GENERIC_LENGTH) + "...[truncated]"
+    : value;
+}
+
+function serializeFallback(error: unknown): string {
+  // JSON.stringify can throw (circular refs, BigInt) or return undefined
+  // (symbol, undefined, function). Fall back to String() in both cases so we
+  // never mask the original error being recorded.
+  try {
+    const json = JSON.stringify(error);
+    if (json != null) return json;
+  } catch {
+    // fall through
+  }
+  try {
+    return String(error);
+  } catch {
+    return "[unserializable error]";
+  }
+}
 
 export function recordSpanException(span: Span, error: unknown) {
   if (error instanceof Error) {
     span.recordException(sanitizeSpanError(error));
   } else if (typeof error === "string") {
-    span.recordException(error.replace(/\0/g, ""));
+    span.recordException(truncateGeneric(error.replace(/\0/g, "")));
   } else {
-    span.recordException(JSON.stringify(error).replace(/\0/g, ""));
+    span.recordException(truncateGeneric(serializeFallback(error).replace(/\0/g, "")));
   }
 
   span.setStatus({ code: SpanStatusCode.ERROR });
 }
 
 function sanitizeSpanError(error: Error) {
-  // Create a new error object with the same name, message and stack trace
-  const sanitizedError = new Error(error.message.replace(/\0/g, ""));
+  const sanitizedError = new Error(truncateMessage(error.message.replace(/\0/g, "")));
   sanitizedError.name = error.name.replace(/\0/g, "");
-  sanitizedError.stack = error.stack?.replace(/\0/g, "");
+  sanitizedError.stack = truncateStack(error.stack?.replace(/\0/g, "")) || undefined;
 
   return sanitizedError;
 }

packages/core/src/v3/tracer.ts

@@ -145,11 +145,7 @@ export class TriggerTracer {
           }
 
           if (!spanEnded) {
-            if (typeof e === "string" || e instanceof Error) {
-              span.recordException(e);
-            }
-
-            span.setStatus({ code: SpanStatusCode.ERROR });
+            recordSpanException(span, e);
           }
 
           throw e;