chore(trigger): tighten task limits, fix queues that never serialized, schedule missing self-hosted crons

Author: waleedlatif1

apps/docs/content/docs/platform/self-hosting/background-jobs.mdx

@@ -48,11 +48,13 @@ Point cron at an **internal** address where possible (the in-cluster Service, or
 | Outbox processing | `/api/webhooks/outbox/process` | `*/1 * * * *` | Transactional-outbox retries for billing, membership, enterprise issuance, and workflow-deployment side effects |
 | Workspace file search dispatch | `/api/cron/workspace-file-search-dispatch` | `*/1 * * * *` | Dispatches indexing work for workspace file search |
 | Knowledge projection | `/api/cron/knowledge-projection` | `*/1 * * * *` | Brings knowledge base search up to date with document, permission, and chunk changes |
+| Table row-change fold | `/api/cron/fold-table-row-changes` | `*/1 * * * *` | Folds the table row-change log into each table's stored row count and version |
 | Connector sync | `/api/knowledge/connectors/sync` | `*/5 * * * *` | Knowledge base connector syncs |
 | Connector member sync | `/api/knowledge/connectors/member-sync` | `*/5 * * * *` | Per-member content/access sync for indexed connectors; live Search sources are skipped |
 | Connector directory sync | `/api/knowledge/connectors/directory-sync` | `*/5 * * * *` | Refreshes stored groups for indexed administrator-mode connectors; live Search sources are skipped |
 | Workspace events poll | `/api/workspace-events/poll` | `*/15 * * * *` | Workspace event triggers |
 | Table row TTL cleanup | `/api/cron/cleanup-table-row-ttl` | `*/15 * * * *` | Deletes table rows whose TTL column has expired |
+| Stale execution cleanup | `/api/cron/cleanup-stale-executions` | `*/30 * * * *` | Fails runs whose worker died without finishing — workflow executions, background jobs, table jobs and table runs — so they stop showing as running |
 | OAuth token cleanup | `/api/cron/cleanup-oauth-tokens` | `0 * * * *` | Deletes access and refresh tokens after the retention tail |
 | SCIM reconciliation | `/api/cron/scim-reconcile` | `17 * * * *` | Re-applies directory group mappings |
 | Data drains | `/api/cron/run-data-drains` | `0 * * * *` | Enterprise data drains |
@@ -61,6 +63,7 @@ Point cron at an **internal** address where possible (the in-cluster Service, or
 | Reconcile billing seats | `/api/cron/reconcile-billing-seats` | `0 * * * *` | Billing only — safe to disable when self-hosted |
 | Reconcile inbox entitlement | `/api/cron/reconcile-inbox-entitlement` | `0 3 * * *` | Inbox access reconciliation |
 | Cleanup sandbox images | `/api/cron/cleanup-sandbox-images` | `30 4 * * *` | Reclaims sandbox images |
+| Idempotency key cleanup | `/api/webhooks/cleanup/idempotency` | `0 5 * * *` | Deletes deduplication records older than seven days |
 
 <Callout type="info">
   **Subscription renewal** covers Microsoft Teams chat triggers, whose Microsoft Graph subscriptions are hard-capped at about three days. Without it, Teams triggers work for a couple of days and then quietly stop. Gmail, Outlook, Drive, Calendar, and Sheets triggers are **polled** instead — they depend on the per-minute poll jobs above, not on this one.

apps/sim/background/cleanup-dispatch.ts

@@ -15,6 +15,7 @@ export const CLEANUP_DISPATCH_MAX_ATTEMPTS = 3
  */
 export const cleanupDispatchTask = task({
   id: 'cleanup-dispatch',
+  maxDuration: 900,
   queue: { concurrencyLimit: 1 },
   retry: { maxAttempts: CLEANUP_DISPATCH_MAX_ATTEMPTS },
   run: ({ jobType }: CleanupDispatchPayload) => dispatchCleanupJobs(jobType),

apps/sim/background/cleanup-file-versions.ts

@@ -188,7 +188,13 @@ export async function runCleanupFileVersions(payload: CleanupJobPayload): Promis
 
 export const cleanupFileVersionsTask = task({
   id: 'cleanup-file-versions',
-  machine: 'large-1x',
+  /**
+   * Sized from production telemetry: peak RSS 219 MB across ~15,600 runs, so
+   * `small-2x` (1 GB) holds over four times the worst case. Each pass works in
+   * bounded batches of version rows, so the peak does not grow with backlog.
+   */
+  machine: 'small-2x',
+  maxDuration: 600,
   queue: retentionCleanupQueue,
   retry: { maxAttempts: 1 },
   run: runCleanupFileVersions,

apps/sim/background/cleanup-stale-executions.ts

@@ -49,17 +49,19 @@ const GENERIC_STALE_PROCESSING_ERROR = `Job terminated: stuck in processing for
 const EXECUTION_DEADLINE_ERROR = getTimeoutErrorMessage()
 /**
  * Table jobs run as detached workers with progress heartbeats, independently of workflow timeout
- * policy. Preserve their historical 90-minute task window plus five-minute cleanup grace.
+ * policy. A running job is treated as dead once its last progress write (`updatedAt`) is older
+ * than this. It measures the gap since the last heartbeat, not run length, so it does not track
+ * any task's `maxDuration`; it only has to exceed the longest stretch a live worker goes without
+ * writing progress.
  */
 const TABLE_JOB_STALE_THRESHOLD_MINUTES = 95
 /** Terminal table-jobs older than this are pruned; only the latest job per table is ever read. */
 const TABLE_JOB_RETENTION_HOURS = 24
 /**
  * A table run dispatch whose holder has not made progress for this long is
- * treated as dead. Same shape and window as the table-job threshold above: the
- * 90-minute Trigger.dev task ceiling (`maxDuration` in `trigger.config.ts`) plus
- * five minutes of cleanup grace, measured from the dispatcher's own per-window
- * heartbeat rather than from when the run was requested.
+ * treated as dead. Same shape and window as the table-job threshold above,
+ * measured from the dispatcher's own per-window heartbeat rather than from when
+ * the run was requested, so it is independent of the task's `maxDuration`.
  */
 const TABLE_DISPATCH_STALE_THRESHOLD_MINUTES = 95
 /** Per-run ceiling on reaped dispatches, so one tick cannot fan out unbounded SSE. */
@@ -795,6 +797,7 @@ export async function runCleanupStaleExecutions() {
 
 export const cleanupStaleExecutionsTask = task({
   id: 'cleanup-stale-executions',
+  maxDuration: 1800,
   queue: { concurrencyLimit: 1 },
   run: () => runCleanupStaleExecutions(),
 })

apps/sim/background/cleanup-table-row-ttl.ts

@@ -335,6 +335,7 @@ export async function runCleanupTableRowTtl(
 
 export const cleanupTableRowTtlTask = task({
   id: 'cleanup-table-row-ttl',
+  maxDuration: 600,
   queue: { concurrencyLimit: 1 },
   run: () => runCleanupTableRowTtl(),
 })

apps/sim/background/fork-content-copy.ts

@@ -20,6 +20,14 @@ import {
 export const forkContentCopyTask = task({
   id: 'fork-content-copy',
   machine: 'large-2x',
+  /**
+   * Inside `STALE_ACTIVE_MS` (30 minutes, in the background-work store), after
+   * which the outbox cron marks the fork's status row failed. A run outliving
+   * that would keep copying under a status that already says it failed; the
+   * remaining five minutes absorb queue wait, since the row starts its clock at
+   * fork time.
+   */
+  maxDuration: 1500,
   retry: { maxAttempts: 1 },
   queue: {
     name: 'fork-content-copy',

apps/sim/background/knowledge-processing.ts

@@ -306,13 +306,16 @@ export async function runDocumentProcessing(
  */
 export const interactiveProcessingQueue = queue({
   name: INTERACTIVE_PROCESSING_QUEUE_NAME,
-  concurrencyLimit: envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20),
+  concurrencyLimit: envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20, { min: 1, integer: true }),
 })
 
 /** Referenced by no dispatch site: named per trigger, declared here so the deploy registers it. */
 export const backfillProcessingQueue = queue({
   name: BACKFILL_PROCESSING_QUEUE_NAME,
-  concurrencyLimit: envNumber(env.KB_CONFIG_BACKFILL_CONCURRENCY_LIMIT, 20),
+  concurrencyLimit: envNumber(env.KB_CONFIG_BACKFILL_CONCURRENCY_LIMIT, 20, {
+    min: 1,
+    integer: true,
+  }),
 })
 
 export const processDocument = task({

apps/sim/background/lifecycle-email.ts

@@ -59,6 +59,8 @@ async function sendLifecycleEmail({ userId, type }: LifecycleEmailParams): Promi
 
 export const lifecycleEmailTask = task({
   id: LIFECYCLE_EMAIL_TASK_ID,
+  maxDuration: 120,
+  queue: { concurrencyLimit: 10 },
   retry: { maxAttempts: 2 },
   run: async (params: LifecycleEmailParams) => {
     await sendLifecycleEmail(params)

apps/sim/background/mothership-inbox-execution.ts

@@ -11,6 +11,7 @@ export interface MothershipInboxExecutionParams {
 export const mothershipInboxExecution = task({
   id: 'mothership-inbox-execution',
   machine: { preset: 'medium-1x' },
+  queue: { concurrencyLimit: 10 },
   retry: {
     maxAttempts: 2,
     minTimeoutInMs: 5000,

apps/sim/background/quickbooks-webhook-ingress.ts

@@ -143,6 +143,7 @@ export async function enqueueQuickBooksWebhookIngress(
 export const quickBooksWebhookIngressTask = task({
   id: 'quickbooks-webhook-ingress',
   machine: 'small-1x',
+  maxDuration: 300,
   retry: {
     maxAttempts: QUICKBOOKS_WEBHOOK_INGRESS_MAX_ATTEMPTS,
     factor: 2,