chore(trigger): fix queues that never serialized and schedule missing self-hosted crons (#8715)

* chore(trigger): tighten task limits, fix queues that never serialized, schedule missing self-hosted crons

* refactor(trigger): derive the fork ceiling from STALE_ACTIVE_MS and trim sizing comments

* chore(trigger): drop caps whose work grows with user data or that lose cleanup on kill

* chore(trigger): keep unbounded-fan-out tasks on the global ceiling; bump chart for new cronjobs

Author: waleedlatif1

apps/docs/content/docs/platform/self-hosting/background-jobs.mdx

@@ -48,11 +48,13 @@ Point cron at an **internal** address where possible (the in-cluster Service, or
 | Outbox processing | `/api/webhooks/outbox/process` | `*/1 * * * *` | Transactional-outbox retries for billing, membership, enterprise issuance, and workflow-deployment side effects |
 | Workspace file search dispatch | `/api/cron/workspace-file-search-dispatch` | `*/1 * * * *` | Dispatches indexing work for workspace file search |
 | Knowledge projection | `/api/cron/knowledge-projection` | `*/1 * * * *` | Brings knowledge base search up to date with document, permission, and chunk changes |
+| Table row-change fold | `/api/cron/fold-table-row-changes` | `*/1 * * * *` | Folds the table row-change log into each table's stored row count and version |
 | Connector sync | `/api/knowledge/connectors/sync` | `*/5 * * * *` | Knowledge base connector syncs |
 | Connector member sync | `/api/knowledge/connectors/member-sync` | `*/5 * * * *` | Per-member content/access sync for indexed connectors; live Search sources are skipped |
 | Connector directory sync | `/api/knowledge/connectors/directory-sync` | `*/5 * * * *` | Refreshes stored groups for indexed administrator-mode connectors; live Search sources are skipped |
 | Workspace events poll | `/api/workspace-events/poll` | `*/15 * * * *` | Workspace event triggers |
 | Table row TTL cleanup | `/api/cron/cleanup-table-row-ttl` | `*/15 * * * *` | Deletes table rows whose TTL column has expired |
+| Stale execution cleanup | `/api/cron/cleanup-stale-executions` | `*/30 * * * *` | Fails runs whose worker died without finishing — workflow executions, background jobs, table jobs and table runs — so they stop showing as running |
 | OAuth token cleanup | `/api/cron/cleanup-oauth-tokens` | `0 * * * *` | Deletes access and refresh tokens after the retention tail |
 | SCIM reconciliation | `/api/cron/scim-reconcile` | `17 * * * *` | Re-applies directory group mappings |
 | Data drains | `/api/cron/run-data-drains` | `0 * * * *` | Enterprise data drains |
@@ -61,6 +63,7 @@ Point cron at an **internal** address where possible (the in-cluster Service, or
 | Reconcile billing seats | `/api/cron/reconcile-billing-seats` | `0 * * * *` | Billing only — safe to disable when self-hosted |
 | Reconcile inbox entitlement | `/api/cron/reconcile-inbox-entitlement` | `0 3 * * *` | Inbox access reconciliation |
 | Cleanup sandbox images | `/api/cron/cleanup-sandbox-images` | `30 4 * * *` | Reclaims sandbox images |
+| Idempotency key cleanup | `/api/webhooks/cleanup/idempotency` | `0 5 * * *` | Deletes deduplication records older than seven days |
 
 <Callout type="info">
   **Subscription renewal** covers Microsoft Teams chat triggers, whose Microsoft Graph subscriptions are hard-capped at about three days. Without it, Teams triggers work for a couple of days and then quietly stop. Gmail, Outlook, Drive, Calendar, and Sheets triggers are **polled** instead — they depend on the per-minute poll jobs above, not on this one.

apps/sim/background/cleanup-stale-executions.ts

@@ -49,17 +49,16 @@ const GENERIC_STALE_PROCESSING_ERROR = `Job terminated: stuck in processing for
 const EXECUTION_DEADLINE_ERROR = getTimeoutErrorMessage()
 /**
  * Table jobs run as detached workers with progress heartbeats, independently of workflow timeout
- * policy. Preserve their historical 90-minute task window plus five-minute cleanup grace.
+ * policy. A running job is treated as dead once its last progress write (`updatedAt`) is older
+ * than this, so it must exceed the longest gap between a live worker's heartbeats.
  */
 const TABLE_JOB_STALE_THRESHOLD_MINUTES = 95
 /** Terminal table-jobs older than this are pruned; only the latest job per table is ever read. */
 const TABLE_JOB_RETENTION_HOURS = 24
 /**
  * A table run dispatch whose holder has not made progress for this long is
- * treated as dead. Same shape and window as the table-job threshold above: the
- * 90-minute Trigger.dev task ceiling (`maxDuration` in `trigger.config.ts`) plus
- * five minutes of cleanup grace, measured from the dispatcher's own per-window
- * heartbeat rather than from when the run was requested.
+ * treated as dead. Same rule as the table-job threshold above, measured from the
+ * dispatcher's own per-window heartbeat rather than from when the run was requested.
  */
 const TABLE_DISPATCH_STALE_THRESHOLD_MINUTES = 95
 /** Per-run ceiling on reaped dispatches, so one tick cannot fan out unbounded SSE. */

apps/sim/background/cleanup-table-row-ttl.ts

@@ -335,6 +335,7 @@ export async function runCleanupTableRowTtl(
 
 export const cleanupTableRowTtlTask = task({
   id: 'cleanup-table-row-ttl',
+  maxDuration: 600,
   queue: { concurrencyLimit: 1 },
   run: () => runCleanupTableRowTtl(),
 })

apps/sim/background/lifecycle-email.ts

@@ -59,6 +59,7 @@ async function sendLifecycleEmail({ userId, type }: LifecycleEmailParams): Promi
 
 export const lifecycleEmailTask = task({
   id: LIFECYCLE_EMAIL_TASK_ID,
+  queue: { concurrencyLimit: 10 },
   retry: { maxAttempts: 2 },
   run: async (params: LifecycleEmailParams) => {
     await sendLifecycleEmail(params)

apps/sim/background/run-data-drain.ts

@@ -9,6 +9,18 @@ interface RunDataDrainPayload {
 
 export const runDataDrainTask = task({
   id: 'run-data-drain',
+  /**
+   * The drain cursor commits only after the last chunk is delivered, so a run
+   * cut off by its ceiling restarts from the old cursor and re-exports the same
+   * rows. A shorter cap would loop a large drain forever.
+   */
+  maxDuration: 5400,
+  /**
+   * Enqueue sites key runs by `data-drain:<id>`; this limit is what makes that
+   * key serialize one drain's runs instead of granting each key the
+   * environment's full concurrency.
+   */
+  queue: { concurrencyLimit: 1 },
   run: async ({ drainId, trigger }: RunDataDrainPayload, { signal }) =>
     runDrain(drainId, trigger, { signal }),
 })

apps/sim/background/table-run-dispatcher.ts

@@ -29,9 +29,14 @@ export const tableRunDispatcherTask = task({
    * 0.03 for p90, so the larger preset is bought for its RAM.
    */
   machine: 'small-2x',
+  /**
+   * The trigger site keys each run by its `dispatchId`, so the limit applies
+   * per dispatch: 1 serializes a duplicate run of the same dispatch without
+   * throttling distinct dispatches against each other.
+   */
   queue: {
     name: 'table-run-dispatcher',
-    concurrencyLimit: 8,
+    concurrencyLimit: 1,
   },
   run: async (payload: TableRunDispatcherPayload) => {
     const { dispatchId, concurrency } = payload

apps/sim/lib/table/dispatcher.ts

@@ -52,9 +52,8 @@ const STALE_DISPATCH_EVENT_CONCURRENCY = 10
  * How long past its stale threshold a dispatch may be spared by cell activity
  * before it is reclaimed anyway. Bounds the one masking case the probe cannot
  * resolve — two table-wide dispatches sharing a group — which continuous
- * activity would otherwise hide forever. Far beyond any single window: the
- * Trigger.dev run ceiling is ninety minutes, and a live dispatch heartbeats
- * between windows no matter what its cells are doing.
+ * activity would otherwise hide forever. Far beyond any single window: a live
+ * dispatch heartbeats between windows no matter what its cells are doing.
  */
 const DISPATCH_ABSOLUTE_STALE_MS = 24 * 60 * 60 * 1000
 

apps/sim/trigger.config.ts

@@ -87,7 +87,7 @@ const grafanaTelemetry = grafanaFullyConfigured
 export default defineConfig({
   project: env.TRIGGER_PROJECT_ID!,
   runtime: 'node-24',
-  logLevel: 'log',
+  logLevel: 'info',
   maxDuration: 5400,
   retries: {
     enabledInDev: false,

docker/crontab

@@ -56,6 +56,9 @@ SHELL=/bin/sh
 # Deletes table rows whose TTL column has expired
 */15 * * * * curl -fsS -m 60 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/cleanup-table-row-ttl"
 
+# Fails runs whose worker died without a terminal state and prunes their history
+*/30 * * * * curl -fsS -m 60 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/cleanup-stale-executions"
+
 # Deletes OAuth token rows after the retention tail
 0 * * * * curl -fsS -m 120 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/cleanup-oauth-tokens"
 
@@ -68,6 +71,9 @@ SHELL=/bin/sh
 # Reclaims sandbox images
 30 4 * * * curl -fsS -m 300 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/cleanup-sandbox-images"
 
+# Deletes idempotency records past the seven-day dedupe window
+0 5 * * * curl -fsS -m 300 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/webhooks/cleanup/idempotency"
+
 # Inbox entitlement reconciliation
 0 3 * * * curl -fsS -m 120 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/reconcile-inbox-entitlement"
 

helm/sim/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: sim
 description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents
 type: application
-version: 1.11.6
+version: 1.11.7
 appVersion: "v0.8.26"
 kubeVersion: ">=1.25.0-0"
 home: https://sim.ai