fix(delivery): stable provider idempotency tokens, and correctness fixes around them

Six independent fixes found while investigating a Slack transport failure. None
of them are that failure; all of them are live.

**Money writes could be delivered twice.** Square (8 tools), Brex (5) and Outlook
Calendar minted their provider idempotency token with `generateId()` at
request-build time. That is stable inside the transport retry loop —
`prepareToolRequest` runs once above it — but a BLOCK-level retry re-enters the
handler and mints a fresh one, defeating the provider's dedupe. A builder ticking
"retry" on a Square block turned a committed write into a second card charge, and
a Brex one into a second money transfer. Tokens now come from
`deriveDeliveryKey`, a pure function of execution + block + tool + invocation, so
every retry layer derives the same value. Stripe joins them: all ~50 tools
previously sent no `Idempotency-Key` at all.

The comment on `brex/create_transfer.ts` claimed a fresh key per transfer
*prevents* duplicate money movement. That is inverted — fresh per *attempt* is
what permits it — and is corrected here.

**`invocationId` is required, not optional.** Deriving from `executionId` alone
would be worse than the bug: five loop iterations paying five invoices would
share one token, the provider would honour the first and silently drop four real
payments, and it would look like five successes.

Also:

- `INTERNAL_API_BASE_URL` is now ignored on Trigger.dev workers. It names a route
  that resolves only inside the app container, and several modules run in both
  runtimes — `guardrails/mask-client.ts` says so in its own TSDoc — so setting it
  produced `PII redaction failed: Unable to connect` on every worker-side
  redaction. Mirrored into `packages/testing`'s urls mock, which reimplements the
  function and would otherwise have diverged.
- `engines.bun` raised to >=1.3.14. Measured on 1.2.15, which the old floor
  permitted: a fully-delivered POST is silently replayed and the caller sees 200.
- CloudWatch `put-metric-data` pinned to `maxAttempts: 1`. The AWS SDK default of
  3 already replayed it, and `PutMetricData` aggregates rather than overwrites, so
  a duplicate silently corrupts the customer's metric series and alarm thresholds.
- `webhookIdempotency` given a bounded in-progress lease. An untimed run held a
  SEVEN DAY lease while concurrent duplicates polled it once a second.

Verified: `tsc --noEmit` clean, 30,100 tests pass.

Author: icecrasher321

apps/desktop/package.json

@@ -9,7 +9,7 @@
   "type": "module",
   "main": "dist/main.cjs",
   "engines": {
-    "bun": ">=1.2.13",
+    "bun": ">=1.3.14",
     "node": ">=20.0.0"
   },
   "scripts": {

apps/realtime/package.json

@@ -5,7 +5,7 @@
   "license": "Apache-2.0",
   "type": "module",
   "engines": {
-    "bun": ">=1.2.13",
+    "bun": ">=1.3.14",
     "node": ">=20.0.0"
   },
   "scripts": {

apps/sim/app/api/tools/cloudwatch/put-metric-data/retry-semantics.test.ts

@@ -0,0 +1,73 @@
+/**
+ * Pins what `maxAttempts` *means*, using the real AWS SDK.
+ *
+ * `route.test.ts` asserts the route configures `maxAttempts: 1`; on its own that
+ * only pins a number. This file counts how many datapoints a real
+ * `CloudWatchClient` actually hands to a peer that accepts the request and then
+ * dies before writing a response byte -- the ambiguous transport failure the
+ * SDK's retry layer replays, and the one that makes CloudWatch aggregate a
+ * duplicate. It fails if a future SDK bump changes the default budget or stops
+ * honouring the pin.
+ *
+ * Deliberately not mocking `@aws-sdk/client-cloudwatch` here: the SDK's retry
+ * middleware is the subject under test, so it must be the real one.
+ *
+ * @vitest-environment node
+ */
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { CloudWatchClient, PutMetricDataCommand } from '@aws-sdk/client-cloudwatch'
+import { describe, expect, it } from 'vitest'
+
+/** `@smithy/util-retry`'s `DEFAULT_MAX_ATTEMPTS`, which every unpinned client inherits. */
+const SDK_DEFAULT_MAX_ATTEMPTS = 3
+
+async function countDeliveries(clientConfig: Record<string, unknown>): Promise<number> {
+  let received = 0
+  const server = http.createServer((req, res) => {
+    req.on('data', () => {})
+    req.on('end', () => {
+      if (String(req.headers['x-amz-target'] ?? '').endsWith('PutMetricData')) {
+        received++
+        req.socket.destroy()
+        return
+      }
+      res.writeHead(200, { 'content-type': 'application/x-amz-json-1.0' })
+      res.end('{}')
+    })
+  })
+  await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
+  const { port } = server.address() as AddressInfo
+
+  const client = new CloudWatchClient({
+    region: 'us-east-1',
+    endpoint: `http://127.0.0.1:${port}`,
+    credentials: { accessKeyId: 'AKIAEXAMPLE', secretAccessKey: 'secret' },
+    ...clientConfig,
+  })
+
+  try {
+    await client.send(
+      new PutMetricDataCommand({
+        Namespace: 'Sim/Test',
+        MetricData: [{ MetricName: 'Requests', Value: 1 }],
+      })
+    )
+  } catch {
+    /* Every attempt fails by design; the delivery count is the assertion. */
+  } finally {
+    client.destroy()
+    await new Promise<void>((resolve) => server.close(() => resolve()))
+  }
+  return received
+}
+
+describe('aws sdk retry semantics for PutMetricData', () => {
+  it('delivers the datapoint exactly once when maxAttempts is pinned to 1', async () => {
+    await expect(countDeliveries({ maxAttempts: 1 })).resolves.toBe(1)
+  })
+
+  it('aggregates a duplicate for every retry the default budget allows', async () => {
+    await expect(countDeliveries({})).resolves.toBe(SDK_DEFAULT_MAX_ATTEMPTS)
+  })
+})

apps/sim/app/api/tools/cloudwatch/put-metric-data/route.test.ts

@@ -0,0 +1,65 @@
+/**
+ * @vitest-environment node
+ */
+import { createMockRequest, hybridAuthMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockSend, mockDestroy, capturedConfigs } = vi.hoisted(() => ({
+  mockSend: vi.fn(),
+  mockDestroy: vi.fn(),
+  capturedConfigs: [] as Record<string, unknown>[],
+}))
+
+vi.mock('@aws-sdk/client-cloudwatch', () => ({
+  CloudWatchClient: class {
+    constructor(config: Record<string, unknown>) {
+      capturedConfigs.push(config)
+    }
+    send = mockSend
+    destroy = mockDestroy
+  },
+  PutMetricDataCommand: class {
+    constructor(readonly input: Record<string, unknown>) {}
+  },
+}))
+
+import { POST } from '@/app/api/tools/cloudwatch/put-metric-data/route'
+
+const body = {
+  region: 'us-east-1',
+  accessKeyId: 'AKIAEXAMPLE',
+  secretAccessKey: 'secret',
+  namespace: 'Sim/Test',
+  metricName: 'Requests',
+  value: 1,
+}
+
+function postRoute() {
+  return POST(createMockRequest('POST', body))
+}
+
+describe('cloudwatch put-metric-data delivery class', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    capturedConfigs.length = 0
+    hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ success: true, userId: 'user-1' })
+    mockSend.mockResolvedValue({})
+  })
+
+  it('pins the client to a single attempt so the SDK cannot replay the datapoint', async () => {
+    const response = await postRoute()
+
+    expect(response.status).toBe(200)
+    expect(capturedConfigs).toHaveLength(1)
+    expect(capturedConfigs[0].maxAttempts).toBe(1)
+  })
+
+  it('never lets an ambiguous failure turn into a second PutMetricData', async () => {
+    mockSend.mockRejectedValue(Object.assign(new Error('socket hang up'), { name: 'TimeoutError' }))
+
+    const response = await postRoute()
+
+    expect(response.status).toBe(500)
+    expect(mockSend).toHaveBeenCalledTimes(1)
+  })
+})

apps/sim/app/api/tools/cloudwatch/put-metric-data/route.ts

@@ -9,10 +9,38 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { awsCloudwatchPutMetricDataContract } from '@/lib/api/contracts/tools/aws/cloudwatch-put-metric-data'
 import { parseToolRequest } from '@/lib/api/server'
 import { checkInternalAuth } from '@/lib/auth/hybrid'
+import type { DeliveryDeclaration } from '@/lib/core/http/classes'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 
 const logger = createLogger('CloudWatchPutMetricData')
 
+/**
+ * PutMetricData is additive, not last-write-wins: CloudWatch folds every
+ * datapoint it receives for a (namespace, metric, dimensions, timestamp) tuple
+ * into the same statistic set. Two deliveries of one user call therefore
+ * publish `SampleCount=2, Sum=2 x value` instead of `SampleCount=1, Sum=value`,
+ * silently doubling the customer's series and any alarm threshold read off it.
+ * Nothing on the wire distinguishes this from a correct write, and the datapoint
+ * cannot be retracted -- CloudWatch has no delete-datapoint API.
+ */
+const PUT_METRIC_DATA_DELIVERY = {
+  deliveryClass: 'once',
+  why: "the customer's metric series would double-count this datapoint, skewing every statistic and alarm derived from it",
+  userVisibleEffect:
+    'a datapoint with SampleCount=2 and Sum=2x the reported value, permanently, with no way to retract it',
+} satisfies DeliveryDeclaration
+
+/**
+ * The AWS SDK's own retry layer replays a request whenever the transport fails
+ * ambiguously (ECONNRESET, socket hangup, 500/502/503/504) -- exactly the cases
+ * where the peer may already have committed. `@smithy/util-retry` defaults to
+ * `DEFAULT_MAX_ATTEMPTS = 3`, so an unpinned client turns one severed socket
+ * into three aggregated datapoints. Pinning to 1 trades a lost datapoint for a
+ * correct series, which is the right trade for an additive metric: a gap is
+ * visible and self-healing, a doubled value is neither.
+ */
+const NON_IDEMPOTENT_MAX_ATTEMPTS = 1
+
 export const POST = withRouteHandler(async (request: NextRequest) => {
   try {
     const auth = await checkInternalAuth(request)
@@ -31,6 +59,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
 
     const client = new CloudWatchClient({
       region: validatedData.region,
+      maxAttempts: NON_IDEMPOTENT_MAX_ATTEMPTS,
       credentials: {
         accessKeyId: validatedData.accessKeyId,
         secretAccessKey: validatedData.secretAccessKey,
@@ -80,7 +109,12 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
       client.destroy()
     }
   } catch (error) {
-    logger.error('PutMetricData failed', { error: toError(error).message })
+    logger.error('PutMetricData failed', {
+      error: toError(error).message,
+      deliveryClass: PUT_METRIC_DATA_DELIVERY.deliveryClass,
+      outcome: 'indeterminate',
+      duplicateEffect: PUT_METRIC_DATA_DELIVERY.userVisibleEffect,
+    })
     return NextResponse.json(
       { error: `Failed to publish CloudWatch metric: ${toError(error).message}` },
       { status: 500 }

apps/sim/background/webhook-execution.ts

@@ -23,7 +23,11 @@ import {
   getTimeoutErrorMessage,
   RESERVATION_TTL_BUFFER_MS,
 } from '@/lib/core/execution-limits'
-import { IdempotencyService, webhookIdempotency } from '@/lib/core/idempotency'
+import {
+  IdempotencyService,
+  WEBHOOK_IN_PROGRESS_LEASE_SECONDS,
+  webhookIdempotency,
+} from '@/lib/core/idempotency'
 import {
   type EnvironmentResolutionSnapshot,
   getEffectiveEnvironmentSnapshot,
@@ -364,9 +368,12 @@ export async function executeWebhookJob(
           idempotencyKey,
           runOperation,
           undefined,
-          executionDeadlineAt === undefined
-            ? undefined
-            : { inProgressExpiresAt: executionDeadlineAt + RESERVATION_TTL_BUFFER_MS }
+          {
+            inProgressExpiresAt:
+              executionDeadlineAt === undefined
+                ? Date.now() + WEBHOOK_IN_PROGRESS_LEASE_SECONDS * 1000
+                : executionDeadlineAt + RESERVATION_TTL_BUFFER_MS,
+          }
         )
         if (!operationStarted) {
           await releaseExecutionSlot(executionId)

apps/sim/executor/execution/block-executor.ts

@@ -237,7 +237,7 @@ export class BlockExecutor {
       const output = await this.runHandlerWithRetry(blockCtx, block, blockLog, () =>
         handler.executeWithNode
           ? handler.executeWithNode(blockCtx, block, resolvedInputs, nodeMetadata)
-          : handler.execute(blockCtx, block, resolvedInputs)
+          : handler.execute(blockCtx, block, resolvedInputs, nodeMetadata)
       )
 
       const isStreamingExecution =

apps/sim/executor/handlers/generic/generic-handler.test.ts

@@ -1,7 +1,6 @@
 import '@sim/testing/mocks/executor'
 
 import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
-import { HarmonicBlock } from '@/blocks/blocks/harmonic'
 import { KnowledgeBlock } from '@/blocks/blocks/knowledge'
 import { getBlock } from '@/blocks/index'
 import { BlockType } from '@/executor/constants'
@@ -92,7 +91,7 @@ describe('GenericBlockHandler', () => {
     const inputs = { param1: 'resolvedValue1' }
     const expectedToolParams = {
       ...inputs,
-      _context: { workflowId: mockContext.workflowId },
+      _context: { workflowId: mockContext.workflowId, blockId: mockBlock.id },
     }
     const expectedOutput: any = { customResult: 'OK' }
 
@@ -491,47 +490,6 @@ describe('GenericBlockHandler', () => {
     expect(transform).toHaveBeenCalledTimes(1)
   })
 
-  it('does not expose an invalid resolved Harmonic batch value in handler errors', async () => {
-    const resolvedSecret = 'sk-live-invalid-json-secret'
-    mockBlock.metadata = { id: 'harmonic', name: 'Harmonic' }
-    mockGetBlock.mockReturnValue(HarmonicBlock)
-    mockExecuteTool.mockResolvedValue({
-      success: false,
-      error: 'Harmonic "personUrns" must be a JSON array',
-    })
-
-    const registry = new ResolvedSecretTraceRegistry([
-      {
-        name: 'BATCH_IDENTIFIERS',
-        plaintext: resolvedSecret,
-        encryptedValue: 'encrypted-batch-identifiers',
-      },
-    ])
-    registry.recordResolvedAtInputPath('BATCH_IDENTIFIERS', resolvedSecret, ['personUrns'])
-    registry.recordResolvedInputProjection(['personUrns'], resolvedSecret, '{{BATCH_IDENTIFIERS}}')
-    mockContext.resolvedSecretTraceRegistry = registry
-
-    let thrown: unknown
-    try {
-      await handler.execute(mockContext, mockBlock, {
-        operation: 'harmonic_batch_get_people',
-        oauthCredential: 'credential-id',
-        personUrns: resolvedSecret,
-      })
-    } catch (error) {
-      thrown = error
-    }
-
-    expect(thrown).toBeInstanceOf(Error)
-    expect((thrown as Error).message).toBe('Harmonic "personUrns" must be a JSON array')
-    expect((thrown as Error).message).not.toContain(resolvedSecret)
-    expect(mockExecuteTool).toHaveBeenCalledWith(
-      'some_custom_tool',
-      expect.objectContaining({ personUrns: resolvedSecret }),
-      { executionContext: mockContext }
-    )
-  })
-
   it('should throw error if the associated tool is not found', async () => {
     const inputs = { param1: 'value' }
 

apps/sim/executor/handlers/generic/generic-handler.ts

@@ -4,7 +4,7 @@ import { toError } from '@sim/utils/errors'
 import { isPlainRecord } from '@sim/utils/object'
 import { getBlock } from '@/blocks/index'
 import { isMcpTool } from '@/executor/constants'
-import type { BlockHandler, ExecutionContext } from '@/executor/types'
+import type { BlockHandler, BlockNodeMetadata, ExecutionContext } from '@/executor/types'
 import { readStatusCode } from '@/executor/utils/errors'
 import { prepareResolvedSecretProjectedInputs } from '@/executor/utils/resolved-secret-input-projection'
 import type { ResolvedSecretInputPath } from '@/executor/utils/resolved-secret-trace-registry'
@@ -152,7 +152,8 @@ export class GenericBlockHandler implements BlockHandler {
   async execute(
     ctx: ExecutionContext,
     block: SerializedBlock,
-    inputs: Record<string, any>
+    inputs: Record<string, any>,
+    nodeMetadata?: BlockNodeMetadata
   ): Promise<any> {
     const isMcp = block.config.tool ? isMcpTool(block.config.tool) : false
     let tool = null
@@ -301,6 +302,19 @@ export class GenericBlockHandler implements BlockHandler {
             userId: ctx.userId,
             isDeployedContext: ctx.isDeployedContext,
             enforceCredentialAccess: ctx.enforceCredentialAccess,
+            blockId: block.id,
+            /*
+             * The identity a `keyed` tool derives its provider idempotency token
+             * from. `executionOrder` is assigned before the block executor's
+             * retry wrapper, so it is identical across the transport loop, the
+             * hosted-key loop and a block-level retry — and it differs per loop
+             * iteration and per parallel branch, so five iterations paying five
+             * invoices derive five distinct tokens rather than collapsing into
+             * one the provider would dedupe down to a single payment.
+             */
+            ...(nodeMetadata?.executionOrder !== undefined
+              ? { invocationId: String(nodeMetadata.executionOrder) }
+              : {}),
           },
         },
         { executionContext: ctx }