Skip to content

Commit 2008383

Browse files
authored
fix(realtime): end a failed socket-token attempt promptly and honor Retry-After (#8952)
* fix(realtime): end the socket attempt when the token mint fails The Socket.IO auth callback never invoked cb when POST /api/auth/socket-token failed (429, 5xx, network), so the open transport sat until the realtime server's 45 s connectTimeout while the UI showed Reconnecting, and Retry-After was ignored. The client `timeout` cannot help: the Manager clears it when the transport opens, and resets its backoff on every open, so each retry ran at the 1 s floor. On a retryable mint failure, disconnect the socket and reconnect it on the existing auth backoff (1 s to 30 s, matching the reconnection config), never before the response's Retry-After. 401 keeps its cb({ token: null }) path. * fix(realtime): clear connected state and jitter Retry-After on token retry disconnect() on a socket that has not connected yet emits no 'disconnect' event. After a native reconnect is followed by a failed token mint, isConnected stayed true for the whole outage. Clear it, and the socket id, exactly as the 'disconnect' handler does. Spread a Retry-After-driven delay by up to +20% above the floor, so tabs behind one IP do not retry in the same second. * fix(realtime): tighten the token-retry comment * fix(realtime): ignore token failures from a superseded handshake and report persistent token outages
1 parent 0d0bab6 commit 2008383

1 file changed

Lines changed: 104 additions & 32 deletions

File tree

‎apps/sim/app/workspace/providers/socket-provider.tsx‎

Lines changed: 104 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,10 @@ import type {
2525
WorkflowRevertedBroadcast,
2626
WorkflowUpdatedBroadcast,
2727
} from '@sim/realtime-protocol/events'
28+
import { getErrorMessage } from '@sim/utils/errors'
2829
import { generateId } from '@sim/utils/id'
29-
import { backoffWithJitter } from '@sim/utils/retry'
30+
import { randomFloat } from '@sim/utils/random'
31+
import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry'
3032
import { useQueryClient } from '@tanstack/react-query'
3133
import { useParams } from 'next/navigation'
3234
import type { Socket } from 'socket.io-client'
@@ -70,6 +72,25 @@ const MAX_AUTH_RETRY_ATTEMPTS = 5
7072
const AUTH_RETRY_BASE_MS = 1000
7173
const AUTH_RETRY_MAX_MS = 30000
7274

75+
/**
76+
* Upper bound on an honored socket-token `Retry-After`. The token route's IP
77+
* bucket refills over 60 s, so the default 30 s cap would retry into a 429.
78+
*/
79+
const SOCKET_TOKEN_RETRY_AFTER_MAX_MS = 5 * 60_000
80+
81+
/** Up to +20% spread above a `Retry-After` floor, matching `backoffWithJitter`'s band. */
82+
const SOCKET_TOKEN_RETRY_AFTER_JITTER = 0.2
83+
84+
/** A socket-token mint that failed for a retryable reason, with the server's `Retry-After`. */
85+
class SocketTokenUnavailableError extends Error {
86+
readonly retryAfterMs: number | null
87+
88+
constructor(retryAfterMs: number | null) {
89+
super('Failed to generate socket token')
90+
this.retryAfterMs = retryAfterMs
91+
}
92+
}
93+
7394
function getTabSessionId(): string {
7495
if (typeof window === 'undefined') return ''
7596

@@ -185,6 +206,7 @@ export function SocketProvider({ children, user }: SocketProviderProps) {
185206
const joinRetryTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null)
186207
const authRetryAttemptsRef = useRef(0)
187208
const authRetryTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null)
209+
const tokenRetryAttemptsRef = useRef(0)
188210
const sessionRejectedRef = useRef(false)
189211
const connectFailureCountRef = useRef(0)
190212
const connectFailureReportedRef = useRef(false)
@@ -367,7 +389,9 @@ export function SocketProvider({ children, user }: SocketProviderProps) {
367389
if (res.status === 401) {
368390
throw new Error('Authentication required')
369391
}
370-
throw new Error('Failed to generate socket token')
392+
throw new SocketTokenUnavailableError(
393+
parseRetryAfter(res.headers.get('retry-after'), SOCKET_TOKEN_RETRY_AFTER_MAX_MS)
394+
)
371395
}
372396
const body = await res.json().catch(() => ({}))
373397
const token = body?.token
@@ -407,33 +431,6 @@ export function SocketProvider({ children, user }: SocketProviderProps) {
407431
timestamp: new Date().toISOString(),
408432
})
409433

410-
const socketInstance = io(socketUrl, {
411-
transports: ['websocket', 'polling'],
412-
withCredentials: true,
413-
reconnectionAttempts: Number.POSITIVE_INFINITY,
414-
reconnectionDelay: 1000,
415-
reconnectionDelayMax: 30000,
416-
timeout: 10000,
417-
auth: async (cb) => {
418-
// Reset per attempt so the flag describes THIS handshake only —
419-
// otherwise one early 401 would still be set when a later, unrelated
420-
// denial exhausts the retry budget.
421-
sessionRejectedRef.current = false
422-
try {
423-
const freshToken = await generateSocketToken()
424-
cb({ token: freshToken })
425-
} catch (error) {
426-
logger.error('Failed to generate fresh token for connection:', error)
427-
if (error instanceof Error && error.message === 'Authentication required') {
428-
// True auth failure - pass null token, server will reject with "Authentication required"
429-
sessionRejectedRef.current = true
430-
cb({ token: null })
431-
}
432-
// For server errors, don't call cb - connection will timeout and Socket.IO will retry
433-
}
434-
},
435-
})
436-
437434
/**
438435
* Reports a realtime connection that is not coming back.
439436
*
@@ -448,9 +445,11 @@ export function SocketProvider({ children, user }: SocketProviderProps) {
448445
* Fires at most once per socket instance, at the point the toast
449446
* appears, rather than once per retry.
450447
*
451-
* Called from `connect_error` and nowhere else. That is the only handler
452-
* that sees exactly one event per attempt: a failed *reconnect* also
453-
* emits the manager's `reconnect_error` (`manager.reconnect()` calls
448+
* Called once per failed attempt: from `connect_error`, and from the auth
449+
* callback when it ends an attempt itself because no token could be minted
450+
* (that attempt never reaches `connect_error`). `connect_error` is the only
451+
* socket handler that sees exactly one event per attempt: a failed
452+
* *reconnect* also emits the manager's `reconnect_error` (`manager.reconnect()` calls
454453
* `open()`, whose error path emits `error` — which the socket re-emits as
455454
* `connect_error` — and then emits `reconnect_error` itself), so counting
456455
* in both would advance twice per try and trip the threshold early.
@@ -473,11 +472,84 @@ export function SocketProvider({ children, user }: SocketProviderProps) {
473472
})
474473
}
475474

475+
// Each auth callback owns one handshake; a token request that outlives it must
476+
// not end the attempt that replaced it.
477+
let authAttemptGeneration = 0
478+
479+
const socketInstance = io(socketUrl, {
480+
transports: ['websocket', 'polling'],
481+
withCredentials: true,
482+
reconnectionAttempts: Number.POSITIVE_INFINITY,
483+
reconnectionDelay: 1000,
484+
reconnectionDelayMax: 30000,
485+
timeout: 10000,
486+
auth: async (cb) => {
487+
// Reset per attempt so the flag describes THIS handshake only —
488+
// otherwise one early 401 would still be set when a later, unrelated
489+
// denial exhausts the retry budget.
490+
sessionRejectedRef.current = false
491+
const attempt = ++authAttemptGeneration
492+
try {
493+
const freshToken = await generateSocketToken()
494+
cb({ token: freshToken })
495+
} catch (error) {
496+
logger.error('Failed to generate fresh token for connection:', error)
497+
if (error instanceof Error && error.message === 'Authentication required') {
498+
// True auth failure - pass null token, server will reject with "Authentication required"
499+
sessionRejectedRef.current = true
500+
cb({ token: null })
501+
return
502+
}
503+
if (
504+
attempt !== authAttemptGeneration ||
505+
!socketInstance.active ||
506+
socketInstance.connected
507+
) {
508+
return
509+
}
510+
511+
// Withholding `cb` would park the transport until the server's 45 s connect
512+
// timeout, and the Manager resets its backoff on every open, so pace it here.
513+
tokenRetryAttemptsRef.current += 1
514+
const retryAfterMs =
515+
error instanceof SocketTokenUnavailableError ? error.retryAfterMs : null
516+
const backoffMs = backoffWithJitter(tokenRetryAttemptsRef.current, null, {
517+
baseMs: AUTH_RETRY_BASE_MS,
518+
maxMs: AUTH_RETRY_MAX_MS,
519+
})
520+
// Spread tabs behind one IP that share the same 429 window.
521+
const delayMs =
522+
retryAfterMs === null
523+
? backoffMs
524+
: Math.max(
525+
backoffMs,
526+
retryAfterMs * (1 + SOCKET_TOKEN_RETRY_AFTER_JITTER * randomFloat())
527+
)
528+
socketInstance.disconnect()
529+
setIsConnected(false)
530+
setIsConnecting(false)
531+
setCurrentSocketId(null)
532+
setIsReconnecting(true)
533+
reportPersistentConnectFailure(getErrorMessage(error))
534+
logger.warn('Socket token unavailable, reconnecting after delay', {
535+
attempt: tokenRetryAttemptsRef.current,
536+
delayMs: Math.round(delayMs),
537+
})
538+
clearAuthRetryTimeout()
539+
authRetryTimeoutRef.current = setTimeout(() => {
540+
authRetryTimeoutRef.current = null
541+
socketInstance.connect()
542+
}, delayMs)
543+
}
544+
},
545+
})
546+
476547
socketInstance.on('connect', () => {
477548
setIsConnected(true)
478549
setIsConnecting(false)
479550
setIsReconnecting(false)
480551
authRetryAttemptsRef.current = 0
552+
tokenRetryAttemptsRef.current = 0
481553
connectFailureCountRef.current = 0
482554
connectFailureReportedRef.current = false
483555
clearAuthRetryTimeout()

0 commit comments

Comments
 (0)