Skip to content

Commit fe48d74

Browse files
committed
refactor(connectors): decouple rehydrate from fullSync deletion semantics
The transclusion refresh only needs re-hydration, but reusing the fullSync flag also activated its deletion-cleanup semantics — which bypass three previously unreachable safety guards (empty-listing wipe, listingCapped, and the >50% mass-deletion threshold). Since fullSync had no caller before this PR, the new 'Full resync' button would have exposed all three to any KB editor. Introduce a dedicated 'rehydrate' request that ONLY forces re-hydration + re-index of already-synced docs. Listing and deletion reconciliation are identical to a normal sync (all safety guards stay armed). fullSync's cleanup semantics remain dormant and untouched.
1 parent 5789a6a commit fe48d74

7 files changed

Lines changed: 51 additions & 27 deletions

File tree

apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -138,11 +138,11 @@ describe('Connector Manual Sync API Route', () => {
138138
expect(mockDispatchSync).toHaveBeenCalledWith('conn-456', {
139139
billingAttribution,
140140
requestId: 'test-req-id',
141-
fullSync: false,
141+
rehydrate: false,
142142
})
143143
})
144144

145-
it('dispatches a full resync when fullSync=true is set', async () => {
145+
it('dispatches a full resync when rehydrate=true is set', async () => {
146146
const billingAttribution = {
147147
actorUserId: 'external-admin',
148148
workspaceId: 'ws-1',
@@ -173,15 +173,15 @@ describe('Connector Manual Sync API Route', () => {
173173
'POST',
174174
undefined,
175175
{},
176-
'http://localhost:3000/api/knowledge/kb-123/connectors/conn-456/sync?fullSync=true'
176+
'http://localhost:3000/api/knowledge/kb-123/connectors/conn-456/sync?rehydrate=true'
177177
)
178178
const response = await POST(req as never, { params: mockParams })
179179

180180
expect(response.status).toBe(200)
181181
expect(mockDispatchSync).toHaveBeenCalledWith('conn-456', {
182182
billingAttribution,
183183
requestId: 'test-req-id',
184-
fullSync: true,
184+
rehydrate: true,
185185
})
186186
})
187187
})

apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route
2929
const parsed = await parseRequest(triggerKnowledgeConnectorSyncContract, request, context)
3030
if (!parsed.success) return parsed.response
3131
const { id: knowledgeBaseId, connectorId } = parsed.data.params
32-
const fullSync = parsed.data.query?.fullSync === 'true'
32+
const rehydrate = parsed.data.query?.rehydrate === 'true'
3333

3434
try {
3535
const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
@@ -83,7 +83,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route
8383
})
8484

8585
logger.info(
86-
`[${requestId}] Manual ${fullSync ? 'full ' : ''}sync triggered for connector ${connectorId}`
86+
`[${requestId}] Manual sync${rehydrate ? ' (full rehydrate)' : ''} triggered for connector ${connectorId}`
8787
)
8888

8989
captureServerEvent(
@@ -112,12 +112,12 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route
112112
knowledgeBaseName: writeCheck.knowledgeBase.name,
113113
connectorType: connectorRows[0].connectorType,
114114
connectorStatus: connectorRows[0].status,
115-
syncType: fullSync ? 'manual-full' : 'manual',
115+
syncType: rehydrate ? 'manual-rehydrate' : 'manual',
116116
},
117117
request,
118118
})
119119

120-
dispatchSync(connectorId, { billingAttribution, requestId, fullSync }).catch((error) => {
120+
dispatchSync(connectorId, { billingAttribution, requestId, rehydrate }).catch((error) => {
121121
logger.error(
122122
`[${requestId}] Failed to dispatch manual sync for connector ${connectorId}`,
123123
error

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -127,14 +127,14 @@ export function ConnectorsSection({
127127
}, [])
128128

129129
const handleSync = useCallback(
130-
(connectorId: string, fullSync = false) => {
130+
(connectorId: string, rehydrate = false) => {
131131
if (isSyncOnCooldown(connectorId)) return
132132

133133
syncTriggeredAt.current[connectorId] = Date.now()
134134
addToSet(setSyncingIds, connectorId)
135135

136136
triggerSync(
137-
{ knowledgeBaseId, connectorId, fullSync },
137+
{ knowledgeBaseId, connectorId, rehydrate },
138138
{
139139
onSuccess: () => {
140140
setError(null)
@@ -226,7 +226,7 @@ export function ConnectorsSection({
226226
isSyncPending={syncingIds.has(connector.id)}
227227
isUpdating={updatingIds.has(connector.id)}
228228
syncCooldown={isSyncOnCooldown(connector.id)}
229-
onSync={(fullSync) => handleSync(connector.id, fullSync)}
229+
onSync={(rehydrate) => handleSync(connector.id, rehydrate)}
230230
onTogglePause={() => handleTogglePause(connector)}
231231
onEdit={() => setEditingConnector(connector)}
232232
onDelete={() => setDeleteTarget(connector.id)}
@@ -285,7 +285,7 @@ interface ConnectorCardProps {
285285
isSyncPending: boolean
286286
isUpdating: boolean
287287
syncCooldown: boolean
288-
onSync: (fullSync?: boolean) => void
288+
onSync: (rehydrate?: boolean) => void
289289
onEdit: () => void
290290
onTogglePause: () => void
291291
onDelete: () => void

apps/sim/hooks/queries/kb/connectors.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -204,18 +204,18 @@ export function useDeleteConnector() {
204204
interface TriggerSyncParams {
205205
knowledgeBaseId: string
206206
connectorId: string
207-
/** Force a full resync (re-hydrate + re-index rendered content). Defaults to incremental. */
208-
fullSync?: boolean
207+
/** Force re-hydration + re-index of rendered content (the "Full resync" action). */
208+
rehydrate?: boolean
209209
}
210210

211211
async function triggerSync({
212212
knowledgeBaseId,
213213
connectorId,
214-
fullSync,
214+
rehydrate,
215215
}: TriggerSyncParams): Promise<void> {
216216
await requestJson(triggerKnowledgeConnectorSyncContract, {
217217
params: { id: knowledgeBaseId, connectorId },
218-
query: fullSync ? { fullSync: 'true' } : {},
218+
query: rehydrate ? { rehydrate: 'true' } : {},
219219
})
220220
}
221221

apps/sim/lib/api/contracts/knowledge/connectors.ts

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -152,13 +152,14 @@ export const deleteKnowledgeConnectorContract = defineRouteContract({
152152

153153
export const triggerKnowledgeConnectorSyncQuerySchema = z.object({
154154
/**
155-
* Force a full resync: re-list everything and, for connectors whose rendered
156-
* content can drift without a hash change (e.g. Confluence transclusions),
157-
* re-hydrate and re-index every document rather than only hash-changed ones.
158-
* Omitted performs the normal incremental, hash-gated sync. Modeled as a string
159-
* literal (not a boolean) so it round-trips cleanly through the query string.
155+
* Force re-hydration: for connectors whose rendered content can drift without a
156+
* hash change (e.g. Confluence transclusions), re-fetch and re-index every
157+
* already-synced document rather than only hash-changed ones. Listing and
158+
* deletion reconciliation are unchanged from a normal sync. Omitted performs the
159+
* normal hash-gated sync. Modeled as a string literal (not a boolean) so it
160+
* round-trips cleanly through the query string.
160161
*/
161-
fullSync: z.enum(['true', 'false']).optional(),
162+
rehydrate: z.enum(['true', 'false']).optional(),
162163
})
163164

164165
export const triggerKnowledgeConnectorSyncContract = defineRouteContract({

apps/sim/lib/knowledge/connectors/queue.ts

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,21 @@ const logger = createLogger('ConnectorSyncQueue')
1919
export interface ConnectorSyncPayload {
2020
connectorId: string
2121
fullSync?: boolean
22+
/**
23+
* Force re-hydration + re-indexing of already-synced documents for connectors
24+
* whose rendered content can drift without a hash change (see
25+
* `ConnectorMeta.rehydrateOnFullSync`). Unlike `fullSync`, this does NOT alter
26+
* listing or bypass any deletion-reconciliation safety guard.
27+
*/
28+
rehydrate?: boolean
2229
requestId: string
2330
billingAttribution: BillingAttributionSnapshot
2431
}
2532

2633
export interface DispatchSyncOptions {
2734
billingAttribution: BillingAttributionSnapshot
2835
fullSync?: boolean
36+
rehydrate?: boolean
2937
requestId?: string
3038
}
3139

@@ -46,13 +54,17 @@ export function assertConnectorSyncPayload(value: unknown): ConnectorSyncPayload
4654
if (value.fullSync !== undefined && typeof value.fullSync !== 'boolean') {
4755
throw new Error('Connector sync payload fullSync must be a boolean when provided')
4856
}
57+
if (value.rehydrate !== undefined && typeof value.rehydrate !== 'boolean') {
58+
throw new Error('Connector sync payload rehydrate must be a boolean when provided')
59+
}
4960
if (value.billingAttribution === undefined) {
5061
throw new Error('Connector sync payload requires billing attribution')
5162
}
5263

5364
return {
5465
connectorId: value.connectorId,
5566
fullSync: value.fullSync as boolean | undefined,
67+
rehydrate: value.rehydrate as boolean | undefined,
5668
requestId: value.requestId,
5769
billingAttribution: assertBillingAttributionSnapshot(value.billingAttribution),
5870
}
@@ -74,6 +86,7 @@ export async function dispatchSync(
7486
const payload = assertConnectorSyncPayload({
7587
connectorId,
7688
fullSync: options?.fullSync,
89+
rehydrate: options?.rehydrate,
7790
requestId,
7891
billingAttribution: options?.billingAttribution,
7992
})
@@ -147,6 +160,7 @@ export async function dispatchSync(
147160

148161
executeSync(connectorId, {
149162
fullSync: payload.fullSync,
163+
rehydrate: payload.rehydrate,
150164
billingAttribution: payload.billingAttribution,
151165
}).catch((error) => {
152166
logger.error(`Sync failed for connector ${connectorId}`, {

apps/sim/lib/knowledge/connectors/sync-engine.ts

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,11 @@ async function resolveAccessToken(
318318
*/
319319
export async function executeSync(
320320
connectorId: string,
321-
options: { billingAttribution: BillingAttributionSnapshot; fullSync?: boolean }
321+
options: {
322+
billingAttribution: BillingAttributionSnapshot
323+
fullSync?: boolean
324+
rehydrate?: boolean
325+
}
322326
): Promise<SyncResult> {
323327
const billingAttribution = assertBillingAttributionSnapshot(options?.billingAttribution)
324328
const result: SyncResult = {
@@ -438,11 +442,16 @@ export async function executeSync(
438442
isIncremental && connector.lastSyncAt ? new Date(connector.lastSyncAt) : undefined
439443

440444
/**
441-
* On an explicit full resync, re-hydrate and re-index connectors whose rendered
442-
* content can drift without a hash change (transclusions) — see
443-
* `ConnectorMeta.rehydrateOnFullSync`. Incremental syncs stay hash-gated.
445+
* Re-hydrate and re-index connectors whose rendered content can drift without a
446+
* hash change (transclusions) — see `ConnectorMeta.rehydrateOnFullSync`. Driven
447+
* by the dedicated `rehydrate` request (the "Full resync" action) or implied by a
448+
* true `fullSync`. This ONLY affects hydration/indexing — it does not change
449+
* listing or bypass any deletion-reconciliation guard. Incremental syncs of other
450+
* connectors stay hash-gated.
444451
*/
445-
const forceRehydrate = Boolean(options?.fullSync && connectorConfig.rehydrateOnFullSync)
452+
const forceRehydrate = Boolean(
453+
(options?.rehydrate || options?.fullSync) && connectorConfig.rehydrateOnFullSync
454+
)
446455

447456
for (let pageNum = 0; hasMore && pageNum < MAX_PAGES; pageNum++) {
448457
if (pageNum > 0 && connectorConfig.auth.mode === 'oauth') {

0 commit comments

Comments
 (0)