diff --git a/README.md b/README.md index a4457cdf0a..8b4ee9a525 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

- teable logo + teable logo

Turn your data into AI workflows and custom apps that 100% fit your business

@@ -30,6 +30,8 @@

+![Teable cover](static/assets/images/teable-cover.png) + ## Quick Guide 1. Want to see it in action? Try Teable on our hosted cloud at [teable.ai](https://teable.ai). @@ -74,12 +76,12 @@ More features land all the time — see the { + const field = createMultiTextField(); + + const OPERATORS = [ + { name: 'is', op: is.value }, + { name: 'isNot', op: isNot.value }, + { name: 'contains', op: contains.value }, + { name: 'doesNotContain', op: doesNotContain.value }, + ]; + + it.each(OPERATORS)('binds the value for `$name` instead of inlining it', ({ op }) => { + const { sql, bindings } = build(field, { + conjunction: 'and', + filterSet: [{ fieldId: field.id, operator: op, value: INJECTION }], + }); + + // The value is passed as a bound parameter — the raw payload never appears + // in the SQL text, so it cannot break out of the statement. + expect(sql).not.toContain('OR 1=1'); + // Rendered as a jsonpath containment predicate with a bound placeholder. + expect(sql).toContain('::jsonb @'); + expect(sql.endsWith('?)')).toBe(true); + + // The payload lives inside a jsonpath binding, quoted as a string literal. + const jsonPathBinding = bindings.find( + (b): b is string => typeof b === 'string' && b.includes('$[*]') + ); + expect(jsonPathBinding).toBeDefined(); + expect(jsonPathBinding).toContain('OR 1=1'); // present, but as data in the binding + }); + + it('multi-select fields still route to the safe json adapter', () => { + const ms = createMultiSelectField(); + expect(ms.dbFieldType).toBe(DbFieldType.Json); + const { sql } = build(ms, { + conjunction: 'and', + filterSet: [{ fieldId: ms.id, operator: contains.value, value: INJECTION }], + }); + expect(sql).not.toContain('OR 1=1'); + }); +}); diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts index 56d58aeab2..02e556ff2f 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts @@ -1,6 +1,9 @@ import type { IFilterOperator, ILiteralValue } from '@teable/core'; import type { Knex } from 'knex'; -import { escapeJsonbRegex } from '../../../../../utils/postgres-regex-escape'; +import { + escapeJsonPathRegexLiteral, + escapeJsonPathStringLiteral, +} from '../../../../../utils/postgres-regex-escape'; import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; @@ -12,7 +15,10 @@ export class MultipleStringCellValueFilterAdapter extends CellValueFilterPostgre _dbProvider: IDbProvider ): Knex.QueryBuilder { this.ensureLiteralValue(value, _operator); - builderClient.whereRaw(`${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ == "${value}")'`); + // Bind the jsonpath as a parameter; never concatenate the raw value into the + // SQL string (a single quote would otherwise break out and inject SQL). + const jsonPath = `$[*] ? (@ == "${escapeJsonPathStringLiteral(String(value))}")`; + builderClient.whereRaw(`${this.tableColumnRef}::jsonb @\\? ?`, [jsonPath]); return builderClient; } @@ -22,9 +28,8 @@ export class MultipleStringCellValueFilterAdapter extends CellValueFilterPostgre value: ILiteralValue, _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw( - `NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb @\\? '$[*] \\? (@ == "${value}")'` - ); + const jsonPath = `$[*] ? (@ == "${escapeJsonPathStringLiteral(String(value))}")`; + builderClient.whereRaw(`NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb @\\? ?`, [jsonPath]); return builderClient; } @@ -34,11 +39,9 @@ export class MultipleStringCellValueFilterAdapter extends CellValueFilterPostgre value: ILiteralValue, _dbProvider: IDbProvider ): Knex.QueryBuilder { - const escapedValue = escapeJsonbRegex(String(value)); this.ensureLiteralValue(value, _operator); - builderClient.whereRaw( - `${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ like_regex "${escapedValue}" flag "i")'` - ); + const jsonPath = `$[*] ? (@ like_regex "${escapeJsonPathRegexLiteral(String(value))}" flag "i")`; + builderClient.whereRaw(`${this.tableColumnRef}::jsonb @\\? ?`, [jsonPath]); return builderClient; } @@ -48,11 +51,9 @@ export class MultipleStringCellValueFilterAdapter extends CellValueFilterPostgre value: ILiteralValue, _dbProvider: IDbProvider ): Knex.QueryBuilder { - const escapedValue = escapeJsonbRegex(String(value)); this.ensureLiteralValue(value, _operator); - builderClient.whereRaw( - `NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb @\\? '$[*] \\? (@ like_regex "${escapedValue}" flag "i")'` - ); + const jsonPath = `$[*] ? (@ like_regex "${escapeJsonPathRegexLiteral(String(value))}" flag "i")`; + builderClient.whereRaw(`NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb @\\? ?`, [jsonPath]); return builderClient; } } diff --git a/apps/nestjs-backend/src/event-emitter/listeners/trash.listener.ts b/apps/nestjs-backend/src/event-emitter/listeners/trash.listener.ts index 0198cad2c2..61293dfbf7 100644 --- a/apps/nestjs-backend/src/event-emitter/listeners/trash.listener.ts +++ b/apps/nestjs-backend/src/event-emitter/listeners/trash.listener.ts @@ -110,14 +110,24 @@ export class TrashListener { if (!deletedTime) return; - await this.prismaService.trash.create({ - data: { + // Delete events can fire more than once for the same resource (e.g. soft delete + // followed by permanent delete), so recording the trash entry must be idempotent. + await this.prismaService.trash.upsert({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + resourceType_resourceId: { resourceType, resourceId }, + }, + create: { resourceId, resourceType, parentId, deletedTime, deletedBy: user?.id as string, }, + update: { + deletedTime, + deletedBy: user?.id as string, + }, }); } } diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-api.client.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-api.client.ts index 4366454fab..571e94d448 100644 --- a/apps/nestjs-backend/src/features/airtable-import/airtable-api.client.ts +++ b/apps/nestjs-backend/src/features/airtable-import/airtable-api.client.ts @@ -14,6 +14,11 @@ const minRequestIntervalMs = 220; const rateLimitWaitMs = 30_000; const maxRetries = 3; const recordsPageSize = 100; +// Hang-breaker: without it a silently dropped connection (proxy, LB idle +// reset) stalls the whole import forever. Used as a stall window — reset on +// every received body chunk — so it only fires when no data moves at all, +// and the abort is retried like any network error. +const requestTimeoutMs = 30_000; export class AirtableApiError extends Error { constructor( @@ -109,11 +114,26 @@ export class AirtableApiClient { while (true) { await this.throttle(); const accessToken = await this.getAccessToken(); + // Stall-based, not a whole-request deadline: the timer refreshes on + // every body chunk, so a slow-but-flowing large page is never killed — + // only a silently dropped connection (no headers or no data for the + // window) aborts and retries like any network error. + const controller = new AbortController(); + const stallTimer = setTimeout( + () => controller.abort(new Error(`no response data for ${requestTimeoutMs}ms`)), + requestTimeoutMs + ); let response: Response; + let bodyText: string; try { response = await fetch(`${airtableApiBaseUrl}${path}`, { headers: { Authorization: `Bearer ${accessToken}` }, + signal: controller.signal, }); + bodyText = await this.readBody(response, stallTimer); + if (response.ok) { + return JSON.parse(bodyText) as T; + } } catch (e) { if (attempt >= maxRetries) { throw new AirtableApiError( @@ -124,13 +144,11 @@ export class AirtableApiClient { await sleep(1000 * 2 ** attempt); attempt++; continue; + } finally { + clearTimeout(stallTimer); } - if (response.ok) { - return (await response.json()) as T; - } - - const errorBody = await this.parseError(response); + const errorBody = this.parseError(bodyText); if (response.status === 429 && attempt < maxRetries) { await sleep(rateLimitWaitMs); attempt++; @@ -152,9 +170,22 @@ export class AirtableApiClient { } } - private async parseError(response: Response): Promise<{ type?: string; message?: string }> { + /** Read the body chunkwise, refreshing the stall timer on every chunk. */ + private async readBody(response: Response, stallTimer: NodeJS.Timeout): Promise { + if (!response.body) { + return ''; + } + const chunks: Buffer[] = []; + for await (const chunk of response.body as unknown as AsyncIterable) { + stallTimer.refresh(); + chunks.push(Buffer.from(chunk)); + } + return Buffer.concat(chunks).toString('utf8'); + } + + private parseError(bodyText: string): { type?: string; message?: string } { try { - const body = (await response.json()) as { + const body = JSON.parse(bodyText) as { error?: string | { type?: string; message?: string }; }; if (typeof body.error === 'string') { diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-import.controller.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-import.controller.ts index e55e430b80..48d60e07b7 100644 --- a/apps/nestjs-backend/src/features/airtable-import/airtable-import.controller.ts +++ b/apps/nestjs-backend/src/features/airtable-import/airtable-import.controller.ts @@ -1,4 +1,12 @@ -import { BadRequestException, Body, Controller, Logger, Post, Res } from '@nestjs/common'; +import { + BadRequestException, + Body, + Controller, + ForbiddenException, + Logger, + Post, + Res, +} from '@nestjs/common'; import { type Action } from '@teable/core'; import { importAirtableAnalyzeRoSchema, @@ -11,6 +19,7 @@ import { Response as ExpressResponse } from 'express'; import { ClsService } from 'nestjs-cls'; import type { IClsStore } from '../../types/cls'; import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { TokenAccess } from '../auth/decorators/token.decorator'; import { PermissionService } from '../auth/permission.service'; import { AirtableApiError } from './airtable-api.client'; import { AirtableImportService } from './airtable-import.service'; @@ -38,11 +47,34 @@ export class AirtableImportController { private readonly cls: ClsService ) {} + /** + * Stored integration credentials are guarded by the user|integrations scope + * on /api/user-integrations; a PAT reaching these @TokenAccess routes with + * an integrationId must carry that same scope or it could use (and probe) + * the owner's stored Airtable OAuth token without ever being granted it. + */ + private async assertIntegrationScope(integrationId: string | undefined): Promise { + if (!integrationId) return; + const accessTokenId = this.cls.get('accessTokenId'); + if (!accessTokenId) return; + const { scopes } = await this.permissionService.getAccessToken(accessTokenId); + if (!scopes.includes('user|integrations')) { + throw new ForbiddenException( + 'The access token requires the user|integrations scope to use a stored integration credential.' + ); + } + } + + // Personal access tokens may call these routes (the guard blocks PATs on + // permissionless routes by default); the stream handler checks the concrete + // write target itself via validPermissions below. + @TokenAccess() @Post('import-airtable/analyze') async analyze( @Body(new ZodValidationPipe(importAirtableAnalyzeRoSchema)) analyzeRo: IImportAirtableAnalyzeRo ): Promise { + await this.assertIntegrationScope(analyzeRo.integrationId); try { return await this.airtableImportService.analyze(analyzeRo); } catch (error) { @@ -53,6 +85,7 @@ export class AirtableImportController { } } + @TokenAccess() @Post('import-airtable/stream') async importStream( @Body(new ZodValidationPipe(importAirtableRoSchema)) @@ -77,6 +110,7 @@ export class AirtableImportController { requiredPermissions, this.cls.get('accessTokenId') ); + await this.assertIntegrationScope(importAirtableRo.integrationId); const sseHeartbeatMs = 15_000; res.setHeader('Content-Type', 'text/event-stream'); diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-import.service.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-import.service.ts index 14d7a80752..fcfa5fd828 100644 --- a/apps/nestjs-backend/src/features/airtable-import/airtable-import.service.ts +++ b/apps/nestjs-backend/src/features/airtable-import/airtable-import.service.ts @@ -1,4 +1,4 @@ -import { Readable } from 'node:stream'; +import { pipeline, Readable, Transform } from 'node:stream'; import { BadRequestException, Inject, Injectable, Logger, Optional } from '@nestjs/common'; import type { IColumnMetaRo, @@ -71,6 +71,10 @@ export type IAirtableImportProgressReporter = (progress: IAirtableImportProgress const linkUpdateBatchSize = 100; const attachmentConcurrency = 3; +// Stall watchdog for one attachment transfer: aborts only when NO bytes move +// for this window (a silently dropped connection otherwise blocks the import +// forever). A slow-but-flowing transfer of any size is never interrupted. +const attachmentStallTimeoutMs = 60_000; const maxListRestarts = 2; const unknownErrorText = 'unknown error'; @@ -1014,30 +1018,83 @@ export class AirtableImportService { `attachment "${attachment.filename}" (${attachment.size} bytes) exceeds the ${maxSize}-byte upload limit` ); } - const response = await fetch(attachment.url); - if (!response.ok || !response.body) { - throw new Error(`download failed with status ${response.status}`); - } - const contentLength = attachment.size ?? Number(response.headers.get('content-length')); - if (!contentLength || !Number.isFinite(contentLength)) { - await response.body.cancel(); - throw new Error('attachment size is unknown'); - } - const contentType = - attachment.type ?? response.headers.get('content-type') ?? 'application/octet-stream'; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const stream = Readable.fromWeb(response.body as any); + // The watchdog resets on every chunk that moves through the transfer, so + // it fires when either side goes dead: a dropped download stops producing + // chunks, and a stalled storage upload stops consuming them (backpressure) + // — or, once the download has drained, stops the race below. + const controller = new AbortController(); + let onStall!: (error: Error) => void; + const stalled = new Promise((_, reject) => { + onStall = reject; + }); + // The watchdog can fire while the fetch below is still awaiting headers, + // before Promise.race() observes this promise — keep the rejection handled. + stalled.catch(() => undefined); + const monitor = new Transform({ + transform(chunk, _encoding, callback) { + stallTimer.refresh(); + callback(null, chunk); + }, + }); + // Same pre-consumer window: destroy(error) before pipeline() attaches + // would otherwise raise an unhandled 'error' event and crash the process. + monitor.on('error', () => undefined); + const stallTimer = setTimeout(() => { + const error = new Error(`attachment transfer stalled for ${attachmentStallTimeoutMs}ms`); + controller.abort(error); + monitor.destroy(error); + onStall(error); + }, attachmentStallTimeoutMs); try { - return await this.attachmentsService.uploadFromStream( - stream, + const response = await fetch(attachment.url, { signal: controller.signal }); + if (!response.ok || !response.body) { + // Release the socket instead of holding it until GC (expired CDN + // links make this a common path). + await response.body?.cancel().catch(() => undefined); + throw new Error(`download failed with status ${response.status}`); + } + const rawContentLength = response.headers.get('content-length'); + const contentLength = + attachment.size ?? (rawContentLength === null ? Number.NaN : Number(rawContentLength)); + if (!Number.isFinite(contentLength) || contentLength < 0) { + await response.body.cancel(); + throw new Error('attachment size is unknown'); + } + const contentType = + attachment.type ?? response.headers.get('content-type') ?? 'application/octet-stream'; + // pipeline() into the monitor rather than a bare on('data') listener: a + // data listener would start the flow before uploadFromStream attaches + // its consumer and silently drop those chunks, while the Transform + // buffers them with backpressure. pipeline also tears down the CDN + // socket when either side fails. A download error is surfaced through + // onStall so the transfer fails immediately with the real cause even if + // axios has not attached its consumer yet (it would otherwise hang on + // the destroyed stream until the watchdog mislabels it as a stall). + // eslint-disable-next-line @typescript-eslint/no-explicit-any + pipeline(Readable.fromWeb(response.body as any), monitor, (error) => { + if (error) onStall(error); + }); + // The signal also cancels the storage PUT when the watchdog fires after + // the download has drained (destroying an ended stream cannot). + const upload = this.attachmentsService.uploadFromStream( + monitor, { filename: attachment.filename, contentType, contentLength }, - UploadType.Table + UploadType.Table, + { signal: controller.signal } ); - } catch (error) { - // Release the still-open CDN socket on any upload failure (e.g. the - // size-limit check) instead of leaking it. - stream.destroy(); - throw error; + // The watchdog may win the race; the losing upload's late rejection + // must not surface as an unhandled rejection. + upload.catch(() => undefined); + try { + return await Promise.race([upload, stalled]); + } catch (error) { + // Release the still-open CDN socket on any upload failure (e.g. the + // size-limit check) instead of leaking it. + monitor.destroy(); + throw error; + } + } finally { + clearTimeout(stallTimer); } } diff --git a/apps/nestjs-backend/src/features/attachments/attachments.service.ts b/apps/nestjs-backend/src/features/attachments/attachments.service.ts index 976c50f427..a68173ec20 100644 --- a/apps/nestjs-backend/src/features/attachments/attachments.service.ts +++ b/apps/nestjs-backend/src/features/attachments/attachments.service.ts @@ -141,7 +141,6 @@ export class AttachmentsService { if (contentLength > MAX_FILE_SIZE) { this.throwFileSizeExceeded(MAX_FILE_SIZE); } - const hash = presignedParams.hash; const dir = StorageAdapter.getDir(type); const bucket = StorageAdapter.getBucket(type); const res = await this.storageAdapter.presigned(bucket, dir, { @@ -150,7 +149,7 @@ export class AttachmentsService { const { path, token } = res; await this.cacheService.set( `attachment:signature:${token}`, - { path, bucket, hash }, + { path, bucket }, signatureRo.expiresIn ?? second(this.storageConfig.tokenExpireIn) ); return res; @@ -299,7 +298,8 @@ export class AttachmentsService { async uploadFromStream( stream: Readable, params: { filename: string; contentType: string; contentLength: number }, - uploadType: UploadType = UploadType.Table + uploadType: UploadType = UploadType.Table, + options?: { signal?: AbortSignal } ): Promise { const MAX_FILE_SIZE = this.thresholdConfig.maxOpenapiAttachmentUploadSize; const { filename, contentType, contentLength } = params; @@ -313,7 +313,7 @@ export class AttachmentsService { contentType, internal: true, }); - await this.uploadStreamToStorage(url, stream, contentType, contentLength); + await this.uploadStreamToStorage(url, stream, contentType, contentLength, options?.signal); return await this.notifyToAttachmentItem(token, filename); } @@ -486,7 +486,8 @@ export class AttachmentsService { url: string, stream: Readable, contentType: string, - contentLength: number + contentLength: number, + signal?: AbortSignal ): Promise { try { await axios.put(url, stream, { @@ -494,6 +495,7 @@ export class AttachmentsService { 'Content-Type': getSafeUploadContentType(contentType), 'Content-Length': contentLength, }, + signal, }); } catch (error) { stream.destroy(); diff --git a/apps/nestjs-backend/src/features/auth/permission.service.spec.ts b/apps/nestjs-backend/src/features/auth/permission.service.spec.ts index 5700cf9a52..c710e53969 100644 --- a/apps/nestjs-backend/src/features/auth/permission.service.spec.ts +++ b/apps/nestjs-backend/src/features/auth/permission.service.spec.ts @@ -303,6 +303,62 @@ describe('PermissionService', () => { }); }); + describe('getAccessToken (GHSA-c57x: OAuth scope escalation)', () => { + it('does NOT add base|read_all to an OAuth token that did not consent to it', async () => { + // A user consented only to table|read for this OAuth client. + prismaServiceMock.accessToken.findFirstOrThrow.mockResolvedValue({ + scopes: JSON.stringify(['table|read'] satisfies Action[]), + spaceIds: null, + baseIds: null, + clientId: 'cltoauthclient00', // IdPrefix.OAuthClient + userId: 'usrxxxxxxxx', + hasFullAccess: null, + } as any); + // OAuth collaborator resolution goes through txClient().collaborator. + prismaServiceMock.txClient.mockReturnValue(prismaServiceMock as any); + prismaServiceMock.collaborator.findMany.mockResolvedValue([]); + + const result = await service.getAccessToken('actxxxxxxxx'); + + // The token must not gain read access it was never approved for. + expect(result.scopes).not.toContain('base|read_all'); + expect(result.scopes).toEqual(['table|read']); + }); + + it('preserves base|read_all for an OAuth token that DID consent to it', async () => { + prismaServiceMock.accessToken.findFirstOrThrow.mockResolvedValue({ + scopes: JSON.stringify(['table|read', 'base|read_all'] satisfies Action[]), + spaceIds: null, + baseIds: null, + clientId: 'cltoauthclient00', + userId: 'usrxxxxxxxx', + hasFullAccess: null, + } as any); + prismaServiceMock.txClient.mockReturnValue(prismaServiceMock as any); + prismaServiceMock.collaborator.findMany.mockResolvedValue([]); + + const result = await service.getAccessToken('actxxxxxxxx'); + + expect(result.scopes).toContain('base|read_all'); + }); + + it('does NOT add base|read_all to a regular (non-OAuth) PAT', async () => { + prismaServiceMock.accessToken.findFirstOrThrow.mockResolvedValue({ + scopes: JSON.stringify(['table|read'] satisfies Action[]), + spaceIds: null, + baseIds: null, + clientId: null, // regular personal access token + userId: 'usrxxxxxxxx', + hasFullAccess: null, + } as any); + + const result = await service.getAccessToken('actxxxxxxxx'); + + expect(result.scopes).not.toContain('base|read_all'); + expect(result.scopes).toEqual(['table|read']); + }); + }); + describe('getPermissions', () => { it('should return permissions for a user', async () => { const resourceId = 'bsexxxxxx'; diff --git a/apps/nestjs-backend/src/features/auth/permission.service.ts b/apps/nestjs-backend/src/features/auth/permission.service.ts index 9863f83e5a..5f2c9fdebc 100644 --- a/apps/nestjs-backend/src/features/auth/permission.service.ts +++ b/apps/nestjs-backend/src/features/auth/permission.service.ts @@ -168,8 +168,11 @@ export class PermissionService { if (clientId && clientId.startsWith(IdPrefix.OAuthClient)) { const { spaceIds: spaceIdsByOAuth, baseIds: baseIdsByOAuth } = await this.getOAuthAccessBy(userId); + // Only expose base|read_all when the user actually consented to it. + // Previously it was concatenated unconditionally, granting third-party + // OAuth apps broader read access than the scopes they were approved for. return { - scopes: scopes.concat('base|read_all'), + scopes, spaceIds: spaceIdsByOAuth, baseIds: baseIdsByOAuth, }; diff --git a/apps/nestjs-backend/src/features/base/base-controller.spec.ts b/apps/nestjs-backend/src/features/base/base-controller.spec.ts index 3221db4933..b4504f682c 100644 --- a/apps/nestjs-backend/src/features/base/base-controller.spec.ts +++ b/apps/nestjs-backend/src/features/base/base-controller.spec.ts @@ -1,11 +1,12 @@ import { BaseController } from './base.controller'; -const createController = (v2Reason: string | undefined) => { +const createController = (useV2Export: boolean) => { const baseService = { getBaseById: vi.fn().mockResolvedValue({ id: 'bseTest', - v2Status: v2Reason ? { useV2: true, reason: v2Reason } : undefined, + v2Status: { useV2: true, reason: useV2Export ? 'new_base' : 'space_feature' }, }), + shouldUseV2BaseExport: vi.fn().mockResolvedValue(useV2Export), }; const baseExportService = { exportBaseZip: vi.fn().mockResolvedValue('v1-export'), @@ -34,8 +35,24 @@ const createController = (v2Reason: string | undefined) => { describe('BaseController', () => { describe('exportBase', () => { - it('uses the v2 exporter for v2-created bases', async () => { - const { controller, baseExportService, baseExportV2Service } = createController('new_base'); + it('uses the v2 exporter for physical v2 bases', async () => { + const { controller, baseExportService, baseExportV2Service, baseService } = + createController(true); + + await expect(controller.exportBase('bseTest')).resolves.toBe('v2-export'); + + expect(baseService.shouldUseV2BaseExport).toHaveBeenCalledWith('bseTest'); + expect(baseExportV2Service.exportBaseZip).toHaveBeenCalledWith('bseTest', true); + expect(baseExportService.exportBaseZip).not.toHaveBeenCalled(); + }); + + it('uses the v2 exporter even when FORCE_V2_ALL overrides canary reason', async () => { + const { controller, baseExportService, baseExportV2Service, baseService } = + createController(true); + baseService.getBaseById.mockResolvedValue({ + id: 'bseTest', + v2Status: { useV2: true, reason: 'env_force_v2_all' }, + }); await expect(controller.exportBase('bseTest')).resolves.toBe('v2-export'); @@ -44,8 +61,7 @@ describe('BaseController', () => { }); it('keeps rollout-only v2 decisions on the legacy exporter', async () => { - const { controller, baseExportService, baseExportV2Service } = - createController('space_feature'); + const { controller, baseExportService, baseExportV2Service } = createController(false); await expect(controller.exportBase('bseTest', '0')).resolves.toBe('v1-export'); diff --git a/apps/nestjs-backend/src/features/base/base.controller.ts b/apps/nestjs-backend/src/features/base/base.controller.ts index d900ae4977..e3a70d48ce 100644 --- a/apps/nestjs-backend/src/features/base/base.controller.ts +++ b/apps/nestjs-backend/src/features/base/base.controller.ts @@ -510,8 +510,8 @@ export class BaseController { async exportBase(@Param('baseId') baseId: string, @Query('includeData') includeData?: string) { const includeDataValue = includeData === undefined ? true : !['false', '0'].includes(includeData.toLowerCase()); - const base = await this.baseService.getBaseById(baseId); - if (base.v2Status?.reason === 'new_base') { + await this.baseService.getBaseById(baseId); + if (await this.baseService.shouldUseV2BaseExport(baseId)) { return await this.baseExportV2Service.exportBaseZip(baseId, includeDataValue); } return await this.baseExportService.exportBaseZip(baseId, includeDataValue); @@ -548,11 +548,10 @@ export class BaseController { res.on('close', () => clearInterval(heartbeat)); try { - const base = await this.baseService.getBaseById(baseId); - const exporter = - base.v2Status?.reason === 'new_base' - ? this.baseExportV2Service.exportBaseZip.bind(this.baseExportV2Service) - : this.baseExportService.exportBaseZip.bind(this.baseExportService); + await this.baseService.getBaseById(baseId); + const exporter = (await this.baseService.shouldUseV2BaseExport(baseId)) + ? this.baseExportV2Service.exportBaseZip.bind(this.baseExportV2Service) + : this.baseExportService.exportBaseZip.bind(this.baseExportService); const result = await exporter(baseId, includeDataValue, (phase, detail, event) => { sendEvent({ type: 'progress', diff --git a/apps/nestjs-backend/src/features/base/base.service.ts b/apps/nestjs-backend/src/features/base/base.service.ts index d9be5b57bb..3a34995deb 100644 --- a/apps/nestjs-backend/src/features/base/base.service.ts +++ b/apps/nestjs-backend/src/features/base/base.service.ts @@ -247,6 +247,18 @@ export class BaseService { }; } + /** + * Export/import zip format follows physical schema (v2Enabled), not canary reason. + * FORCE_V2_ALL may report reason env_force_v2_all while the base is still physically V2. + */ + async shouldUseV2BaseExport(baseId: string): Promise { + const base = await this.prismaService.base.findFirst({ + select: { v2Enabled: true }, + where: { id: baseId, deletedTime: null }, + }); + return Boolean(base?.v2Enabled); + } + async getBaseById(baseId: string) { const base = await this.prismaService.base .findFirstOrThrow({ @@ -421,8 +433,6 @@ export class BaseService { lastModifiedBy: userId, }, }); - - return base; } catch (error) { await this.prismaService.base.update({ where: { id: base.id }, @@ -433,6 +443,9 @@ export class BaseService { }); throw error; } + + await this.markBaseVisited(base.id, spaceId); + return base; } async updateBase(baseId: string, updateBaseRo: IUpdateBaseRo) { @@ -655,7 +668,7 @@ export class BaseService { }, }) .catch((error) => { - this.logger.warn(`Failed to seed last-visit for duplicated base ${baseId}: ${error}`); + this.logger.warn(`Failed to seed last-visit for base ${baseId}: ${error}`); }); } @@ -816,6 +829,7 @@ export class BaseService { templateId, fromBaseId, }); + await this.markBaseVisited(result.id, spaceId); return result; } diff --git a/apps/nestjs-backend/src/features/canary/canary.service.spec.ts b/apps/nestjs-backend/src/features/canary/canary.service.spec.ts index a098e02046..b339a2a932 100644 --- a/apps/nestjs-backend/src/features/canary/canary.service.spec.ts +++ b/apps/nestjs-backend/src/features/canary/canary.service.spec.ts @@ -51,7 +51,7 @@ describe('CanaryService', () => { expect(cls.get).not.toHaveBeenCalled(); }); - it('reports new_base for a marked new base even when force v2 all is enabled', async () => { + it('reports env_force_v2_all over new_base when force v2 all is enabled', async () => { process.env.ENABLE_CANARY_FEATURE = 'true'; process.env.FORCE_V2_ALL = 'true'; const { service, settingService, cls } = createService({ @@ -64,7 +64,7 @@ describe('CanaryService', () => { 'createRecord' ); - expect(decision).toEqual({ useV2: true, reason: 'new_base' }); + expect(decision).toEqual({ useV2: true, reason: 'env_force_v2_all' }); expect(settingService.getSetting).not.toHaveBeenCalled(); expect(cls.get).not.toHaveBeenCalled(); }); @@ -130,4 +130,15 @@ describe('CanaryService', () => { expect(decision).toEqual({ useV2: true, reason: 'header_override' }); }); + + it('skips loading canary spaceIds when FORCE_V2_ALL is enabled', async () => { + process.env.ENABLE_CANARY_FEATURE = 'true'; + process.env.FORCE_V2_ALL = 'true'; + const { service, settingService } = createService({ + config: { enabled: true, spaceIds: ['spc1'], forceV2All: false }, + }); + + await expect(service.isSpaceInCanary('spc1')).resolves.toBe(false); + expect(settingService.getSetting).not.toHaveBeenCalled(); + }); }); diff --git a/apps/nestjs-backend/src/features/canary/canary.service.ts b/apps/nestjs-backend/src/features/canary/canary.service.ts index 50d223b6ba..f27ed3a726 100644 --- a/apps/nestjs-backend/src/features/canary/canary.service.ts +++ b/apps/nestjs-backend/src/features/canary/canary.service.ts @@ -38,8 +38,8 @@ export class CanaryService { } /** - * Check if V2 is forced globally via environment variable (FORCE_V2_ALL=true) - * This is the fallback priority for bases that are not explicitly marked as new-base V2. + * Check if V2 is forced globally via environment variable (FORCE_V2_ALL=true). + * Highest priority for both routing and attribution reason. */ isForceV2AllEnabled(): boolean { return process.env.FORCE_V2_ALL === 'true'; @@ -59,13 +59,20 @@ export class CanaryService { /** * Check if a space is in canary release * Priority: - * 1. If canary feature is disabled globally, return false - * 2. If x-canary header is set, use header value (true/false) - * 3. Otherwise, check space against configured spaceIds + * 1. FORCE_V2_ALL — canary list unused; skip loading large spaceIds config + * 2. If canary feature is disabled globally, return false + * 3. If x-canary header is set, use header value (true/false) + * 4. Otherwise, check space against configured spaceIds * * @param spaceId - The space ID to check (caller should provide this from their context) */ async isSpaceInCanary(spaceId: string): Promise { + // Under FORCE_V2_ALL routing never consults canary spaceIds — avoid reading + // the (often huge) setting blob just for isCanary UI badges. + if (this.isForceV2AllEnabled()) { + return false; + } + // Check if canary feature is enabled globally if (!this.isCanaryFeatureEnabled()) { return false; @@ -133,13 +140,21 @@ export class CanaryService { } /** - * New bases are V2-first regardless of canary, request headers, or rollout config. + * Resolve V2 for a base-scoped feature. + * Priority: + * 1. FORCE_V2_ALL env (highest — reason env_force_v2_all) + * 2. Base marked v2Enabled (reason new_base) + * 3. Space/canary/header/config decisions * Unsupported features still do not call this path because they have no @UseV2Feature marker. */ async shouldUseV2ForBaseWithReason( base: IBaseV2DecisionContext | null | undefined, feature: V2Feature ): Promise { + if (this.isForceV2AllEnabled()) { + return { useV2: true, reason: 'env_force_v2_all' }; + } + if (base?.v2Enabled) { return { useV2: true, reason: 'new_base' }; } diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts index 3adfd726c6..1fb9624543 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts @@ -818,7 +818,7 @@ export class FieldSupplementService { const mergedLookupOptions = newLookupOptions && oldLookupOptions ? { ...oldLookupOptions, ...newLookupOptions } - : (newLookupOptions ?? oldLookupOptions); + : newLookupOptions ?? oldLookupOptions; return this.prepareLookupField(tableId, { ...oldFieldVo, @@ -1743,11 +1743,7 @@ export class FieldSupplementService { Boolean(oldFieldVo.isLookup) && fieldRo.lookupOptions !== undefined); if (isLookupField && hasMajorChange) { - return this.prepareUpdateLookupField( - tableId, - { ...fieldRo, isLookup: true }, - oldFieldVo - ); + return this.prepareUpdateLookupField(tableId, { ...fieldRo, isLookup: true }, oldFieldVo); } switch (fieldRo.type) { diff --git a/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.spec.ts b/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.spec.ts index f315b09311..88718309d4 100644 --- a/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.spec.ts +++ b/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.spec.ts @@ -153,9 +153,11 @@ describe('MailSenderService transporter pooling', () => { return t as any; }); - // 51 distinct configs: the 51st set() LRU-evicts the first entry before its send runs + // max + 1 distinct configs: the last set() LRU-evicts the first entry before its send runs + const cacheMax = (service as unknown as { transporterCache: { max: number } }).transporterCache + .max; const results = await Promise.all( - Array.from({ length: 51 }, (_, i) => + Array.from({ length: cacheMax + 1 }, (_, i) => service.sendMailByConfig( { to: `user${i}@example.com` }, { ...smtpConfig, auth: { user: `user-${i}`, pass: 'pass' } } @@ -163,7 +165,7 @@ describe('MailSenderService transporter pooling', () => { ) ); - expect(results).toHaveLength(51); + expect(results).toHaveLength(cacheMax + 1); expect(transporters[0].sendMail).toHaveBeenCalledTimes(1); expect(transporters[0].close).toHaveBeenCalledTimes(1); }); diff --git a/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.ts b/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.ts index d1689959f6..811fbf2506 100644 --- a/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.ts +++ b/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.ts @@ -14,6 +14,7 @@ import { } from '@teable/openapi'; import { isString } from 'lodash'; import { LRUCache } from 'lru-cache'; +import ms from 'ms'; import { I18nService } from 'nestjs-i18n'; import type { Transporter } from 'nodemailer'; import { createTransport } from 'nodemailer'; @@ -42,8 +43,8 @@ export class MailSenderService implements OnModuleDestroy { // Connection pool per SMTP config: transporter objects hold live sockets, so this // must live in process memory (not in the shared cache service) private readonly transporterCache = new LRUCache({ - max: 50, - ttl: 30 * 60 * 1000, + max: 200, + ttl: ms('30m'), updateAgeOnGet: true, // Closing a nodemailer pool rejects its queued sends, so busy entries are // only marked here; the last send's finally closes them once drained @@ -153,6 +154,11 @@ export class MailSenderService implements OnModuleDestroy { const key = this.getTransporterCacheKey(config); let entry = this.transporterCache.get(key); if (!entry) { + // Don't log the key: it hashes the password, so even a prefix is an + // offline-testable verifier + this.logger.debug( + `Transporter cache miss (host=${config.host}, size=${this.transporterCache.size})` + ); const created = await this.createTransporter(config); // A concurrent request may have populated the same key while we created ours; // reuse theirs so set() does not dispose (close) a transporter that is mid-send diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts index 99aa003341..c8eba3d8e2 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts @@ -10,6 +10,7 @@ import { CreateRecordResult, CreateRecordsResult, DuplicateRecordResult, + FieldId, ListTableRecordsQuery, ListTableRecordsResult, UpdateRecordResult, @@ -46,6 +47,7 @@ describe('RecordOpenApiV2Service', () => { const cacheSetDetail = vi.fn(); const getDataDatabaseForTable = vi.fn(); const dataPrismaForTable = vi.fn(); + const resolveForRecordSearch = vi.fn(); const assertTableRecordWritable = vi.fn(); let service: RecordOpenApiV2Service; @@ -175,6 +177,7 @@ describe('RecordOpenApiV2Service', () => { url: 'postgresql://meta', isMetaFallback: true, }); + resolveForRecordSearch.mockResolvedValue(undefined); commandExecute.mockResolvedValue({ isErr: () => false, value: UpdateRecordsResult.create(2, []), @@ -211,7 +214,9 @@ describe('RecordOpenApiV2Service', () => { emitAtomic: vi.fn().mockResolvedValue(undefined), withOperation: vi.fn().mockImplementation((_operation, fn: () => Promise) => fn()), } as never, - { assertTableRecordWritable } as never + { assertTableRecordWritable } as never, + undefined, + { resolveForRecordSearch } as never ); }); @@ -427,6 +432,36 @@ describe('RecordOpenApiV2Service', () => { ]); }); + it('passes a meta-backed generated search vector access path into v2 list queries', async () => { + const tableId = `tbl${'c'.repeat(16)}`; + const search = ['order 123'] as [string]; + const accessPath = { + kind: 'generated_tsvector' as const, + generatedColumnName: '__tqops_search_vector', + languageConfig: 'simple', + searchScope: 'all_fields' as const, + coveredFieldIds: [FieldId.create(noteFieldId)._unsafeUnwrap()], + }; + resolveForRecordSearch.mockResolvedValueOnce(accessPath); + + await service.getRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + search, + skip: 0, + take: 2, + }); + + expect(resolveForRecordSearch).toHaveBeenCalledWith({ + container: { resolve }, + tableId, + search, + }); + expect(getDocIdsByQuery).not.toHaveBeenCalled(); + const query = execute.mock.calls[0]?.[1]; + expect(query).toBeInstanceOf(ListTableRecordsQuery); + expect((query as ListTableRecordsQuery).recordSearchAccessPath).toBe(accessPath); + }); + it('normalizes legacy ISO date filters for v2 date comparisons', async () => { const tableId = `tbl${'c'.repeat(16)}`; const dateFieldId = `fld${'d'.repeat(16)}`; diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts index 0f1e609438..e0cfcf85df 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts @@ -68,6 +68,7 @@ import { type IPasteCommandInput, type IQueryBus, type IRecordReadQuerySource, + type IRecordSearchAccessPath, type PasteStreamResult, type RecordFilter, type RecordFilterDateValue, @@ -94,6 +95,7 @@ import { createFieldInstanceByVo } from '../../field/model/factory'; import { TableService } from '../../table/table.service'; import { SpaceDataDbMigrationGuardService } from '../../space/space-data-db-migration-guard.service'; import { buildUndoRedoEnginePreferenceKey } from '../../undo-redo/open-api/undo-redo-engine-preference'; +import { TableQuerySearchVectorRuntimeService } from '../../v2/table-query-search-vector-runtime.service'; import { V2ContainerService } from '../../v2/v2-container.service'; import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; import { convertLinkPasteCellValue } from '../paste-link-cell-value'; @@ -153,7 +155,9 @@ export class RecordOpenApiV2Service { private readonly dataDbClientManager: DataDbClientManager, private readonly audit: AuditScope, private readonly spaceDataDbMigrationGuard: SpaceDataDbMigrationGuardService, - @Optional() private readonly attachmentsService?: AttachmentsService + @Optional() private readonly attachmentsService?: AttachmentsService, + @Optional() + private readonly tableQuerySearchVectorRuntimeService?: TableQuerySearchVectorRuntimeService ) {} private async assertTableRecordWritable(tableId: string): Promise { @@ -301,7 +305,18 @@ export class RecordOpenApiV2Service { order: item.order, })); const normalizedGroupBy = effectiveQuery.groupBy?.map((item) => item.fieldId); - const queryExtra = await this.loadQueryExtraWithTrace(context, tableId, effectiveQuery); + const recordSearchAccessPath = await this.resolveRecordSearchAccessPath( + context, + tableId, + container, + effectiveQuery.search + ); + const queryExtra = await this.loadQueryExtraWithTrace( + context, + tableId, + effectiveQuery, + recordSearchAccessPath + ); const queryBus = container.resolve(v2CoreTokens.queryBus); const pageResult = await this.withRecordReadSpan( @@ -344,7 +359,9 @@ export class RecordOpenApiV2Service { }, context, queryBus, - recordReadQuerySource ? { recordReadQuerySource } : undefined + recordReadQuerySource || recordSearchAccessPath + ? { recordReadQuerySource, recordSearchAccessPath } + : undefined ) ); const orderedRecords = pageResult.records; @@ -639,7 +656,10 @@ export class RecordOpenApiV2Service { input: IListTableRecordsQueryInput, context: IExecutionContext, queryBus: IQueryBus, - options?: { recordReadQuerySource?: IRecordReadQuerySource } + options?: { + recordReadQuerySource?: IRecordReadQuerySource; + recordSearchAccessPath?: IRecordSearchAccessPath; + } ): Promise<{ records: Array<{ id: string; fields: Record }>; pagination: { hasMore: boolean }; @@ -688,6 +708,32 @@ export class RecordOpenApiV2Service { }; } + private async resolveRecordSearchAccessPath( + context: IExecutionContext, + tableId: string, + container: DependencyContainer, + search: IGetRecordsRo['search'] + ): Promise { + const runtimeService = this.tableQuerySearchVectorRuntimeService; + if (!runtimeService) { + return undefined; + } + + return await this.withRecordReadSpan( + context, + 'teable.RecordOpenApiV2Service.resolveRecordSearchAccessPath', + { + 'record.read.has_search': Boolean(search), + }, + () => + runtimeService.resolveForRecordSearch({ + container, + tableId, + search, + }) + ); + } + private sanitizeReadableSortAndGroup( query: Pick, enabledFieldIds?: ReadonlyArray @@ -709,10 +755,20 @@ export class RecordOpenApiV2Service { }; } - private shouldLoadQueryExtra(query: IGetRecordsRo): boolean { + private shouldLoadQueryExtra( + query: IGetRecordsRo, + recordSearchAccessPath?: IRecordSearchAccessPath + ): boolean { if (query.includeQueryExtra === false) { return false; } + if ( + recordSearchAccessPath?.kind === 'generated_tsvector' && + query.search && + query.includeQueryExtra !== true + ) { + return false; + } const hasQueryExtraSource = Boolean( query.search || query.groupBy?.length || query.collapsedGroupIds?.length ); @@ -733,9 +789,10 @@ export class RecordOpenApiV2Service { private async loadQueryExtraWithTrace( context: IExecutionContext, tableId: string, - query: IGetRecordsRo + query: IGetRecordsRo, + recordSearchAccessPath?: IRecordSearchAccessPath ): Promise { - const shouldLoad = this.shouldLoadQueryExtra(query); + const shouldLoad = this.shouldLoadQueryExtra(query, recordSearchAccessPath); return await this.withRecordReadSpan( context, @@ -744,6 +801,7 @@ export class RecordOpenApiV2Service { 'record.read.query_extra_enabled': shouldLoad, 'record.read.include_query_extra': query.includeQueryExtra !== false, 'record.read.has_search': Boolean(query.search), + 'record.read.search_access_path': recordSearchAccessPath?.kind ?? 'default', 'record.read.group_by_count': query.groupBy?.length ?? 0, 'record.read.collapsed_group_count': query.collapsedGroupIds?.length ?? 0, 'record.read.has_explicit_projection': Boolean(query.projection), diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api.module.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api.module.ts index 8ede4f3efd..e79e9faf66 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api.module.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api.module.ts @@ -13,6 +13,7 @@ import { SpaceDataDbMigrationGuardModule } from '../../space/space-data-db-migra import { TableModule } from '../../table/table.module'; import { TableDomainQueryModule } from '../../table-domain'; import { V2Module } from '../../v2/v2.module'; +import { TableQuerySearchVectorRuntimeService } from '../../v2/table-query-search-vector-runtime.service'; import { ViewOpenApiModule } from '../../view/open-api/view-open-api.module'; import { ViewModule } from '../../view/view.module'; import { RecordModifyModule } from '../record-modify/record-modify.module'; @@ -43,7 +44,7 @@ import { RecordOpenApiService } from './record-open-api.service'; forwardRef(() => SelectionModule), ], controllers: [RecordOpenApiController], - providers: [RecordOpenApiService, RecordOpenApiV2Service], + providers: [RecordOpenApiService, RecordOpenApiV2Service, TableQuerySearchVectorRuntimeService], exports: [RecordOpenApiService, RecordOpenApiV2Service], }) export class RecordOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/share/share-socket.service.spec.ts b/apps/nestjs-backend/src/features/share/share-socket.service.spec.ts new file mode 100644 index 0000000000..1a1d08ed55 --- /dev/null +++ b/apps/nestjs-backend/src/features/share/share-socket.service.spec.ts @@ -0,0 +1,34 @@ +import { HttpErrorCode } from '@teable/core'; +import { describe, expect, it } from 'vitest'; +import { ShareSocketService } from './share-socket.service'; + +const createService = () => new ShareSocketService({} as never, {} as never, {} as never); + +describe('ShareSocketService computed activity authorization', () => { + it('allows activity for the shared table', () => { + const service = createService(); + + expect(() => + service.authorizeComputedActivityRead( + { shareId: 'shrTest', tableId: 'tblShared' }, + 'tblShared' + ) + ).not.toThrow(); + }); + + it('rejects activity for a different table', () => { + const service = createService(); + + expect(() => + service.authorizeComputedActivityRead( + { shareId: 'shrTest', tableId: 'tblShared' }, + 'tblOther' + ) + ).toThrowError( + expect.objectContaining({ + code: HttpErrorCode.RESTRICTED_RESOURCE, + message: 'Table(tblOther) permission not allowed: read', + }) + ); + }); +}); diff --git a/apps/nestjs-backend/src/features/share/share-socket.service.ts b/apps/nestjs-backend/src/features/share/share-socket.service.ts index b4a06e41b0..efb78722bc 100644 --- a/apps/nestjs-backend/src/features/share/share-socket.service.ts +++ b/apps/nestjs-backend/src/features/share/share-socket.service.ts @@ -172,4 +172,17 @@ export class ShareSocketService { ); } } + authorizeComputedActivityRead(shareInfo: IShareViewInfo, tableId: string): void { + if (shareInfo.tableId === tableId) return; + + throw new CustomHttpException( + `Table(${tableId}) permission not allowed: read`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.notAllowedTables', + }, + } + ); + } } diff --git a/apps/nestjs-backend/src/features/share/share.controller.ts b/apps/nestjs-backend/src/features/share/share.controller.ts index 8f5feb6527..dc8ef0bbbd 100644 --- a/apps/nestjs-backend/src/features/share/share.controller.ts +++ b/apps/nestjs-backend/src/features/share/share.controller.ts @@ -281,6 +281,18 @@ export class ShareController { return this.shareSocketService.getFieldDocIdsByQuery(shareInfo, query); } + @ShareLinkView() + @UseGuards(ShareAuthGuard) + @AllowAnonymous() + @Get('/:shareId/socket/computed-activity/authorize') + authorizeComputedActivityRead( + @Request() req: { shareInfo: IShareViewInfo }, + @Query('tableId') tableId: string + ): void { + const { shareInfo } = req; + this.shareSocketService.authorizeComputedActivityRead(shareInfo, tableId); + } + @ShareLinkView() @UseGuards(ShareAuthGuard) @AllowAnonymous() diff --git a/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts b/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts index 37192bfe47..d554d03566 100644 --- a/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts +++ b/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts @@ -551,6 +551,31 @@ describe('DataDbPreflightService', () => { }); }); + it('skips the cross-space link scan when includeRelatedSpaces is false', async () => { + const tableMetaFindMany = vi.fn(); + const fieldFindMany = vi.fn(); + const service = new DataDbPreflightService({ + ...defaultRelatedSpacesPrisma(), + tableMeta: { findMany: tableMetaFindMany }, + field: { findMany: fieldFindMany }, + spaceDataDbBinding: { + ...defaultRelatedSpacesPrisma().spaceDataDbBinding, + findUnique: vi.fn().mockResolvedValue(null), + }, + spaceDataDbMigrationJob: { + findFirst: vi.fn().mockResolvedValue(null), + findMany: vi.fn().mockResolvedValue([]), + }, + } as never); + + await expect(service.getSummary('spcxxx', { includeRelatedSpaces: false })).resolves.toEqual({ + mode: 'default', + state: 'ready', + }); + expect(tableMetaFindMany).not.toHaveBeenCalled(); + expect(fieldFindMany).not.toHaveBeenCalled(); + }); + it('returns default summary when the migration job table is not available yet', async () => { const service = new DataDbPreflightService({ ...defaultRelatedSpacesPrisma(), diff --git a/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts b/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts index d2bf86a8df..c61179dcda 100644 --- a/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts +++ b/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts @@ -327,7 +327,15 @@ export class DataDbPreflightService { } } - async getSummary(spaceId: string): Promise { + async getSummary( + spaceId: string, + options?: { includeRelatedSpaces?: boolean } + ): Promise { + // Cross-space link scan walks field.options with a global inbound prefilter and + // routinely takes multi-seconds in production. Callers that only need binding + // status (admin dialog first paint) can set includeRelatedSpaces=false. + const includeRelatedSpaces = options?.includeRelatedSpaces !== false; + if (this.prismaService) { const migrationSummary = await this.getActiveMigrationSummary(spaceId); if (migrationSummary) { @@ -335,12 +343,16 @@ export class DataDbPreflightService { // polling clients skip the cross-space link scan entirely. return migrationSummary; } - const relatedSpaces = await resolveSpaceDataDbRelatedSpaces(this.prismaService, spaceId); - const binding = await this.prismaService.spaceDataDbBinding.findUnique({ + const bindingPromise = this.prismaService.spaceDataDbBinding.findUnique({ where: { spaceId }, include: { dataDbConnection: true }, }); + const relatedSpacesPromise = includeRelatedSpaces + ? resolveSpaceDataDbRelatedSpaces(this.prismaService, spaceId) + : Promise.resolve(undefined); + const [binding, relatedSpaces] = await Promise.all([bindingPromise, relatedSpacesPromise]); + if (binding?.mode === 'byodb' && binding.dataDbConnection) { return { mode: binding.mode, @@ -355,13 +367,13 @@ export class DataDbPreflightService { capabilities: binding.dataDbConnection.capabilities as | IDataDbConnectionSummaryVo['capabilities'] | undefined, - relatedSpaces, + ...(relatedSpaces ? { relatedSpaces } : {}), }; } return { mode: 'default', state: 'ready', - relatedSpaces, + ...(relatedSpaces ? { relatedSpaces } : {}), }; } return { diff --git a/apps/nestjs-backend/src/features/space/space-data-db-migration.service.ts b/apps/nestjs-backend/src/features/space/space-data-db-migration.service.ts index 130b208a5d..0d9f7174e0 100644 --- a/apps/nestjs-backend/src/features/space/space-data-db-migration.service.ts +++ b/apps/nestjs-backend/src/features/space/space-data-db-migration.service.ts @@ -612,6 +612,10 @@ type IDeltaReplayColumnMetadata = { type IDeltaReplayStats = { phase: string; + // Seq floor of this job's own delta rows. `seq` is a bigserial shared by every + // migration job on the source, so absolute values carry the offset burned by + // earlier jobs; lag/progress must be computed relative to this baseline. + baselineSeq?: number; lastCapturedSeq: number; lastReplayedSeq: number; lag: number; @@ -621,6 +625,17 @@ type IDeltaReplayStats = { completedAt?: string; }; +type IDeltaCaptureScope = { + column: string; + ids: string[]; +}; + +type IDeltaCaptureRelation = { + schemaName: string; + tableName: string; + scope?: IDeltaCaptureScope; +}; + type IStaleMigrationJobRecord = IMigrationJobRecord & { lastModifiedTime: Date | null; }; @@ -2141,6 +2156,10 @@ export class SpaceDataDbMigrationService { return `${deltaTriggerNamePrefix}${jobId.replace(/\W/g, '_').slice(-32)}`; } + private deltaDeleteTriggerName(jobId: string) { + return `${this.deltaTriggerName(jobId)}_del`; + } + private isMigrationDeltaTrigger(trigger: IBaseRelationTriggerSignature) { return trigger.triggerName.startsWith(deltaTriggerNamePrefix); } @@ -2150,7 +2169,7 @@ export class SpaceDataDbMigrationService { } private getDeltaCaptureRelations(inventory: ISpaceDataDbInventory, sourceSchema: string) { - const baseRelations = inventory.physicalSchemas.flatMap((schema) => + const baseRelations: IDeltaCaptureRelation[] = inventory.physicalSchemas.flatMap((schema) => schema.relations .filter((relation) => relationKindsWithRows.has(relation.relationKind)) .map((relation) => ({ @@ -2158,10 +2177,18 @@ export class SpaceDataDbMigrationService { tableName: relation.relationName, })) ); - const sharedRelations = Object.values(sharedTables).map((tableName) => ({ - schemaName: sourceSchema, - tableName, - })); + // Shared tables live in the source's public schema and receive writes from + // every space on the instance. Scope their capture triggers to this job's + // rows (mirroring shouldReplayDeltaRow) so a migration does not have to log + // and then page through the whole instance's shared-table churn. + const scopeBySharedTable = this.getDeltaCaptureScopeBySharedTable(inventory); + const sharedRelations: IDeltaCaptureRelation[] = Object.values(sharedTables).map( + (tableName) => ({ + schemaName: sourceSchema, + tableName, + scope: scopeBySharedTable[tableName], + }) + ); const seen = new Set(); return [...baseRelations, ...sharedRelations].filter((relation) => { const key = `${relation.schemaName}.${relation.tableName}`; @@ -2173,6 +2200,28 @@ export class SpaceDataDbMigrationService { }); } + private getDeltaCaptureScopeBySharedTable( + inventory: ISpaceDataDbInventory + ): Record { + const scoped = (column: string, ids: string[]): IDeltaCaptureScope | undefined => + ids.length ? { column, ids } : undefined; + const tableScopeIds = inventory.sharedTableIds.length + ? inventory.sharedTableIds + : inventory.tableIds; + return { + [sharedTables.recordHistory]: scoped('table_id', tableScopeIds), + [sharedTables.tableTrash]: scoped('table_id', tableScopeIds), + [sharedTables.recordTrash]: scoped('table_id', tableScopeIds), + [sharedTables.computedUpdateOutbox]: scoped('base_id', inventory.baseIds), + [sharedTables.computedUpdateDeadLetter]: scoped('base_id', inventory.baseIds), + [sharedTables.computedUpdateOutboxSeed]: scoped('table_id', inventory.tableIds), + // computed_update_pause_scope and __undo_log key their scope on composite + // or derived values that a trigger WHEN clause cannot express cheaply; + // their churn is negligible, so keep capturing them unscoped and let + // shouldReplayDeltaRow filter at replay time. + }; + } + private async openSourceSnapshot(sourceUrl: string): Promise { const client = this.clientFactory(sourceUrl); let transaction: IDataDbPreflightTransaction | null = null; @@ -2286,16 +2335,41 @@ export class SpaceDataDbMigrationService { $$; `); + const deleteTriggerName = this.deltaDeleteTriggerName(job.id); for (const relation of this.getDeltaCaptureRelations(inventory, sourceSchema)) { + const qualifiedRelation = qualify(relation.schemaName, relation.tableName); + if (!relation.scope) { + await client.raw( + ` + DROP TRIGGER IF EXISTS ${quoteIdent(triggerName)} ON ${qualifiedRelation}; + DROP TRIGGER IF EXISTS ${quoteIdent(deleteTriggerName)} ON ${qualifiedRelation}; + CREATE TRIGGER ${quoteIdent(triggerName)} + AFTER INSERT OR UPDATE OR DELETE ON ${qualifiedRelation} + FOR EACH ROW + EXECUTE FUNCTION ${qualify(sourceSchema, deltaCaptureFunction)}(${sqlLiteral(job.id)}); + ` + ); + continue; + } + // A single multi-event trigger cannot reference NEW and OLD in one WHEN + // clause, so scoped relations get an INSERT/UPDATE trigger filtered on + // NEW and a DELETE trigger filtered on OLD. The WHEN clause runs before + // the function call, so out-of-scope rows cost almost nothing. + const scopeArray = `ARRAY[${relation.scope.ids.map((id) => sqlLiteral(id)).join(', ')}]::text[]`; + const scopeColumn = quoteIdent(relation.scope.column); await client.raw( ` - DROP TRIGGER IF EXISTS ${quoteIdent(triggerName)} ON ${qualify( - relation.schemaName, - relation.tableName - )}; + DROP TRIGGER IF EXISTS ${quoteIdent(triggerName)} ON ${qualifiedRelation}; + DROP TRIGGER IF EXISTS ${quoteIdent(deleteTriggerName)} ON ${qualifiedRelation}; CREATE TRIGGER ${quoteIdent(triggerName)} - AFTER INSERT OR UPDATE OR DELETE ON ${qualify(relation.schemaName, relation.tableName)} + AFTER INSERT OR UPDATE ON ${qualifiedRelation} FOR EACH ROW + WHEN (NEW.${scopeColumn} = ANY (${scopeArray})) + EXECUTE FUNCTION ${qualify(sourceSchema, deltaCaptureFunction)}(${sqlLiteral(job.id)}); + CREATE TRIGGER ${quoteIdent(deleteTriggerName)} + AFTER DELETE ON ${qualifiedRelation} + FOR EACH ROW + WHEN (OLD.${scopeColumn} = ANY (${scopeArray})) EXECUTE FUNCTION ${qualify(sourceSchema, deltaCaptureFunction)}(${sqlLiteral(job.id)}); ` ); @@ -2333,6 +2407,7 @@ export class SpaceDataDbMigrationService { const triggerName = this.deltaTriggerName(job.id); const client = this.clientFactory(sourceDataDb.url); try { + const deleteTriggerName = this.deltaDeleteTriggerName(job.id); for (const relation of this.getDeltaCaptureRelations(inventory, sourceSchema)) { await client .raw( @@ -2342,6 +2417,14 @@ export class SpaceDataDbMigrationService { )}` ) .catch(() => undefined); + await client + .raw( + `DROP TRIGGER IF EXISTS ${quoteIdent(deleteTriggerName)} ON ${qualify( + relation.schemaName, + relation.tableName + )}` + ) + .catch(() => undefined); } await client .raw(`DELETE FROM ${qualify(sourceSchema, deltaLogTable)} WHERE "job_id" = ?`, [job.id]) @@ -2368,6 +2451,24 @@ export class SpaceDataDbMigrationService { return Number(rows[0]?.maxSeq ?? 0); } + private async getSourceDeltaMinSeq( + client: IDataDbPreflightClient, + sourceSchema: string, + jobId: string + ): Promise { + const rows = normalizeRawRows<{ minSeq: string | number | bigint | null }>( + await client.raw( + `SELECT MIN("seq") AS "minSeq" FROM ${qualify( + sourceSchema, + deltaLogTable + )} WHERE "job_id" = ?`, + [jobId] + ) + ); + const minSeq = rows[0]?.minSeq; + return minSeq === null || minSeq === undefined ? null : Number(minSeq); + } + private async tryGetSourceDeltaMaxSeq( client: IDataDbPreflightClient, sourceSchema: string, @@ -2638,7 +2739,20 @@ export class SpaceDataDbMigrationService { if (stats.lastCapturedSeq <= 0) { return undefined; } - const seqFraction = Math.min(Math.max(stats.lastReplayedSeq / stats.lastCapturedSeq, 0), 1); + // Measure progress against this job's own seq range; absolute seq values + // include the offset burned by earlier jobs on the shared sequence. + const baselineSeq = Math.min( + Math.max(Number(stats.baselineSeq ?? 0), 0), + stats.lastCapturedSeq + ); + const capturedSpan = stats.lastCapturedSeq - baselineSeq; + if (capturedSpan <= 0) { + return undefined; + } + const seqFraction = Math.min( + Math.max((stats.lastReplayedSeq - baselineSeq) / capturedSpan, 0), + 1 + ); // The first replay pass covers the front of the delta stage; the cutover // pass after the final write gate covers the tail. if (stats.phase === 'delta_replaying') { @@ -2700,6 +2814,18 @@ export class SpaceDataDbMigrationService { const existingLastReplayedSeq = Number(existingDeltaStats?.lastReplayedSeq ?? 0); let lastReplayedSeq = Number.isFinite(existingLastReplayedSeq) ? existingLastReplayedSeq : 0; let lastCapturedSeq = await this.getSourceDeltaMaxSeq(sourceClient, sourceSchema, jobId); + // `seq` is one bigserial shared by every migration job on this source, so + // this job's rows start wherever earlier jobs left the sequence. Floor the + // cursor at the job's own first row so lag and progress never report the + // ranges burned by previous migrations as pending catch-up work. + const existingBaselineSeq = Number(existingDeltaStats?.baselineSeq ?? NaN); + const jobMinSeq = await this.getSourceDeltaMinSeq(sourceClient, sourceSchema, jobId); + const computedBaselineSeq = jobMinSeq === null ? lastCapturedSeq : jobMinSeq - 1; + const baselineSeq = Math.max( + Number.isFinite(existingBaselineSeq) ? existingBaselineSeq : 0, + computedBaselineSeq + ); + lastReplayedSeq = Math.max(lastReplayedSeq, baselineSeq); try { while (true) { @@ -2737,6 +2863,7 @@ export class SpaceDataDbMigrationService { const lag = Math.max(0, lastCapturedSeq - lastReplayedSeq); const stats: IDeltaReplayStats = { phase, + baselineSeq, lastCapturedSeq, lastReplayedSeq, lag, diff --git a/apps/nestjs-backend/src/features/space/space-data-db-related-spaces.ts b/apps/nestjs-backend/src/features/space/space-data-db-related-spaces.ts index 4a371d4c5a..1bb012e762 100644 --- a/apps/nestjs-backend/src/features/space/space-data-db-related-spaces.ts +++ b/apps/nestjs-backend/src/features/space/space-data-db-related-spaces.ts @@ -125,13 +125,14 @@ const findLinkFieldsOfSpaces = async ( })); }; -// Coarse SQL prefilter for inbound links: match EVERY `"foreignTableId":"..."` -// occurrence in the JSON blob (regexp_matches with 'g') so the scan returns only -// candidate rows instead of every link-ish field in the instance. Matching all -// occurrences keeps the prefilter a superset of the truth: a nested occurrence can -// only add a false positive, never hide a row whose real top-level target is in the -// frontier. `extractForeignTableId` (real JSON parsing) stays the source of truth -// for every returned row, so a regex false positive can never fabricate an edge. +// Coarse SQL prefilter for inbound links: match ANY `"foreignTableId":""` +// substring in the JSON blob so the scan returns only candidate rows instead of +// every link-ish field in the instance. Nested occurrences keep the prefilter a +// superset of the truth (false positives only). Prefer strpos + unnest over +// regexp_matches(..., 'g') — production traces showed the regex prefilter alone +// dominating GET data-db (~2.5s of a ~3.3s admin summary). +// `extractForeignTableId` (real JSON parsing) stays the source of truth for every +// returned row, so a prefilter false positive can never fabricate an edge. const findLinkFieldsReferencingTables = async ( prismaService: PrismaService, tableIds: string[] @@ -156,16 +157,13 @@ const findLinkFieldsReferencingTables = async ( AND ((f.type = $1 AND f.is_lookup IS NOT TRUE) OR f.type = $2 OR (f.is_lookup IS TRUE AND f.is_conditional_lookup IS TRUE)) - AND EXISTS ( - SELECT 1 - FROM regexp_matches( - CASE WHEN f.is_lookup IS TRUE AND f.is_conditional_lookup IS TRUE - THEN f.lookup_options - ELSE f.options END, - '"foreignTableId"\\s*:\\s*"([^"]+)"', - 'g' - ) AS matches(captured) - WHERE matches.captured[1] = ANY($3::text[]) + AND ( + CASE WHEN f.is_lookup IS TRUE AND f.is_conditional_lookup IS TRUE + THEN f.lookup_options + ELSE f.options END + ) LIKE ANY ( + SELECT ('%"foreignTableId":"' || frontier.table_id || '"%') + FROM unnest($3::text[]) AS frontier(table_id) )`, FieldType.Link, FieldType.ConditionalRollup, diff --git a/apps/nestjs-backend/src/features/space/space.controller.ts b/apps/nestjs-backend/src/features/space/space.controller.ts index 4780944472..451670367f 100644 --- a/apps/nestjs-backend/src/features/space/space.controller.ts +++ b/apps/nestjs-backend/src/features/space/space.controller.ts @@ -34,6 +34,8 @@ import { ICreateSpaceRo, dataDbPreflightRoSchema, IDataDbPreflightRo, + type ISpaceDataDbSummaryQuery, + spaceDataDbSummaryQuerySchema, updateSpaceRoSchema, IUpdateSpaceRo, emailSpaceInvitationRoSchema, @@ -148,8 +150,12 @@ export class SpaceController { @Permissions('space|read') @Get(':spaceId/data-db') - async getSpaceDataDb(@Param('spaceId') spaceId: string): Promise { - return await this.dataDbPreflightService.getSummary(spaceId); + async getSpaceDataDb( + @Param('spaceId') spaceId: string, + @Query(new ZodValidationPipe(spaceDataDbSummaryQuerySchema)) + query: ISpaceDataDbSummaryQuery + ): Promise { + return await this.dataDbPreflightService.getSummary(spaceId, query); } @Permissions('space|update') diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts index 96abf474ed..ba4474f3ff 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts @@ -19,6 +19,7 @@ import type { IGetAbnormalVo, ITableFullVo, ITableListVo, + ITableSearchVectorStatusVo, ITableVo, } from '@teable/openapi'; import { @@ -270,6 +271,14 @@ export class TableController { return this.tableIndexService.getActivatedTableIndexes(tableId); } + @Get(':tableId/search-vector-status') + @Permissions('table|read') + async getSearchVectorStatus( + @Param('tableId') tableId: string + ): Promise { + return this.tableIndexService.getSearchVectorStatus(tableId); + } + @Get(':tableId/abnormal-index') @Permissions('table|read') async getAbnormalTableIndex( diff --git a/apps/nestjs-backend/src/features/table/table-index.service.spec.ts b/apps/nestjs-backend/src/features/table/table-index.service.spec.ts new file mode 100644 index 0000000000..b2e34443d7 --- /dev/null +++ b/apps/nestjs-backend/src/features/table/table-index.service.spec.ts @@ -0,0 +1,177 @@ +import { HttpErrorCode } from '@teable/core'; +import { domainError, type DomainError } from '@teable/v2-core'; +import { v2TableOpsTokens, type TableSearchVectorStatus } from '@teable/v2-table-query-ops'; +import { err, ok, type Result } from 'neverthrow'; +import { describe, expect, it, vi } from 'vitest'; + +import { TableIndexService } from './table-index.service'; + +const tableId = 'tblSearchVectorTest'; + +const createService = (input: { + registered: boolean; + runtimeEnabled?: boolean; + result?: Result; +}) => { + const reader = { + read: vi.fn().mockResolvedValue( + input.result ?? + ok({ + tableId, + state: 'ready', + configured: true, + languageConfig: 'simple', + coveredFieldCount: 3, + } satisfies TableSearchVectorStatus) + ), + }; + const container = { + isRegistered: vi.fn().mockReturnValue(input.registered), + resolve: vi.fn().mockReturnValue(reader), + }; + const v2ContainerService = { + getContainerForTable: vi.fn().mockResolvedValue(container), + isTableQuerySearchVectorRuntimeEnabled: vi.fn().mockReturnValue(input.runtimeEnabled ?? true), + }; + const v2ExecutionContextFactory = { + createContext: vi.fn().mockResolvedValue({ requestId: 'test' }), + }; + const prismaService = { + txClient: () => ({ + tableMeta: { findUnique: vi.fn().mockResolvedValue({ baseId: 'bseV2' }) }, + base: { + findUnique: vi.fn().mockResolvedValue({ spaceId: 'spcV2', v2Enabled: true }), + }, + }), + }; + const canaryService = { + shouldUseV2ForBaseWithReason: vi.fn().mockResolvedValue({ useV2: true, reason: 'new_base' }), + }; + const service = new TableIndexService( + {} as never, + prismaService as never, + {} as never, + {} as never, + {} as never, + canaryService as never, + v2ContainerService as never, + v2ExecutionContextFactory as never + ); + + return { service, reader, container, v2ContainerService, v2ExecutionContextFactory }; +}; + +describe('TableIndexService search vector status', () => { + it('returns disabled when the host module does not mount v2 services', async () => { + const service = new TableIndexService( + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + { + shouldUseV2ForBaseWithReason: vi + .fn() + .mockResolvedValue({ useV2: true, reason: 'new_base' }), + } as never, + undefined as never, + undefined as never + ); + + await expect(service.getSearchVectorStatus(tableId)).resolves.toEqual({ + tableId, + state: 'disabled', + configured: false, + active: false, + coveredFieldCount: 0, + }); + }); + + it('returns disabled when Table Query Ops is not registered', async () => { + const { service, container, reader, v2ExecutionContextFactory } = createService({ + registered: false, + }); + + await expect(service.getSearchVectorStatus(tableId)).resolves.toEqual({ + tableId, + state: 'disabled', + configured: false, + active: false, + coveredFieldCount: 0, + }); + expect(container.isRegistered).toHaveBeenCalledWith(v2TableOpsTokens.searchVectorStatusReader); + expect(v2ExecutionContextFactory.createContext).not.toHaveBeenCalled(); + expect(reader.read).not.toHaveBeenCalled(); + }); + + it('returns the read-only adapter status when registered', async () => { + const { service, reader } = createService({ registered: true }); + + await expect(service.getSearchVectorStatus(tableId)).resolves.toMatchObject({ + state: 'ready', + configured: true, + active: true, + coveredFieldCount: 3, + }); + expect(reader.read).toHaveBeenCalledWith({ requestId: 'test' }, tableId); + }); + + it('does not report a ready config as active while the runtime gate is off', async () => { + const { service } = createService({ registered: true, runtimeEnabled: false }); + + await expect(service.getSearchVectorStatus(tableId)).resolves.toMatchObject({ + state: 'ready', + configured: true, + active: false, + }); + }); + + it('keeps a ready config inactive when the host module does not mount CanaryService', async () => { + const { service } = createService({ registered: true }); + Object.assign(service, { canaryService: undefined }); + + await expect(service.getSearchVectorStatus(tableId)).resolves.toMatchObject({ + state: 'ready', + configured: true, + active: false, + }); + }); + + it('does not report a ready config as active when record reads stay on v1', async () => { + const { service } = createService({ registered: true }); + const tableMetaFindUnique = vi.fn().mockResolvedValue({ baseId: 'bseLegacy' }); + const baseFindUnique = vi.fn().mockResolvedValue({ spaceId: 'spcLegacy', v2Enabled: false }); + const shouldUseV2ForBaseWithReason = vi + .fn() + .mockResolvedValue({ useV2: false, reason: 'feature_not_enabled' }); + Object.assign(service, { + prismaService: { + txClient: () => ({ + tableMeta: { findUnique: tableMetaFindUnique }, + base: { findUnique: baseFindUnique }, + }), + }, + canaryService: { shouldUseV2ForBaseWithReason }, + }); + + await expect(service.getSearchVectorStatus(tableId)).resolves.toMatchObject({ + state: 'ready', + configured: true, + active: false, + }); + expect(shouldUseV2ForBaseWithReason).toHaveBeenCalledWith( + { spaceId: 'spcLegacy', v2Enabled: false }, + 'getRecords' + ); + }); + it('maps adapter errors to an internal HTTP error', async () => { + const { service } = createService({ + registered: true, + result: err(domainError.infrastructure({ message: 'read failed' })), + }); + + await expect(service.getSearchVectorStatus(tableId)).rejects.toMatchObject({ + code: HttpErrorCode.INTERNAL_SERVER_ERROR, + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/table/table-index.service.ts b/apps/nestjs-backend/src/features/table/table-index.service.ts index e8c1fb3170..8599d6c462 100644 --- a/apps/nestjs-backend/src/features/table/table-index.service.ts +++ b/apps/nestjs-backend/src/features/table/table-index.service.ts @@ -1,9 +1,16 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Injectable, Logger } from '@nestjs/common'; +import { Inject, Injectable, Logger, Optional } from '@nestjs/common'; import { FieldType, HttpErrorCode } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { TableIndex } from '@teable/openapi'; -import type { IGetAbnormalVo, ITableIndexType, IToggleIndexRo } from '@teable/openapi'; +import type { + IGetAbnormalVo, + ITableIndexType, + ITableSearchVectorStatusVo, + IToggleIndexRo, +} from '@teable/openapi'; +import type { TableSearchVectorStatusReader } from '@teable/v2-table-query-ops'; +import { v2TableOpsTokens } from '@teable/v2-table-query-ops'; import { ClsService } from 'nestjs-cls'; import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; import { CustomHttpException } from '../../custom.exception'; @@ -12,8 +19,11 @@ import { IDbProvider } from '../../db-provider/db.provider.interface'; import type { IDataDbRoutingOptions } from '../../global/data-db-client-manager.service'; import { DatabaseRouter } from '../../global/database-router.service'; import type { IClsStore } from '../../types/cls'; +import { CanaryService } from '../canary/canary.service'; import type { IFieldInstance } from '../field/model/factory'; import { createFieldInstanceByRaw } from '../field/model/factory'; +import { V2ContainerService } from '../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../v2/v2-execution-context.factory'; @Injectable() export class TableIndexService { @@ -24,9 +34,70 @@ export class TableIndexService { private readonly prismaService: PrismaService, private readonly databaseRouter: DatabaseRouter, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, - @InjectDbProvider() private readonly dbProvider: IDbProvider + @InjectDbProvider() private readonly dbProvider: IDbProvider, + @Inject(CanaryService) + @Optional() + private readonly canaryService?: CanaryService, + @Inject(V2ContainerService) + @Optional() + private readonly v2ContainerService?: V2ContainerService, + @Inject(V2ExecutionContextFactory) + @Optional() + private readonly v2ExecutionContextFactory?: V2ExecutionContextFactory ) {} + async getSearchVectorStatus(tableId: string): Promise { + const disabled: ITableSearchVectorStatusVo = { + tableId, + state: 'disabled', + configured: false, + active: false, + coveredFieldCount: 0, + }; + if (!this.v2ContainerService || !this.v2ExecutionContextFactory) return disabled; + + const container = await this.v2ContainerService.getContainerForTable(tableId); + if (!container.isRegistered(v2TableOpsTokens.searchVectorStatusReader)) return disabled; + + const context = await this.v2ExecutionContextFactory.createContext(container); + const reader = container.resolve( + v2TableOpsTokens.searchVectorStatusReader + ); + const result = await reader.read(context, tableId); + if (result.isErr()) { + this.logger.error(result.error.message, result.error); + throw new CustomHttpException( + 'Failed to read table search vector status', + HttpErrorCode.INTERNAL_SERVER_ERROR + ); + } + return { + ...result.value, + active: + result.value.state === 'ready' && + this.v2ContainerService.isTableQuerySearchVectorRuntimeEnabled() && + (await this.isV2RecordReadEnabled(tableId)), + }; + } + + private async isV2RecordReadEnabled(tableId: string): Promise { + if (!this.canaryService) return false; + const prisma = this.prismaService.txClient(); + const table = await prisma.tableMeta.findUnique({ + where: { id: tableId, deletedTime: null }, + select: { baseId: true }, + }); + if (!table) return false; + + const base = await prisma.base.findUnique({ + where: { id: table.baseId, deletedTime: null }, + select: { spaceId: true, v2Enabled: true }, + }); + if (!base) return false; + + const decision = await this.canaryService.shouldUseV2ForBaseWithReason(base, 'getRecords'); + return decision.useV2; + } async getSearchIndexFields(tableId: string): Promise { const fieldsRaw = await this.prismaService.field.findMany({ where: { diff --git a/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-anomaly.service.spec.ts b/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-anomaly.service.spec.ts index c496a77949..47adaf62ee 100644 --- a/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-anomaly.service.spec.ts +++ b/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-anomaly.service.spec.ts @@ -1,7 +1,10 @@ import { ConflictException, NotFoundException } from '@nestjs/common'; import { describe, expect, it, vi } from 'vitest'; -import { ComputedOutboxAnomalyService } from './computed-outbox-anomaly.service'; +import { + ComputedOutboxAnomalyService, + groupComputedOutboxAnomalies, +} from './computed-outbox-anomaly.service'; const targets = [ { @@ -18,8 +21,85 @@ const targets = [ }, ] as const; +const anomalyFields = { + failedSql: null, + failureKind: null, + failurePhase: null, + affectedTableName: null, +} as const; + +describe('groupComputedOutboxAnomalies', () => { + it('merges repeated root causes into groups and keeps recent samples', () => { + const result = groupComputedOutboxAnomalies( + [ + { + targetId: 'meta-fallback', + storage: 'default', + kind: 'dead', + taskId: 'cuo-2', + baseId: 'bse1', + seedTableId: 'tbl1', + attempts: 8, + maxAttempts: 8, + lastError: 'statement timeout', + failedSql: 'update t set x = 1', + failureKind: 'statement_timeout', + failurePhase: 'execute_plan', + affectedTableName: 't', + occurredAt: new Date('2026-07-15T05:00:00.000Z'), + }, + { + targetId: 'meta-fallback', + storage: 'default', + kind: 'dead', + taskId: 'cuo-1', + baseId: 'bse1', + seedTableId: 'tbl1', + attempts: 8, + maxAttempts: 8, + lastError: 'statement timeout', + failedSql: null, + failureKind: 'statement_timeout', + failurePhase: 'execute_plan', + affectedTableName: 't', + occurredAt: new Date('2026-07-15T04:00:00.000Z'), + }, + { + targetId: 'dcn1', + storage: 'byodb', + kind: 'stale', + taskId: 'cuo-3', + baseId: 'bse2', + seedTableId: 'tbl2', + attempts: 1, + maxAttempts: 8, + lastError: null, + ...anomalyFields, + occurredAt: new Date('2026-07-15T06:00:00.000Z'), + }, + ], + { groupLimit: 10, sampleLimit: 12 } + ); + + expect(result.groupTotal).toBe(2); + expect(result.groups).toHaveLength(2); + expect(result.groups[0]).toMatchObject({ + kind: 'stale', + count: 1, + baseId: 'bse2', + }); + expect(result.groups[1]).toMatchObject({ + kind: 'dead', + count: 2, + baseId: 'bse1', + failedSql: 'update t set x = 1', + items: [{ taskId: 'cuo-2' }, { taskId: 'cuo-1' }], + }); + }); +}); + describe('ComputedOutboxAnomalyService', () => { - it('aggregates recent anomalies without exposing target URLs', async () => { + it('aggregates recent anomalies into groups without exposing target URLs', async () => { const listComputedOutboxMaintenanceAnomalies = vi .fn() .mockResolvedValueOnce({ @@ -33,8 +113,23 @@ describe('ComputedOutboxAnomalyService', () => { attempts: 8, maxAttempts: 8, lastError: 'timeout', + ...anomalyFields, occurredAt: new Date('2026-07-15T04:00:00.000Z'), }, + { + kind: 'dead', + taskId: 'cuo-old-2', + baseId: 'bse1', + seedTableId: 'tbl1', + attempts: 8, + maxAttempts: 8, + lastError: 'timeout', + failedSql: 'select 1', + failureKind: 'statement_timeout', + failurePhase: 'execute_plan', + affectedTableName: null, + occurredAt: new Date('2026-07-15T03:00:00.000Z'), + }, ], }) .mockResolvedValueOnce({ @@ -48,6 +143,7 @@ describe('ComputedOutboxAnomalyService', () => { attempts: 1, maxAttempts: 8, lastError: null, + ...anomalyFields, occurredAt: new Date('2026-07-15T05:00:00.000Z'), }, ], @@ -63,14 +159,26 @@ describe('ComputedOutboxAnomalyService', () => { const result = await service.list(20); expect(result.total).toBe(3); + expect(result.groupTotal).toBe(2); expect(result.unavailableTargetCount).toBe(0); expect( - result.items.map(({ taskId, targetId, storage }) => ({ taskId, targetId, storage })) + result.groups.map(({ count, items, targetId, storage }) => ({ + count, + taskIds: items.map((item) => item.taskId), + targetId, + storage, + })) ).toEqual([ - { taskId: 'cuo-new', targetId: 'dcn1', storage: 'byodb' }, - { taskId: 'cuo-old', targetId: 'meta-fallback', storage: 'default' }, + { count: 1, taskIds: ['cuo-new'], targetId: 'dcn1', storage: 'byodb' }, + { + count: 2, + taskIds: ['cuo-old', 'cuo-old-2'], + targetId: 'meta-fallback', + storage: 'default', + }, ]); - expect(result.items.some((item) => 'url' in item)).toBe(false); + expect(result.groups[1]?.failedSql).toBe('select 1'); + expect(JSON.stringify(result.groups)).not.toContain('postgres://'); }); it('restores a dead letter and publishes a BullMQ wake-up', async () => { diff --git a/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-anomaly.service.ts b/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-anomaly.service.ts index 35141ef56a..d2782e4ba7 100644 --- a/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-anomaly.service.ts +++ b/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-anomaly.service.ts @@ -10,7 +10,11 @@ import type { } from '../../../global/data-db-client-manager.service'; import { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import { IComputedOutboxWakeupAppPublisher } from './computed-outbox-wakeup.publisher'; -import { COMPUTED_OUTBOX_WAKEUP_PUBLISHER } from './constants'; +import { + COMPUTED_OUTBOX_ANOMALY_FETCH_CAP, + COMPUTED_OUTBOX_ANOMALY_GROUP_SAMPLE_LIMIT, + COMPUTED_OUTBOX_WAKEUP_PUBLISHER, +} from './constants'; import { mapWithConcurrency } from './map-with-concurrency'; export type ComputedOutboxAnomaly = IComputedOutboxMaintenanceAnomaly & { @@ -18,6 +22,113 @@ export type ComputedOutboxAnomaly = IComputedOutboxMaintenanceAnomaly & { storage: IComputedOutboxMaintenanceTarget['storage']; }; +export type ComputedOutboxAnomalyGroup = { + groupKey: string; + kind: ComputedOutboxAnomaly['kind']; + targetId: string; + storage: ComputedOutboxAnomaly['storage']; + baseId: string; + seedTableId: string; + lastError: string | null; + failedSql: string | null; + failureKind: string | null; + failurePhase: string | null; + affectedTableName: string | null; + count: number; + latestOccurredAt: Date; + items: ComputedOutboxAnomaly[]; +}; + +export const buildComputedOutboxAnomalyGroupKey = ( + item: Pick +): string => + [item.kind, item.baseId, item.seedTableId, (item.lastError ?? '').slice(0, 500)].join('\u0001'); + +export const groupComputedOutboxAnomalies = ( + items: ReadonlyArray, + options?: { groupLimit?: number; sampleLimit?: number } +): { + groups: ComputedOutboxAnomalyGroup[]; + groupTotal: number; +} => { + const groupLimit = Math.max(1, options?.groupLimit ?? 30); + const sampleLimit = Math.max( + 1, + options?.sampleLimit ?? COMPUTED_OUTBOX_ANOMALY_GROUP_SAMPLE_LIMIT + ); + const groupsByKey = new Map(); + + for (const item of items) { + const groupKey = buildComputedOutboxAnomalyGroupKey(item); + const existing = groupsByKey.get(groupKey); + if (!existing) { + groupsByKey.set(groupKey, { + groupKey, + kind: item.kind, + targetId: item.targetId, + storage: item.storage, + baseId: item.baseId, + seedTableId: item.seedTableId, + lastError: item.lastError, + failedSql: item.failedSql, + failureKind: item.failureKind, + failurePhase: item.failurePhase, + affectedTableName: item.affectedTableName, + count: 1, + latestOccurredAt: item.occurredAt, + items: [item], + }); + continue; + } + + existing.count += 1; + if ( + item.occurredAt.getTime() > existing.latestOccurredAt.getTime() || + (item.occurredAt.getTime() === existing.latestOccurredAt.getTime() && + item.taskId.localeCompare(existing.items[0]?.taskId ?? '') < 0) + ) { + existing.targetId = item.targetId; + existing.storage = item.storage; + existing.lastError = item.lastError; + existing.failedSql = item.failedSql ?? existing.failedSql; + existing.failureKind = item.failureKind ?? existing.failureKind; + existing.failurePhase = item.failurePhase ?? existing.failurePhase; + existing.affectedTableName = item.affectedTableName ?? existing.affectedTableName; + existing.latestOccurredAt = item.occurredAt; + } else if (!existing.failedSql && item.failedSql) { + existing.failedSql = item.failedSql; + existing.failureKind = item.failureKind ?? existing.failureKind; + existing.failurePhase = item.failurePhase ?? existing.failurePhase; + existing.affectedTableName = item.affectedTableName ?? existing.affectedTableName; + } + + if (existing.items.length < sampleLimit) { + existing.items.push(item); + } + } + + const groups = [...groupsByKey.values()] + .map((group) => ({ + ...group, + items: [...group.items].sort( + (left, right) => + right.occurredAt.getTime() - left.occurredAt.getTime() || + left.taskId.localeCompare(right.taskId) + ), + })) + .sort( + (left, right) => + right.latestOccurredAt.getTime() - left.latestOccurredAt.getTime() || + right.count - left.count || + left.groupKey.localeCompare(right.groupKey) + ); + + return { + groupTotal: groups.length, + groups: groups.slice(0, groupLimit), + }; +}; + @Injectable() export class ComputedOutboxAnomalyService { private readonly logger = new Logger(ComputedOutboxAnomalyService.name); @@ -28,19 +139,21 @@ export class ComputedOutboxAnomalyService { private readonly wakeupPublisher: IComputedOutboxWakeupAppPublisher ) {} - async list(limit: number): Promise<{ + async list(groupLimit: number): Promise<{ sampledAt: string; total: number; - items: ComputedOutboxAnomaly[]; + groupTotal: number; + groups: ComputedOutboxAnomalyGroup[]; unavailableTargetCount: number; }> { const targets = await this.dataDbClientManager.listComputedOutboxMaintenanceTargets(); + const fetchLimit = Math.min(COMPUTED_OUTBOX_ANOMALY_FETCH_CAP, Math.max(groupLimit * 40, 200)); const results = await mapWithConcurrency(targets, 4, async (target) => { try { const snapshot = await this.dataDbClientManager.listComputedOutboxMaintenanceAnomalies( target, defaultComputedUpdateOutboxConfig.processingLeaseMs, - limit + fetchLimit ); return { target, snapshot }; } catch (error) { @@ -53,23 +166,27 @@ export class ComputedOutboxAnomalyService { } }); + const items = results + .flatMap((result) => + (result.snapshot?.items ?? []).map((item) => ({ + ...item, + targetId: result.target.cacheKey, + storage: result.target.storage, + })) + ) + .sort( + (left, right) => + right.occurredAt.getTime() - left.occurredAt.getTime() || + left.taskId.localeCompare(right.taskId) + ); + + const { groups, groupTotal } = groupComputedOutboxAnomalies(items, { groupLimit }); + return { sampledAt: new Date().toISOString(), total: results.reduce((sum, result) => sum + (result.snapshot?.total ?? 0), 0), - items: results - .flatMap((result) => - (result.snapshot?.items ?? []).map((item) => ({ - ...item, - targetId: result.target.cacheKey, - storage: result.target.storage, - })) - ) - .sort( - (left, right) => - right.occurredAt.getTime() - left.occurredAt.getTime() || - left.taskId.localeCompare(right.taskId) - ) - .slice(0, limit), + groupTotal, + groups, unavailableTargetCount: results.filter((result) => !result.snapshot).length, }; } diff --git a/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-monitor.service.spec.ts b/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-monitor.service.spec.ts index e17e31a8fb..410f1836e1 100644 --- a/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-monitor.service.spec.ts +++ b/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-monitor.service.spec.ts @@ -93,6 +93,24 @@ describe('ComputedOutboxMonitorService', () => { returnvalue: { secret: 'must-not-leak' }, }, ]), + getFailed: vi.fn().mockResolvedValue([ + { + id: 'job-failed', + data: { + schemaVersion: 1, + wakeupId: 'wake-failed-secret', + taskId: 'cuo-failed', + baseId: 'bse123', + availableAt: '2026-07-13T11:00:00.000Z', + emittedAt: '2026-07-13T11:00:00.000Z', + cause: 'retry', + secret: 'must-not-leak', + }, + finishedOn: 900, + attemptsMade: 3, + failedReason: 'worker crashed: secret-must-stay', + }, + ]), }; const dataDbClientManager = { listComputedOutboxMaintenanceTargets: vi.fn().mockResolvedValue(targets), @@ -143,10 +161,21 @@ describe('ComputedOutboxMonitorService', () => { attemptsMade: 1, }, ], + recentFailed: [ + { + taskId: 'cuo-failed', + baseId: 'bse123', + cause: 'retry', + failedAt: '1970-01-01T00:00:00.900Z', + failedReason: 'worker crashed: secret-must-stay', + attemptsMade: 3, + }, + ], }); expect(queue.getCompleted).toHaveBeenCalledWith(0, 9); - expect(JSON.stringify(result.queue.recentCompleted)).not.toContain('secret'); + expect(queue.getFailed).toHaveBeenCalledWith(0, 9); expect(JSON.stringify(result.queue.recentCompleted)).not.toContain('wake-secret'); + expect(JSON.stringify(result.queue.recentFailed)).not.toContain('wake-failed-secret'); expect(result.outbox).toMatchObject({ targetCount: 2, unavailableTargetCount: 0, @@ -208,6 +237,7 @@ describe('ComputedOutboxMonitorService', () => { getJobCounts: vi.fn().mockResolvedValue({}), getWorkersCount: vi.fn().mockResolvedValue(0), getCompleted: vi.fn().mockResolvedValue([]), + getFailed: vi.fn().mockResolvedValue([]), } as never ); @@ -238,6 +268,7 @@ describe('ComputedOutboxMonitorService', () => { getJobCounts: vi.fn().mockRejectedValue(new Error('redis password leaked')), getWorkersCount: vi.fn(), getCompleted: vi.fn(), + getFailed: vi.fn(), } as never ); @@ -277,6 +308,7 @@ describe('ComputedOutboxMonitorService', () => { getJobCounts: vi.fn().mockResolvedValue({}), getWorkersCount: vi.fn().mockResolvedValue(1), getCompleted: vi.fn().mockResolvedValue([]), + getFailed: vi.fn().mockResolvedValue([]), } as never ); @@ -302,6 +334,7 @@ describe('ComputedOutboxMonitorService', () => { getJobCounts: vi.fn().mockResolvedValue({}), getWorkersCount: vi.fn().mockResolvedValue(1), getCompleted: vi.fn().mockResolvedValue([]), + getFailed: vi.fn().mockResolvedValue([]), } as never ); diff --git a/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-monitor.service.ts b/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-monitor.service.ts index d81f4699f6..1e0bde1fb3 100644 --- a/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-monitor.service.ts +++ b/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/computed-outbox-monitor.service.ts @@ -21,6 +21,7 @@ import { import { COMPUTED_OUTBOX_COMPLETED_RETENTION_COUNT, COMPUTED_OUTBOX_RECENT_COMPLETED_LIMIT, + COMPUTED_OUTBOX_RECENT_FAILED_LIMIT, COMPUTED_OUTBOX_WAKEUP_QUEUE, } from './constants'; import { mapWithConcurrency } from './map-with-concurrency'; @@ -68,6 +69,14 @@ export type ComputedOutboxMonitorSnapshot = { processingDurationMs?: number; attemptsMade: number; }>; + recentFailed: Array<{ + taskId: string; + baseId: string; + cause?: ComputedOutboxWakeupWire['cause']; + failedAt: string; + failedReason: string | null; + attemptsMade: number; + }>; error?: string; }; outbox: OutboxCounts & { @@ -229,6 +238,7 @@ export class ComputedOutboxMonitorService implements OnApplicationBootstrap, OnM completed: 0, completedRetentionLimit: COMPUTED_OUTBOX_COMPLETED_RETENTION_COUNT, recentCompleted: [], + recentFailed: [], }; } @@ -237,7 +247,7 @@ export class ComputedOutboxMonitorService implements OnApplicationBootstrap, OnM return { ...this.emptyQueue(false), error: 'BullMQ queue is not configured' }; } try { - const [counts, workers, completedJobs] = await Promise.all([ + const [counts, workers, completedJobs, failedJobs] = await Promise.all([ this.queue.getJobCounts( 'waiting', 'active', @@ -249,6 +259,7 @@ export class ComputedOutboxMonitorService implements OnApplicationBootstrap, OnM ), this.queue.getWorkersCount(), this.queue.getCompleted(0, COMPUTED_OUTBOX_RECENT_COMPLETED_LIMIT - 1), + this.queue.getFailed(0, COMPUTED_OUTBOX_RECENT_FAILED_LIMIT - 1), ]); return { configured: true, @@ -266,6 +277,10 @@ export class ComputedOutboxMonitorService implements OnApplicationBootstrap, OnM const summary = this.summarizeCompletedJob(job); return summary ? [summary] : []; }), + recentFailed: failedJobs.flatMap((job) => { + const summary = this.summarizeFailedJob(job); + return summary ? [summary] : []; + }), }; } catch (error) { this.logger.warn('computed:outbox:monitor_queue_failed', { @@ -296,6 +311,36 @@ export class ComputedOutboxMonitorService implements OnApplicationBootstrap, OnM }; } + private summarizeFailedJob( + job?: Job + ): ComputedOutboxMonitorSnapshot['queue']['recentFailed'][number] | null { + if (!job || !Number.isFinite(job.finishedOn)) return null; + const finishedOn = job.finishedOn as number; + const failedReason = + typeof job.failedReason === 'string' && job.failedReason.length > 0 + ? job.failedReason.slice(0, 2000) + : null; + const wakeupResult = computedOutboxWakeupWireSchema.safeParse(job.data); + if (!wakeupResult.success) { + return { + taskId: String(job.id ?? 'unknown'), + baseId: 'unknown', + failedAt: new Date(finishedOn).toISOString(), + failedReason, + attemptsMade: Math.max(0, job.attemptsMade ?? 0), + }; + } + + return { + taskId: wakeupResult.data.taskId, + baseId: wakeupResult.data.baseId, + cause: wakeupResult.data.cause, + failedAt: new Date(finishedOn).toISOString(), + failedReason, + attemptsMade: Math.max(0, job.attemptsMade ?? 0), + }; + } + private emptyOutbox(): ComputedOutboxMonitorSnapshot['outbox'] { return { ...emptyCounts(), diff --git a/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/constants.ts b/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/constants.ts index 7c0a2b5372..854ea900ae 100644 --- a/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/constants.ts +++ b/apps/nestjs-backend/src/features/v2/computed-outbox-trigger/constants.ts @@ -3,3 +3,6 @@ export const COMPUTED_OUTBOX_WAKEUP_JOB = 'computed-outbox-wakeup'; export const COMPUTED_OUTBOX_WAKEUP_PUBLISHER = Symbol('computedOutboxWakeupPublisher'); export const COMPUTED_OUTBOX_COMPLETED_RETENTION_COUNT = 2000; export const COMPUTED_OUTBOX_RECENT_COMPLETED_LIMIT = 10; +export const COMPUTED_OUTBOX_RECENT_FAILED_LIMIT = 10; +export const COMPUTED_OUTBOX_ANOMALY_GROUP_SAMPLE_LIMIT = 12; +export const COMPUTED_OUTBOX_ANOMALY_FETCH_CAP = 2000; diff --git a/apps/nestjs-backend/src/features/v2/table-query-search-observability.ts b/apps/nestjs-backend/src/features/v2/table-query-search-observability.ts new file mode 100644 index 0000000000..a2fb0157a0 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/table-query-search-observability.ts @@ -0,0 +1,194 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { metrics } from '@opentelemetry/api'; +import * as Sentry from '@sentry/nestjs'; +import { + createTableQueryMetricAttributes, + type ITableQueryObservability, + type SpanAttributes, + type TableQueryObservabilityEvent, + type TableQuerySearchValidationEvent, +} from '@teable/v2-core'; + +type SentryScopeLike = { + setTag(key: string, value: string): void; + setContext(key: string, value: Record): void; +}; + +const parseAllowlist = (value: string | undefined): ReadonlySet => + new Set( + (value ?? '') + .split(',') + .map((item) => item.trim()) + .filter(Boolean) + ); + +const getSentryScopes = (): SentryScopeLike[] => { + const sentryApi = Sentry as unknown as { + getCurrentScope?: () => SentryScopeLike | undefined; + getIsolationScope?: () => SentryScopeLike | undefined; + getCurrentHub?: () => { getScope?: () => SentryScopeLike | undefined }; + }; + + return [ + sentryApi.getCurrentScope?.(), + sentryApi.getIsolationScope?.(), + sentryApi.getCurrentHub?.()?.getScope?.(), + ].filter((scope): scope is SentryScopeLike => Boolean(scope)); +}; + +const setSentryTag = ( + scope: SentryScopeLike, + key: string, + value: string | number | boolean | undefined +) => { + if (value == null || value === '') return; + scope.setTag(key, String(value)); +}; + +const tableQueryMeter = metrics.getMeter('teable-observability'); + +const requestTotal = tableQueryMeter.createCounter('teable.table_query.requests.total', { + description: 'Table query requests by low-cardinality query and search access path attributes', +}); + +const errorTotal = tableQueryMeter.createCounter('teable.table_query.errors.total', { + description: 'Table query errors by low-cardinality query and search access path attributes', +}); + +const fallbackTotal = tableQueryMeter.createCounter('teable.table_query.search.fallback.total', { + description: 'Search access path fallback decisions by reason', +}); + +const validationTotal = tableQueryMeter.createCounter( + 'teable.table_query.search.validation.total', + { + description: 'Search vector validation attempts by status and access path', + } +); + +const requestDuration = tableQueryMeter.createHistogram('teable.table_query.duration.ms', { + description: 'Table query application duration in milliseconds', + unit: 'ms', +}); + +const dbParentDuration = tableQueryMeter.createHistogram('teable.table_query.db_parent.duration.ms', { + description: 'Parent application span duration for table-query database work', + unit: 'ms', +}); + +const validationDuration = tableQueryMeter.createHistogram( + 'teable.table_query.search.validation.duration.ms', + { + description: 'Search vector validation duration in milliseconds', + unit: 'ms', + } +); + +const validationCostDelta = tableQueryMeter.createHistogram( + 'teable.table_query.search.cost_delta_pct', + { + description: 'Search vector validation cost delta percentage', + unit: '%', + } +); + +export class TableQuerySearchMetricsService implements ITableQueryObservability { + private readonly tableIdMetricAllowlist: ReadonlySet; + + private readonly requestTotal = requestTotal; + + private readonly errorTotal = errorTotal; + + private readonly fallbackTotal = fallbackTotal; + + private readonly validationTotal = validationTotal; + + private readonly requestDuration = requestDuration; + + private readonly dbParentDuration = dbParentDuration; + + private readonly validationDuration = validationDuration; + + private readonly validationCostDelta = validationCostDelta; + + constructor(options?: { tableIdMetricAllowlist?: ReadonlySet }) { + this.tableIdMetricAllowlist = + options?.tableIdMetricAllowlist ?? + parseAllowlist(process.env.TABLE_QUERY_OBSERVABILITY_TABLE_IDS); + } + + recordRequest(event: TableQueryObservabilityEvent): void { + const attrs = this.metricAttributes(event); + this.requestTotal.add(1, attrs); + if (event.durationMs != null) { + this.requestDuration.record(event.durationMs, attrs); + if ( + event.querySource === 'repository.record_find' || + event.querySource === 'repository.record_count' + ) { + this.dbParentDuration.record(event.durationMs, attrs); + } + } + this.annotateSentry(event); + } + + recordError(event: TableQueryObservabilityEvent): void { + const attrs = this.metricAttributes(event); + this.errorTotal.add(1, attrs); + this.annotateSentry(event); + } + + recordSearchFallback(event: TableQueryObservabilityEvent): void { + this.fallbackTotal.add(1, this.metricAttributes(event)); + this.annotateSentry(event); + } + + recordSearchValidation(event: TableQuerySearchValidationEvent): void { + const attrs = this.metricAttributes({ + ...event, + errorKind: event.errorKind ?? event.validationStatus, + }); + this.validationTotal.add(1, attrs); + if (event.durationMs != null) { + this.validationDuration.record(event.durationMs, attrs); + } + if (event.costDeltaPct != null && Number.isFinite(event.costDeltaPct)) { + this.validationCostDelta.record(event.costDeltaPct, attrs); + } + this.annotateSentry(event); + } + + private metricAttributes(event: TableQueryObservabilityEvent): SpanAttributes { + return createTableQueryMetricAttributes({ + ...event, + includeTableId: event.tableId ? this.tableIdMetricAllowlist.has(event.tableId) : false, + }); + } + + private annotateSentry(event: TableQueryObservabilityEvent): void { + for (const scope of getSentryScopes()) { + setSentryTag(scope, 'teable.query.kind', event.queryKind); + setSentryTag(scope, 'teable.query.source', event.querySource); + setSentryTag(scope, 'teable.search.access_path', event.accessPath); + setSentryTag(scope, 'teable.search.mode', event.searchMode); + setSentryTag(scope, 'teable.search.fallback_reason', event.fallbackReason); + setSentryTag(scope, 'teable.table_id', event.tableId); + scope.setContext('table_query_search', { + queryKind: event.queryKind, + querySource: event.querySource, + tableId: event.tableId, + viewId: event.viewId, + searchMode: event.searchMode, + accessPath: event.accessPath, + searchScope: event.searchScope, + languageConfig: event.languageConfig, + fallbackReason: event.fallbackReason, + hasFilter: event.hasFilter, + hasSort: event.hasSort, + hasGroup: event.hasGroup, + includeTotal: event.includeTotal, + errorKind: event.errorKind, + }); + } + } +} diff --git a/apps/nestjs-backend/src/features/v2/table-query-search-vector-runtime.service.spec.ts b/apps/nestjs-backend/src/features/v2/table-query-search-vector-runtime.service.spec.ts new file mode 100644 index 0000000000..1271f52113 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/table-query-search-vector-runtime.service.spec.ts @@ -0,0 +1,95 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + TableQuerySearchVectorRuntimeService, + hasSearchValueForSearchVectorRuntime, + resolveTableQuerySearchVectorRuntimeMode, + toRecordSearchAccessPathFromConfig, +} from './table-query-search-vector-runtime.service'; + +describe('TableQuerySearchVectorRuntimeService', () => { + it.each([ + [undefined, 'off'], + ['', 'off'], + ['off', 'off'], + ['false', 'off'], + ['auto', 'auto'], + ['true', 'auto'], + ['enabled', 'auto'], + [true, 'auto'], + ] as const)('resolves runtime mode %s as %s', (input, expected) => { + expect(resolveTableQuerySearchVectorRuntimeMode(input)).toBe(expected); + }); + + it('converts a ready config row into a generated tsvector access path', () => { + const fieldId = `fld${'a'.repeat(16)}`; + + const accessPath = toRecordSearchAccessPathFromConfig({ + generatedColumnName: '__tqops_search_vector', + languageConfig: 'simple', + fieldIds: JSON.stringify([fieldId]), + searchScope: 'all_fields', + status: 'ready', + }); + + expect(accessPath).toMatchObject({ + kind: 'generated_tsvector', + generatedColumnName: '__tqops_search_vector', + languageConfig: 'simple', + searchScope: 'all_fields', + }); + expect(accessPath?.coveredFieldIds.map((id) => id.toString())).toEqual([fieldId]); + }); + + it('does not create an access path when covered fields are missing or invalid', () => { + expect( + toRecordSearchAccessPathFromConfig({ + generatedColumnName: '__tqops_search_vector', + languageConfig: 'simple', + fieldIds: JSON.stringify(['not-a-field']), + searchScope: 'all_fields', + status: 'ready', + }) + ).toBeUndefined(); + }); + + it('does not reactivate an older ready path when the latest config is pending', () => { + expect( + toRecordSearchAccessPathFromConfig({ + generatedColumnName: '__tqops_search_vector', + languageConfig: 'simple', + fieldIds: JSON.stringify([`fld${'a'.repeat(16)}`]), + searchScope: 'all_fields', + status: 'rebuild_pending', + }) + ).toBeUndefined(); + }); + + it.each([ + [undefined, false], + [[], false], + [[''], false], + [[' '], false], + [['order 123'], true], + ] as const)('resolves runtime search usability for %j as %s', (search, expected) => { + expect(hasSearchValueForSearchVectorRuntime(search)).toBe(expected); + }); + + it('does not read meta config when the global runtime gate is off', async () => { + const service = new TableQuerySearchVectorRuntimeService({ + get: vi.fn().mockReturnValue('off'), + } as never); + const container = { + isRegistered: vi.fn(), + }; + + await expect( + service.resolveForRecordSearch({ + container: container as never, + tableId: `tbl${'a'.repeat(16)}`, + search: ['order 123'], + }) + ).resolves.toBeUndefined(); + expect(container.isRegistered).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/table-query-search-vector-runtime.service.ts b/apps/nestjs-backend/src/features/v2/table-query-search-vector-runtime.service.ts new file mode 100644 index 0000000000..dcc615adef --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/table-query-search-vector-runtime.service.ts @@ -0,0 +1,159 @@ +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { v2MetaDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { FieldId, type IRecordSearchAccessPath } from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; +import type { Kysely } from 'kysely'; +import { sql } from 'kysely'; + +type UnknownRow = Record; + +export type SearchVectorConfigRow = { + readonly generatedColumnName: string; + readonly languageConfig: string; + readonly fieldIds: unknown; + readonly searchScope: string; + readonly status: string; +}; + +export type TableQuerySearchVectorRuntimeMode = 'off' | 'auto'; + +export const tableQuerySearchVectorRuntimeEnv = 'V2_TABLE_QUERY_OPS_SEARCH_VECTOR_RUNTIME'; + +export const resolveTableQuerySearchVectorRuntimeMode = ( + value: unknown +): TableQuerySearchVectorRuntimeMode => { + if (typeof value === 'boolean') { + return value ? 'auto' : 'off'; + } + + if (typeof value !== 'string') { + return 'off'; + } + + const normalized = value.trim().toLowerCase(); + if (['1', 'true', 'yes', 'on', 'enabled', 'auto'].includes(normalized)) { + return 'auto'; + } + + return 'off'; +}; + +const parseFieldIds = (raw: unknown): readonly FieldId[] => { + const parsed = + typeof raw === 'string' + ? (() => { + try { + return JSON.parse(raw) as unknown; + } catch { + return undefined; + } + })() + : raw; + + if (!Array.isArray(parsed)) { + return []; + } + + return parsed.flatMap((value) => { + const fieldIdResult = FieldId.create(value); + return fieldIdResult.isOk() ? [fieldIdResult.value] : []; + }); +}; + +export const toRecordSearchAccessPathFromConfig = ( + row: SearchVectorConfigRow | undefined +): IRecordSearchAccessPath | undefined => { + if (!row) { + return undefined; + } + + if (row.status !== 'ready') { + return undefined; + } + + const searchScope = + row.searchScope === 'all_fields' || row.searchScope === 'selected_fields' + ? row.searchScope + : undefined; + const coveredFieldIds = parseFieldIds(row.fieldIds); + if ( + !row.generatedColumnName || + !row.languageConfig || + !searchScope || + coveredFieldIds.length === 0 + ) { + return undefined; + } + + return { + kind: 'generated_tsvector', + generatedColumnName: row.generatedColumnName, + languageConfig: row.languageConfig, + searchScope, + coveredFieldIds, + }; +}; + +export const hasSearchValueForSearchVectorRuntime = (search: unknown): boolean => { + if (!Array.isArray(search)) { + return false; + } + + const [value] = search; + return typeof value === 'string' && value.trim().length > 0; +}; + +@Injectable() +export class TableQuerySearchVectorRuntimeService { + constructor(private readonly configService: ConfigService) {} + + async resolveForRecordSearch(input: { + readonly container: DependencyContainer; + readonly tableId: string; + readonly search: unknown; + }): Promise { + if (!hasSearchValueForSearchVectorRuntime(input.search) || this.mode() !== 'auto') { + return undefined; + } + + try { + const row = await this.readReadyConfig(input.container, input.tableId); + return toRecordSearchAccessPathFromConfig(row); + } catch { + return undefined; + } + } + + private mode(): TableQuerySearchVectorRuntimeMode { + return resolveTableQuerySearchVectorRuntimeMode( + this.configService.get(tableQuerySearchVectorRuntimeEnv) + ); + } + + private async readReadyConfig( + container: DependencyContainer, + tableId: string + ): Promise { + if (!container.isRegistered(v2MetaDbTokens.db)) { + return undefined; + } + + const metaDb = container.resolve>(v2MetaDbTokens.db); + const result = await sql` + SELECT + generated_column_name AS "generatedColumnName", + language_config AS "languageConfig", + field_ids AS "fieldIds", + search_scope AS "searchScope", + status + FROM table_query_search_vector_config + WHERE table_id = ${tableId} + AND status IN ('ready', 'rebuild_pending', 'stale') + ORDER BY last_modified_time DESC NULLS LAST, created_time DESC NULLS LAST + LIMIT 1 + `.execute(metaDb); + + return result.rows[0]; + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts index 5dc7d68091..e8baca27e2 100644 --- a/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts +++ b/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts @@ -124,6 +124,9 @@ const createContainerMock = (): DependencyContainer => { if (token === v2CoreTokens.undoRedoStore) { return undefined; } + if (token === v2CoreTokens.tracer) { + return undefined; + } throw new Error(`Unexpected token: ${String(token)}`); }), } as unknown as DependencyContainer; @@ -134,7 +137,10 @@ const createService = (providers: InstanceWrapper[] = []) => { getOrThrow: vi.fn().mockReturnValue('postgres://test'), get: vi.fn().mockReturnValue(undefined), }; - const shareDbService = { pubsub: { publish: vi.fn() } }; + const shareDbService = { + pubsub: { publish: vi.fn() }, + setComputedActivitySnapshotLoader: vi.fn(), + }; const cacheService = { getKeyv: vi.fn().mockReturnValue({}) }; const attachmentsStorageService = { getPreviewUrlByPath: vi.fn(), @@ -181,7 +187,10 @@ const createTestingModule = async (providers: InstanceWrapper[] = []) => { getOrThrow: vi.fn().mockReturnValue('postgres://test'), get: vi.fn().mockReturnValue(undefined), }; - const shareDbService = { pubsub: { publish: vi.fn() } }; + const shareDbService = { + pubsub: { publish: vi.fn() }, + setComputedActivitySnapshotLoader: vi.fn(), + }; const cacheService = { getKeyv: vi.fn().mockReturnValue({}) }; const attachmentsStorageService = { getPreviewUrlByPath: vi.fn(), @@ -399,6 +408,56 @@ describe('V2ContainerService', () => { ); }); + it('starts internal search-vector maintenance when generated search runtime is enabled', async () => { + const container = createContainerMock(); + mocks.createV2NodePgContainer.mockResolvedValue(container); + const { service, configService } = createService(); + + configService.get.mockImplementation((key: string) => { + if (key === 'V2_TABLE_QUERY_OPS_ENABLED') return 'true'; + if (key === 'V2_TABLE_QUERY_OPS_SEARCH_VECTOR_RUNTIME') return 'auto'; + return undefined; + }); + + await service.getContainer(); + + expect(mocks.createV2NodePgContainer).toHaveBeenCalledWith( + expect.objectContaining({ + tableQueryOps: expect.objectContaining({ + taskWorkerConfig: expect.objectContaining({ + enabled: true, + allowManualIndexExecution: false, + allowedKinds: ['rebuild_search_vector', 'manual_investigation'], + }), + }), + }) + ); + }); + + it('preserves an explicit remediation allowlist when search runtime is enabled', async () => { + const container = createContainerMock(); + mocks.createV2NodePgContainer.mockResolvedValue(container); + const { service, configService } = createService(); + + configService.get.mockImplementation((key: string) => { + if (key === 'V2_TABLE_QUERY_OPS_ENABLED') return 'true'; + if (key === 'V2_TABLE_QUERY_OPS_SEARCH_VECTOR_RUNTIME') return 'auto'; + if (key === 'V2_TABLE_QUERY_OPS_ALLOWED_TASK_KINDS') return 'manual_investigation'; + return undefined; + }); + + await service.getContainer(); + + expect(mocks.createV2NodePgContainer).toHaveBeenCalledWith( + expect.objectContaining({ + tableQueryOps: expect.objectContaining({ + taskWorkerConfig: expect.objectContaining({ + allowedKinds: ['manual_investigation'], + }), + }), + }) + ); + }); it('enables table query ops by default in preview runtime', async () => { const container = createContainerMock(); mocks.createV2NodePgContainer.mockResolvedValue(container); diff --git a/apps/nestjs-backend/src/features/v2/v2-container.service.ts b/apps/nestjs-backend/src/features/v2/v2-container.service.ts index 391cca585c..1b28aecf05 100644 --- a/apps/nestjs-backend/src/features/v2/v2-container.service.ts +++ b/apps/nestjs-backend/src/features/v2/v2-container.service.ts @@ -17,9 +17,15 @@ import { createV2NodePgContainer, type IV2NodePgContainerOptions } from '@teable import type { AttachmentValueDecoratorService, IAttachmentLookupService, + IComputedActivityReader, IExecutionContext, } from '@teable/v2-core'; -import { ActorId, v2CoreTokens } from '@teable/v2-core'; +import { + ActorId, + mapFieldComputeActivityToRealtime, + mapTableComputeActivityToRealtime, + v2CoreTokens, +} from '@teable/v2-core'; import type { DependencyContainer } from '@teable/v2-di'; import { registerV2ImportServices } from '@teable/v2-import'; import { @@ -40,6 +46,8 @@ import { import { ShareDbService } from '../../share-db/share-db.service'; import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; import { COMPUTED_OUTBOX_WAKEUP_PUBLISHER } from './computed-outbox-trigger/constants'; +import { TableQuerySearchMetricsService } from './table-query-search-observability'; +import { resolveTableQuerySearchVectorRuntimeMode } from './table-query-search-vector-runtime.service'; import { V2AttachmentUrlSignerService } from './v2-attachment-url-signer.service'; import { CommandBusTracingMiddleware } from './v2-command-bus-tracing.middleware'; import { PinoLoggerAdapter } from './v2-logger.adapter'; @@ -69,6 +77,8 @@ const resolveBoolean = (value: unknown, defaultValue = false): boolean => { const executablePhase1RemediationKinds = [ 'create_search_index', + 'create_search_vector', + 'rebuild_search_vector', 'create_filter_index', 'create_sort_index', 'repair_index', @@ -111,7 +121,31 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr @Optional() @Inject(COMPUTED_OUTBOX_WAKEUP_PUBLISHER) private readonly computedOutboxWakeupPublisher: IComputedOutboxWakeupPublisher = noopComputedOutboxWakeupPublisher - ) {} + ) { + this.shareDbService.setComputedActivitySnapshotLoader(async (tableId) => { + const container = await this.getContainerForTable(tableId); + const reader = container.resolve( + v2CoreTokens.computedActivityReader + ); + const result = await reader.getByTableId(undefined, tableId); + if (result.isErr()) throw result.error; + + const documents: Record = {}; + if (result.value.table) { + documents.table = { + version: result.value.table.generation, + data: mapTableComputeActivityToRealtime(result.value.table), + }; + } + for (const field of result.value.fields) { + documents[field.fieldId] = { + version: field.generation, + data: mapFieldComputeActivityToRealtime(field), + }; + } + return documents; + }); + } async onApplicationBootstrap(): Promise { await this.getContainer(); @@ -148,6 +182,14 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr ); } + isTableQuerySearchVectorRuntimeEnabled(): boolean { + return ( + resolveTableQuerySearchVectorRuntimeMode( + this.configService.get('V2_TABLE_QUERY_OPS_SEARCH_VECTOR_RUNTIME') + ) === 'auto' + ); + } + async getContainerForMaintenanceTarget( target: IComputedOutboxMaintenanceTarget ): Promise { @@ -186,6 +228,7 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr const metaConnectionString = this.getMetaConnectionString(); const logger = new PinoLoggerAdapter(this.pinoLogger); const tracer = new OpenTelemetryTracer(); + const tableQueryObservability = new TableQuerySearchMetricsService(); const commandBusMiddlewares = [new CommandBusTracingMiddleware()]; const queryBusMiddlewares = [new QueryBusTracingMiddleware()]; const computedUpdateMode = process.env.V2_COMPUTED_UPDATE_MODE; @@ -215,6 +258,7 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr dataSchema, logger, tracer, + tableQueryObservability, commandBusMiddlewares, queryBusMiddlewares, computedUpdate, @@ -282,13 +326,23 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr const allowManualIndexExecution = resolveBoolean( this.configService.get('V2_TABLE_QUERY_OPS_ALLOW_MANUAL_INDEX_EXECUTION') ); - const allowedKinds = + const searchVectorRuntimeEnabled = + resolveTableQuerySearchVectorRuntimeMode( + this.configService.get('V2_TABLE_QUERY_OPS_SEARCH_VECTOR_RUNTIME') + ) === 'auto'; + const configuredAllowedKinds = parseAllowedRemediationKinds( this.configService.get('V2_TABLE_QUERY_OPS_ALLOWED_TASK_KINDS') ) ?? (allowManualIndexExecution ? executablePhase1RemediationKinds - : (['manual_investigation'] satisfies ReadonlyArray)); + : searchVectorRuntimeEnabled + ? ([ + 'rebuild_search_vector', + 'manual_investigation', + ] satisfies ReadonlyArray) + : (['manual_investigation'] satisfies ReadonlyArray)); + const allowedKinds = configuredAllowedKinds; const analyzerIntervalMs = resolvePositiveInteger( this.configService.get('V2_TABLE_QUERY_OPS_ANALYZER_INTERVAL_MS') ); @@ -328,7 +382,10 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr ...(analyzerBatchSize ? { batchSize: analyzerBatchSize } : {}), }, taskWorkerConfig: { - enabled: resolveBoolean(this.configService.get('V2_TABLE_QUERY_OPS_TASK_WORKER_ENABLED')), + enabled: resolveBoolean( + this.configService.get('V2_TABLE_QUERY_OPS_TASK_WORKER_ENABLED'), + searchVectorRuntimeEnabled + ), workerId: `${workerId}:task-worker`, allowManualIndexExecution, allowedKinds, diff --git a/apps/nestjs-backend/src/features/v2/v2.controller.compute-activity.spec.ts b/apps/nestjs-backend/src/features/v2/v2.controller.compute-activity.spec.ts new file mode 100644 index 0000000000..545afb7135 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2.controller.compute-activity.spec.ts @@ -0,0 +1,68 @@ +import { + ActorId, + GetComputeActivityQuery, + GetComputeActivityResult, + v2CoreTokens, + type IExecutionContext, + type TableComputeActivitySnapshot, +} from '@teable/v2-core'; +import { ok } from 'neverthrow'; +import { describe, expect, it, vi } from 'vitest'; + +import { V2Controller } from './v2.controller'; + +const baseId = `bse${'a'.repeat(16)}`; +const tableId = `tbl${'a'.repeat(16)}`; + +const snapshot: TableComputeActivitySnapshot = { + baseId, + tableId, + table: null, + fields: [], + diagnostics: { + computeMode: 'server', + activeFieldCount: 0, + queuedFieldCount: 0, + calculatingFieldCount: 0, + failedFieldCount: 0, + highComplexityFieldCount: 0, + anomalies: [], + }, +}; + +describe('V2Controller compute activity route', () => { + it('routes the request through the table container, context, and query bus', async () => { + const context: IExecutionContext = { + actorId: ActorId.create('system')._unsafeUnwrap(), + }; + const execute = vi.fn().mockResolvedValue(ok(GetComputeActivityResult.create(snapshot))); + const queryBus = { execute }; + const resolve = vi.fn((token: symbol) => { + if (token === v2CoreTokens.queryBus) return queryBus; + throw new Error(`Unexpected token: ${String(token)}`); + }); + const container = { resolve }; + const getContainerForTable = vi.fn().mockResolvedValue(container); + const createContext = vi.fn().mockResolvedValue(context); + const controller = new V2Controller( + { getContainerForTable } as never, + { createContext } as never + ); + + const result = await controller.tables().getComputeActivity.callable()({ baseId, tableId }); + + expect(result).toEqual({ + ok: true, + data: snapshot, + }); + expect(getContainerForTable).toHaveBeenCalledWith(tableId); + expect(resolve).toHaveBeenCalledWith(v2CoreTokens.queryBus); + expect(createContext).toHaveBeenCalledWith(container); + expect(execute).toHaveBeenCalledOnce(); + const [executedContext, query] = execute.mock.calls[0]!; + expect(executedContext).toBe(context); + expect(query).toBeInstanceOf(GetComputeActivityQuery); + expect(query.baseId.toString()).toBe(baseId); + expect(query.tableId.toString()).toBe(tableId); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/v2.controller.ts b/apps/nestjs-backend/src/features/v2/v2.controller.ts index 28bb61d70f..70b1d3de1d 100644 --- a/apps/nestjs-backend/src/features/v2/v2.controller.ts +++ b/apps/nestjs-backend/src/features/v2/v2.controller.ts @@ -6,11 +6,16 @@ import { v2Contract } from '@teable/v2-contract-http'; import { executeCreateTableEndpoint, executeDeleteRecordsEndpoint, + executeGetComputeActivityEndpoint, executeGetTableByIdEndpoint, executeUpdateRecordsEndpoint, } from '@teable/v2-contract-http-implementation/handlers'; import { v2CoreTokens } from '@teable/v2-core'; -import type { IQueryBus, ICommandBus } from '@teable/v2-core' with { 'resolution-mode': 'import' }; +import type { + ICommandBus, + IComputedActivityReader, + IQueryBus, +} from '@teable/v2-core' with { 'resolution-mode': 'import' }; import { V2ContainerService } from './v2-container.service'; import { V2ExecutionContextFactory } from './v2-execution-context.factory'; @@ -59,12 +64,32 @@ export class V2Controller { const container = await this.v2Container.getContainerForTable(input.tableId); const queryBus = container.resolve(v2CoreTokens.queryBus); const context = await this.v2ContextFactory.createContext(container); - - const result = await executeGetTableByIdEndpoint(context, input, queryBus); + let activityReader: IComputedActivityReader | undefined; + try { + activityReader = container.resolve( + v2CoreTokens.computedActivityReader + ); + } catch { + activityReader = undefined; + } + + const result = await executeGetTableByIdEndpoint(context, input, queryBus, activityReader); if (result.status === 200) return result.body; throwOrpcErrorByStatus(result.status, result.body.error); }), + getComputeActivity: implement(v2Contract.tables.getComputeActivity).handler( + async ({ input }) => { + const container = await this.v2Container.getContainerForTable(input.tableId); + const queryBus = container.resolve(v2CoreTokens.queryBus); + const context = await this.v2ContextFactory.createContext(container); + + const result = await executeGetComputeActivityEndpoint(context, input, queryBus); + if (result.status === 200) return result.body; + + throwOrpcErrorByStatus(result.status, result.body.error); + } + ), deleteRecords: implement(v2Contract.tables.deleteRecords).handler(async ({ input }) => { const container = await this.v2Container.getContainerForTable(input.tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); diff --git a/apps/nestjs-backend/src/global/data-db-client-manager.service.ts b/apps/nestjs-backend/src/global/data-db-client-manager.service.ts index 695e510a09..133c2e6850 100644 --- a/apps/nestjs-backend/src/global/data-db-client-manager.service.ts +++ b/apps/nestjs-backend/src/global/data-db-client-manager.service.ts @@ -58,6 +58,10 @@ export type IComputedOutboxMaintenanceAnomaly = { attempts: number; maxAttempts: number; lastError: string | null; + failedSql: string | null; + failureKind: string | null; + failurePhase: string | null; + affectedTableName: string | null; occurredAt: Date; }; @@ -440,7 +444,7 @@ export class DataDbClientManager { acquireConnectionTimeout: COMPUTED_OUTBOX_MAINTENANCE_CONNECT_TIMEOUT_MS, pool: { min: 0, max: 1 }, }); - const normalizedLimit = Math.max(1, Math.min(100, Math.trunc(limit))); + const normalizedLimit = Math.max(1, Math.min(2000, Math.trunc(limit))); const baseSpaceMapping = target.baseSpaceMapping ?? []; const pauseSpaceJoin = target.storage === 'default' @@ -470,6 +474,10 @@ export class DataDbClientManager { attempts: number | string; maxAttempts: number | string; lastError: string | null; + failedSql: string | null; + failureKind: string | null; + failurePhase: string | null; + affectedTableName: string | null; occurredAt: Date | string; total: number | string; }>; @@ -483,6 +491,10 @@ export class DataDbClientManager { attempts, max_attempts as "maxAttempts", left(last_error, 2000) as "lastError", + left(trace_data #>> '{execution,statement,normalizedSql}', 4000) as "failedSql", + left(trace_data #>> '{failure,kind}', 128) as "failureKind", + left(trace_data #>> '{failure,phase}', 128) as "failurePhase", + left(trace_data #>> '{execution,context,tableName}', 256) as "affectedTableName", failed_at as "occurredAt" from computed_update_dead_letter union all @@ -494,6 +506,10 @@ export class DataDbClientManager { o.attempts, o.max_attempts as "maxAttempts", left(o.last_error, 2000) as "lastError", + null::text as "failedSql", + null::text as "failureKind", + null::text as "failurePhase", + null::text as "affectedTableName", coalesce(o.locked_at, o.updated_at) as "occurredAt" from computed_update_outbox as o ${pauseSpaceJoin} @@ -534,6 +550,10 @@ export class DataDbClientManager { attempts: Number(row.attempts), maxAttempts: Number(row.maxAttempts), lastError: row.lastError, + failedSql: row.failedSql, + failureKind: row.failureKind, + failurePhase: row.failurePhase, + affectedTableName: row.affectedTableName, occurredAt: new Date(row.occurredAt), })), }; diff --git a/apps/nestjs-backend/src/share-db/readonly/field-readonly.service.ts b/apps/nestjs-backend/src/share-db/readonly/field-readonly.service.ts index c647025bdd..ae2ac775ba 100644 --- a/apps/nestjs-backend/src/share-db/readonly/field-readonly.service.ts +++ b/apps/nestjs-backend/src/share-db/readonly/field-readonly.service.ts @@ -59,6 +59,20 @@ export class FieldReadonlyServiceAdapter }) .then((res) => res.data); } + authorizeComputedActivityRead(tableId: string): Promise { + const shareId = this.cls.get('shareViewId'); + const baseShareId = this.cls.get('baseShareId'); + if (!shareId || baseShareId) { + return this.getDocIdsByQuery(tableId).then(() => undefined); + } + + return this.axios + .get(`/share/${shareId}/socket/computed-activity/authorize`, { + headers: { cookie: this.cls.get('cookie') }, + params: { tableId }, + }) + .then(() => undefined); + } getVersionAndType(tableId: string, fieldId: string) { return this.prismaService.field diff --git a/apps/nestjs-backend/src/share-db/share-db.adapter.ts b/apps/nestjs-backend/src/share-db/share-db.adapter.ts index d66c230716..97b6c541f4 100644 --- a/apps/nestjs-backend/src/share-db/share-db.adapter.ts +++ b/apps/nestjs-backend/src/share-db/share-db.adapter.ts @@ -50,9 +50,16 @@ export interface ICollectionSnapshot { type IProjection = { [fieldNameOrId: string]: boolean }; +const computedActivityCollectionPrefix = 'cmp'; + +export type ComputedActivitySnapshotLoader = ( + tableId: string +) => Promise>>; + @Injectable() export class ShareDbAdapter extends ShareDb.DB { private logger = new Logger(ShareDbAdapter.name); + private computedActivitySnapshotLoader?: ComputedActivitySnapshotLoader; // Read by sharedb QueryEmitter (lib/query-emitter.js): ops arriving while a // poll is in flight or within this window are coalesced into a single @@ -74,6 +81,10 @@ export class ShareDbAdapter extends ShareDb.DB { this.closed = false; } + setComputedActivitySnapshotLoader(loader: ComputedActivitySnapshotLoader): void { + this.computedActivitySnapshotLoader = loader; + } + getReadonlyService(type: IdPrefix): IShareDbReadonlyAdapterService { switch (type) { case IdPrefix.View: @@ -241,6 +252,28 @@ export class ShareDbAdapter extends ShareDb.DB { return this.snapshots2Map(snapshots); } + private async loadComputedActivitySnapshots( + tableId: string, + ids: string[] + ): Promise[]> { + await this.fieldService.authorizeComputedActivityRead(tableId); + if (!this.computedActivitySnapshotLoader) return []; + + const documents = await this.computedActivitySnapshotLoader(tableId); + return ids.flatMap((id) => { + const document = documents[id]; + if (!document) return []; + return [ + { + id, + v: Math.max(1, Math.trunc(document.version)), + type: 'json0', + data: document.data, + }, + ]; + }); + } + // Get the named document from the database. The callback is called with (err, // snapshot). A snapshot with a version of zero is returned if the document // has never been created in the database. @@ -272,6 +305,9 @@ export class ShareDbAdapter extends ShareDb.DB { ...authHeaders, }, async () => { + if (docType === computedActivityCollectionPrefix) { + return this.loadComputedActivitySnapshots(collectionId, ids); + } return this.getReadonlyService(docType as IdPrefix).getSnapshotBulk( collectionId, ids, @@ -305,7 +341,7 @@ export class ShareDbAdapter extends ShareDb.DB { } private async getSnapshotData( - docType: IdPrefix, + docType: string, collectionId: string, ids: string[], // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -326,6 +362,9 @@ export class ShareDbAdapter extends ShareDb.DB { ...authHeaders, }, async () => { + if (docType === computedActivityCollectionPrefix) { + return await this.loadComputedActivitySnapshots(collectionId, ids); + } return await this.getReadonlyService(docType as IdPrefix).getSnapshotBulk( collectionId, ids @@ -344,6 +383,15 @@ export class ShareDbAdapter extends ShareDb.DB { return snapshots; } + private getComputedActivityVersionAndType(snapshot?: ISnapshotBase): { + version: number; + type: RawOpType; + } { + if (!snapshot) return { version: 0, type: RawOpType.Del }; + if (snapshot.v === 1) return { version: 0, type: RawOpType.Create }; + return { version: snapshot.v - 1, type: RawOpType.Edit }; + } + private hasGapVersion({ opType, currentVersion, @@ -459,7 +507,17 @@ export class ShareDbAdapter extends ShareDb.DB { options: any, callback: (error: unknown, data?: unknown) => void ) { - const [docType] = collection.split('_'); + const [docType, collectionId] = collection.split('_'); + if (docType === computedActivityCollectionPrefix) { + const snapshots = await this.getSnapshotData(docType, collectionId, [id], options); + const snapshot = snapshots[0]; + await this.internalGetOps(collection, id, from, to, options, callback, { + getVersionAndType: async () => this.getComputedActivityVersionAndType(snapshot), + getSnapshotData: async () => (snapshot ? [snapshot] : []), + }); + return; + } + const readonlyService = this.getReadonlyService(docType as IdPrefix); await this.internalGetOps(collection, id, from, to, options, callback, { getVersionAndType: async (...args) => await readonlyService.getVersionAndType(...args), @@ -476,11 +534,21 @@ export class ShareDbAdapter extends ShareDb.DB { callback: (error: unknown, data?: unknown) => void ) { const [docType, collectionId] = collection.split('_'); - const readonlyService = this.getReadonlyService(docType as IdPrefix); - const versionAndTypeMap = await readonlyService.getVersionAndTypeMap( - collectionId, - Object.keys(fromMap) - ); + const activitySnapshots = + docType === computedActivityCollectionPrefix + ? await this.getSnapshotData(docType, collectionId, Object.keys(fromMap), options) + : null; + const versionAndTypeMap = activitySnapshots + ? Object.fromEntries( + activitySnapshots.map((snapshot) => [ + snapshot.id, + this.getComputedActivityVersionAndType(snapshot), + ]) + ) + : await this.getReadonlyService(docType as IdPrefix).getVersionAndTypeMap( + collectionId, + Object.keys(fromMap) + ); const needGetSnapshotDataIds: string[] = []; for (const [id, from] of Object.entries(fromMap)) { const versionAndType = versionAndTypeMap[id]; @@ -498,20 +566,16 @@ export class ShareDbAdapter extends ShareDb.DB { } } - const snapshotDataMap = await this.getSnapshotData( - docType as IdPrefix, - collectionId, - needGetSnapshotDataIds, - options - ).then((snapshots) => { - return snapshots.reduce( - (acc, snapshot) => { - acc[snapshot.id] = snapshot; - return acc; - }, - {} as Record> - ); - }); + const snapshots = + activitySnapshots ?? + (await this.getSnapshotData(docType, collectionId, needGetSnapshotDataIds, options)); + const snapshotDataMap = snapshots.reduce( + (acc, snapshot) => { + acc[snapshot.id] = snapshot; + return acc; + }, + {} as Record> + ); const result: Record = {}; for (const [id, from] of Object.entries(fromMap)) { let resultError: unknown = null; @@ -544,7 +608,10 @@ export class ShareDbAdapter extends ShareDb.DB { callback(null, result); } - private getOpsFromSnapshot(docType: IdPrefix, snapshot: unknown): IOtOperation[] { + private getOpsFromSnapshot(docType: string, snapshot: unknown): IOtOperation[] { + if (docType === computedActivityCollectionPrefix) { + return [{ p: [], oi: snapshot }]; + } switch (docType) { case IdPrefix.Record: return Object.entries((snapshot as IRecord).fields).map(([fieldId, fieldValue]) => { diff --git a/apps/nestjs-backend/src/share-db/share-db.service.ts b/apps/nestjs-backend/src/share-db/share-db.service.ts index 8e48cc9301..a6b959cc3b 100644 --- a/apps/nestjs-backend/src/share-db/share-db.service.ts +++ b/apps/nestjs-backend/src/share-db/share-db.service.ts @@ -16,7 +16,7 @@ import { authMiddleware } from './auth.middleware'; import type { IRawOpMap } from './interface'; import { RealtimeMetricsService } from './metrics/realtime-metrics.service'; import { RepairAttachmentOpService } from './repair-attachment-op/repair-attachment-op.service'; -import { ShareDbAdapter } from './share-db.adapter'; +import { ShareDbAdapter, type ComputedActivitySnapshotLoader } from './share-db.adapter'; import { RedisPubSub } from './sharedb-redis.pubsub'; const v2ProjectionOpSourcePrefix = '@@v2-projection:'; @@ -105,6 +105,10 @@ export class ShareDbService extends ShareDBClass { return this.connect(); } + setComputedActivitySnapshotLoader(loader: ComputedActivitySnapshotLoader): void { + this.shareDbAdapter.setComputedActivitySnapshotLoader(loader); + } + @Timing() private async updateTableMetaByRawOpMap(rawOpMap?: IRawOpMap[]) { if (!rawOpMap?.length) { diff --git a/apps/nestjs-backend/src/share-db/share-db.spec.ts b/apps/nestjs-backend/src/share-db/share-db.spec.ts index 1820036315..f5867dd451 100644 --- a/apps/nestjs-backend/src/share-db/share-db.spec.ts +++ b/apps/nestjs-backend/src/share-db/share-db.spec.ts @@ -72,6 +72,150 @@ describe('ShareDb', () => { }); }); + it('serves versioned compute activity snapshots after field-read authorization', async () => { + const cls = { + get: vi.fn(() => undefined), + runWith: vi.fn((_store, fn) => fn()), + }; + const fieldService = { + authorizeComputedActivityRead: vi.fn().mockResolvedValue(undefined), + getSnapshotBulk: vi.fn().mockResolvedValue([]), + }; + const adapter = new ShareDbAdapter( + cls as never, + {} as never, + {} as never, + fieldService as never, + {} as never, + {} as never + ); + const loader = vi.fn().mockResolvedValue({ + table: { version: 4, data: { status: 'calculating', generation: 4 } }, + fldFormula: { version: 5, data: { status: 'running', generation: 5 } }, + }); + adapter.setComputedActivitySnapshotLoader(loader); + + const snapshots = await new Promise< + Record + >((resolve, reject) => { + adapter.getSnapshotBulk( + 'cmp_tblTest', + ['table', 'fldFormula', 'fldMissing'], + undefined, + { cookie: 'teable-session=test' }, + (error, data) => { + if (error) { + reject(error); + return; + } + resolve(data as Record); + } + ); + }); + + expect(fieldService.authorizeComputedActivityRead).toHaveBeenCalledWith('tblTest'); + expect(loader).toHaveBeenCalledWith('tblTest'); + expect(snapshots.table).toMatchObject({ + v: 4, + type: 'json0', + data: { status: 'calculating', generation: 4 }, + }); + expect(snapshots.fldFormula).toMatchObject({ + v: 5, + type: 'json0', + data: { status: 'running', generation: 5 }, + }); + expect(snapshots.fldMissing).toMatchObject({ v: 0, type: null, data: undefined }); + }); + + it('reconstructs compute activity operation gaps from the latest snapshot', async () => { + const cls = { + get: vi.fn(() => undefined), + runWith: vi.fn((_store, fn) => fn()), + }; + const fieldService = { + authorizeComputedActivityRead: vi.fn().mockResolvedValue(undefined), + getSnapshotBulk: vi.fn().mockResolvedValue([]), + }; + const adapter = new ShareDbAdapter( + cls as never, + {} as never, + {} as never, + fieldService as never, + {} as never, + {} as never + ); + adapter.setComputedActivitySnapshotLoader(async () => ({ + fldFormula: { version: 3, data: { status: 'idle', generation: 3 } }, + })); + + const ops = await new Promise>((resolve, reject) => { + adapter.getOps( + 'cmp_tblTest', + 'fldFormula', + 1, + null, + { cookie: 'teable-session=test' }, + (error, data) => { + if (error) { + reject(error); + return; + } + resolve(data as Array<{ v: number; op?: unknown[] }>); + } + ); + }); + + expect(ops.map((op) => op.v)).toEqual([1, 2]); + expect(ops.at(-1)?.op).toEqual([{ p: [], oi: { status: 'idle', generation: 3 } }]); + }); + + it('reconstructs the first compute activity generation as a create operation', async () => { + const cls = { + get: vi.fn(() => undefined), + runWith: vi.fn((_store, fn) => fn()), + }; + const fieldService = { + authorizeComputedActivityRead: vi.fn().mockResolvedValue(undefined), + getSnapshotBulk: vi.fn().mockResolvedValue([]), + }; + const adapter = new ShareDbAdapter( + cls as never, + {} as never, + {} as never, + fieldService as never, + {} as never, + {} as never + ); + adapter.setComputedActivitySnapshotLoader(async () => ({ + fldFormula: { version: 1, data: { status: 'queued', generation: 1 } }, + })); + + const ops = await new Promise>((resolve, reject) => { + adapter.getOps( + 'cmp_tblTest', + 'fldFormula', + 0, + null, + { cookie: 'teable-session=test' }, + (error, data) => { + if (error) { + reject(error); + return; + } + resolve(data as Array<{ v: number; create?: unknown }>); + } + ); + }); + + expect(ops).toMatchObject([ + { + v: 0, + create: { type: 'json0', data: { status: 'queued', generation: 1 } }, + }, + ]); + }); + // it('create simple document', (done) => { // const randomTitle = `B:${Math.floor(Math.random() * 1000)}`; // const doc = provider.connect().get('books', randomTitle); diff --git a/apps/nestjs-backend/src/tracing-span-export.spec.ts b/apps/nestjs-backend/src/tracing-span-export.spec.ts new file mode 100644 index 0000000000..ac9639e768 --- /dev/null +++ b/apps/nestjs-backend/src/tracing-span-export.spec.ts @@ -0,0 +1,587 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { SpanContext } from '@opentelemetry/api'; +import { context, SpanKind, SpanStatusCode, trace, TraceFlags } from '@opentelemetry/api'; +import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base'; +import type { ReadableSpan, SpanExporter, SpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { ATTR_HTTP_RESPONSE_STATUS_CODE } from '@opentelemetry/semantic-conventions'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + createSmartSpanProcessor, + getTraceDecision, + hashTraceId, + isDroppedPrismaSpan, + isPriorityTraceSpan, + isSlowOrErrorSpan, + EXPORTED_TTL_MS, + GLOBAL_CAP, + LIVE_TTL_MS, + PENDING_TTL_MS, + PER_TRACE_CAP, + SETTLED_LINGER_MS, + TOMBSTONE_CAP, +} from './tracing-span-export'; + +type ExportCallback = Parameters[1]; + +class FakeExporter implements SpanExporter { + readonly spans: ReadableSpan[] = []; + + export(spans: ReadableSpan[], resultCallback: ExportCallback): void { + this.spans.push(...spans); + resultCallback({ code: 0 }); + } + + shutdown(): Promise { + return Promise.resolve(); + } + + forceFlush(): Promise { + return Promise.resolve(); + } +} + +/** Find traceIds whose deterministic hash decision matches `sampled` at ratio 0.1 */ +const findTraceIds = (count: number, sampled: boolean): string[] => { + const ids: string[] = []; + for (let i = 1; ids.length < count && i < 1_000_000; i++) { + const id = i.toString(16).padStart(32, '0'); + if (getTraceDecision(id, 0.1) === sampled) ids.push(id); + } + return ids; +}; + +const [SAMPLED_TRACE_ID] = findTraceIds(1, true); +const [UNSAMPLED_TRACE_ID] = findTraceIds(1, false); + +let spanSeq = 0; + +interface FakeSpanOptions { + traceId?: string; + name?: string; + kind?: SpanKind; + durationMs?: number; + statusCode?: SpanStatusCode; + attributes?: Record; + /** 'local' = in-process parent, 'remote' = extracted parent (nested SERVER), 'none' = true root */ + parent?: 'local' | 'remote' | 'none'; +} + +const makeSpan = (options: FakeSpanOptions = {}): ReadableSpan => { + const { + traceId = UNSAMPLED_TRACE_ID, + name = `span-${spanSeq}`, + kind = SpanKind.INTERNAL, + durationMs = 5, + statusCode = SpanStatusCode.UNSET, + attributes = {}, + parent = 'local', + } = options; + const spanId = (++spanSeq).toString(16).padStart(16, '0'); + const parentSpanContext: SpanContext | undefined = + parent === 'none' + ? undefined + : { + traceId, + spanId: 'f'.repeat(16), + traceFlags: TraceFlags.SAMPLED, + isRemote: parent === 'remote', + }; + return { + name, + kind, + spanContext: () => ({ traceId, spanId, traceFlags: TraceFlags.SAMPLED }), + parentSpanContext, + startTime: [0, 0], + endTime: [0, 0], + status: { code: statusCode }, + attributes, + links: [], + events: [], + duration: [Math.floor(durationMs / 1000), Math.round((durationMs % 1000) * 1_000_000)], + ended: true, + resource: { attributes: {}, asyncAttributesPending: false }, + instrumentationScope: { name: 'test' }, + droppedAttributesCount: 0, + droppedEventsCount: 0, + droppedLinksCount: 0, + } as unknown as ReadableSpan; +}; + +const DEFAULT_OPTIONS = { + exportRatio: 0.1, + latencyThresholdMs: 1500, + maxQueueSize: 20_000, + maxExportBatchSize: 512, + scheduledDelayMillis: 5000, + priorityScheduledDelayMillis: 1000, + exportTimeoutMillis: 30_000, +}; + +const createProcessor = (overrides: Partial = {}) => { + const batchExporter = new FakeExporter(); + const priorityExporter = new FakeExporter(); + const processor = createSmartSpanProcessor(batchExporter, priorityExporter, { + ...DEFAULT_OPTIONS, + ...overrides, + }); + return { processor, batchExporter, priorityExporter }; +}; + +type ProcessorSpan = Parameters[0]; + +const startSpan = (processor: SpanProcessor, span: ReadableSpan): void => + processor.onStart(span as unknown as ProcessorSpan, context.active()); + +/** onStart + onEnd for a span that opens and closes with no other activity in between */ +const runSpan = (processor: SpanProcessor, span: ReadableSpan): void => { + startSpan(processor, span); + processor.onEnd(span); +}; + +const names = (exporter: FakeExporter): string[] => exporter.spans.map((s) => s.name); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('hashTraceId / getTraceDecision', () => { + it('is deterministic and bounded to [0, 10000)', () => { + const id = 'abcdef01234567890abcdef012345678'; + expect(hashTraceId(id)).toBe(hashTraceId(id)); + expect(hashTraceId(id)).toBeGreaterThanOrEqual(0); + expect(hashTraceId(id)).toBeLessThan(10000); + }); + + it('respects ratio boundaries', () => { + const id = 'abcdef01234567890abcdef012345678'; + expect(getTraceDecision(id, 0)).toBe(false); + expect(getTraceDecision(id, 1)).toBe(true); + }); +}); + +describe('span predicates', () => { + it('detects slow, error and 5xx spans', () => { + expect(isSlowOrErrorSpan(makeSpan({ durationMs: 2000 }), 1500)).toBe(true); + expect(isSlowOrErrorSpan(makeSpan({ statusCode: SpanStatusCode.ERROR }), 1500)).toBe(true); + expect( + isSlowOrErrorSpan(makeSpan({ attributes: { [ATTR_HTTP_RESPONSE_STATUS_CODE]: 502 } }), 1500) + ).toBe(true); + expect(isSlowOrErrorSpan(makeSpan({ durationMs: 100 }), 1500)).toBe(false); + }); + + it('keeps the prisma spine and drops the rest', () => { + expect(isDroppedPrismaSpan(makeSpan({ name: 'prisma:client:serialize' }))).toBe(true); + expect(isDroppedPrismaSpan(makeSpan({ name: 'prisma:engine:db_query' }))).toBe(false); + expect(isDroppedPrismaSpan(makeSpan({ name: 'pg.query' }))).toBe(false); + }); + + it('recognizes priority spans', () => { + expect(isPriorityTraceSpan(makeSpan({ kind: SpanKind.SERVER }))).toBe(true); + expect(isPriorityTraceSpan(makeSpan({ attributes: { 'http.route': '/api/x' } }))).toBe(true); + expect( + isPriorityTraceSpan(makeSpan({ attributes: { 'nest.controller': 'A', 'nest.handler': 'b' } })) + ).toBe(true); + expect(isPriorityTraceSpan(makeSpan({}))).toBe(false); + }); +}); + +describe('createSmartSpanProcessor', () => { + it('exports every span of a sampled trace', async () => { + const { processor, batchExporter, priorityExporter } = createProcessor(); + const root = makeSpan({ + traceId: SAMPLED_TRACE_ID, + name: 'root', + kind: SpanKind.SERVER, + parent: 'none', + }); + startSpan(processor, root); + runSpan(processor, makeSpan({ traceId: SAMPLED_TRACE_ID, name: 'db-call' })); + runSpan( + processor, + makeSpan({ traceId: SAMPLED_TRACE_ID, name: 'route', attributes: { 'http.route': '/x' } }) + ); + processor.onEnd(root); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['db-call']); + expect(names(priorityExporter)).toEqual(['route', 'root']); + }); + + it('unsampled fast trace: only priority spans exported, buffer lingers after settle', async () => { + const { processor, batchExporter, priorityExporter } = createProcessor(); + const root = makeSpan({ name: 'root', kind: SpanKind.SERVER, parent: 'none' }); + startSpan(processor, root); + runSpan(processor, makeSpan({ name: 'child-a' })); + runSpan(processor, makeSpan({ name: 'child-b' })); + processor.onEnd(root); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual([]); + expect(names(priorityExporter)).toEqual(['root']); + + // within the settle linger, a late slow span still promotes the full trace + runSpan(processor, makeSpan({ name: 'late-slow', durationMs: 2000 })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['child-a', 'child-b', 'late-slow']); + }); + + it('discards a settled remote-parent trace after the linger, not the 5-minute TTL', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + const start = Date.now(); + const { processor, batchExporter, priorityExporter } = createProcessor(); + const entry = makeSpan({ name: 'entry-server', kind: SpanKind.SERVER, parent: 'remote' }); + startSpan(processor, entry); + runSpan(processor, makeSpan({ name: 'child' })); + processor.onEnd(entry); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual([]); + expect(names(priorityExporter)).toEqual(['entry-server']); + + vi.setSystemTime(start + SETTLED_LINGER_MS + 60_000); + runSpan(processor, makeSpan({ traceId: findTraceIds(2, false)[1], name: 'other' })); + + // the linger expired: promotion has nothing left to flush + runSpan(processor, makeSpan({ name: 'late-slow', durationMs: 2000 })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['late-slow']); + }); + + it('async tail work re-opens the trace and is exported on promotion', async () => { + const { processor, batchExporter } = createProcessor(); + runSpan(processor, makeSpan({ name: 'root', kind: SpanKind.SERVER, parent: 'none' })); + // fire-and-forget work after the response: buffered through the linger + runSpan(processor, makeSpan({ name: 'tail' })); + runSpan(processor, makeSpan({ name: 'slow', durationMs: 2000 })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['tail', 'slow']); + }); + + it('promotes the whole trace when a slow child ends mid-trace', async () => { + const { processor, batchExporter, priorityExporter } = createProcessor(); + const root = makeSpan({ name: 'root', kind: SpanKind.SERVER, parent: 'none' }); + startSpan(processor, root); + runSpan(processor, makeSpan({ name: 'early-1' })); + runSpan(processor, makeSpan({ name: 'early-2' })); + runSpan(processor, makeSpan({ name: 'slow-child', durationMs: 6000 })); + // spans ending after promotion follow the exported decision directly + runSpan(processor, makeSpan({ name: 'after-promotion' })); + processor.onEnd(root); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['early-1', 'early-2', 'slow-child', 'after-promotion']); + expect(names(priorityExporter)).toEqual(['root']); + }); + + it('flushes the buffer when only the root span is slow', async () => { + const { processor, batchExporter, priorityExporter } = createProcessor(); + const root = makeSpan({ + name: 'slow-root', + kind: SpanKind.SERVER, + parent: 'none', + durationMs: 10_600, + }); + startSpan(processor, root); + runSpan(processor, makeSpan({ name: 'mid-1' })); + runSpan(processor, makeSpan({ name: 'mid-2' })); + processor.onEnd(root); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['mid-1', 'mid-2']); + expect(names(priorityExporter)).toEqual(['slow-root']); + }); + + it('promotes on error status and on http 5xx', async () => { + for (const attrs of [ + { statusCode: SpanStatusCode.ERROR }, + { attributes: { [ATTR_HTTP_RESPONSE_STATUS_CODE]: 500 } }, + ] as FakeSpanOptions[]) { + const { processor, batchExporter } = createProcessor(); + const trigger = makeSpan({ name: 'trigger', ...attrs }); + startSpan(processor, trigger); + runSpan(processor, makeSpan({ name: 'buffered' })); + processor.onEnd(trigger); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['buffered', 'trigger']); + } + }); + + it('does not re-export priority spans on promotion', async () => { + const { processor, batchExporter, priorityExporter } = createProcessor(); + const slowChild = makeSpan({ name: 'slow-child', durationMs: 3000 }); + startSpan(processor, slowChild); + runSpan(processor, makeSpan({ name: 'buffered' })); + runSpan(processor, makeSpan({ name: 'handler', attributes: { 'http.route': '/api/chat' } })); + processor.onEnd(slowChild); + await processor.forceFlush(); + expect(names(priorityExporter)).toEqual(['handler']); + expect(names(batchExporter)).toEqual(['buffered', 'slow-child']); + }); + + it('does not finalize the buffer when a nested remote-parent SERVER span ends mid-trace', async () => { + const { processor, batchExporter, priorityExporter } = createProcessor(); + const outerRoot = makeSpan({ + name: 'outer-root', + kind: SpanKind.SERVER, + parent: 'none', + durationMs: 10_600, + }); + startSpan(processor, outerRoot); + runSpan(processor, makeSpan({ name: 'mid-1' })); + // inner SERVER span of a proxied request: has a remote parent, is NOT the last live span + runSpan(processor, makeSpan({ name: 'inner-server', kind: SpanKind.SERVER, parent: 'remote' })); + runSpan(processor, makeSpan({ name: 'mid-2' })); + processor.onEnd(outerRoot); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['mid-1', 'mid-2']); + expect(names(priorityExporter)).toEqual(['inner-server', 'outer-root']); + }); + + it('trims non-spine prisma spans unless they are slow', async () => { + const { processor, batchExporter } = createProcessor(); + // sampled trace: non-spine prisma dropped, spine kept + runSpan(processor, makeSpan({ traceId: SAMPLED_TRACE_ID, name: 'prisma:client:serialize' })); + runSpan(processor, makeSpan({ traceId: SAMPLED_TRACE_ID, name: 'prisma:engine:db_query' })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['prisma:engine:db_query']); + + // unsampled trace: non-spine prisma is never buffered, so promotion does not flush it + const { processor: p2, batchExporter: b2 } = createProcessor(); + const slow = makeSpan({ name: 'slow', durationMs: 2000 }); + startSpan(p2, slow); + runSpan(p2, makeSpan({ name: 'prisma:client:serialize' })); + runSpan(p2, makeSpan({ name: 'kept' })); + p2.onEnd(slow); + await p2.forceFlush(); + expect(names(b2)).toEqual(['kept', 'slow']); + + // slow non-spine prisma span is still exported (slow/error takes precedence) + const { processor: p3, batchExporter: b3 } = createProcessor(); + runSpan(p3, makeSpan({ name: 'prisma:client:serialize', durationMs: 2000 })); + await p3.forceFlush(); + expect(names(b3)).toEqual(['prisma:client:serialize']); + }); + + it('exportRatio >= 1.0 exports everything immediately without lifecycle tracking', async () => { + const { processor, batchExporter, priorityExporter } = createProcessor({ exportRatio: 1.0 }); + // no onStart on purpose: the ratio >= 1 path needs no live-span bookkeeping + processor.onEnd(makeSpan({ name: 'child' })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['child']); + processor.onEnd(makeSpan({ name: 'root', kind: SpanKind.SERVER, parent: 'none' })); + await processor.forceFlush(); + expect(names(priorityExporter)).toEqual(['root']); + }); + + it('caps the per-trace buffer, dropping the oldest span', async () => { + const { processor, batchExporter } = createProcessor(); + const root = makeSpan({ name: 'root', kind: SpanKind.SERVER, parent: 'none' }); + startSpan(processor, root); + for (let i = 0; i <= PER_TRACE_CAP; i++) { + runSpan(processor, makeSpan({ name: `b${i}` })); + } + runSpan(processor, makeSpan({ name: 'slow', durationMs: 2000 })); + await processor.forceFlush(); + const exportedNames = names(batchExporter); + expect(exportedNames).toHaveLength(PER_TRACE_CAP + 1); + expect(exportedNames[0]).toBe('b1'); + expect(exportedNames).not.toContain('b0'); + expect(exportedNames[exportedNames.length - 1]).toBe('slow'); + }); + + it('evicts the oldest trace entirely when the global cap is exceeded', async () => { + const traceIds = findTraceIds(21, false); + const { processor, batchExporter } = createProcessor(); + // one never-ended span per trace keeps every trace in flight while buffering + startSpan(processor, makeSpan({ traceId: traceIds[0], name: 'hold-0' })); + for (let i = 0; i < 100; i++) { + runSpan(processor, makeSpan({ traceId: traceIds[0], name: `a${i}` })); + } + const perTrace = GLOBAL_CAP / 20; + for (let t = 1; t <= 20; t++) { + startSpan(processor, makeSpan({ traceId: traceIds[t], name: `hold-${t}` })); + for (let i = 0; i < perTrace; i++) { + runSpan(processor, makeSpan({ traceId: traceIds[t], name: `t${t}-${i}` })); + } + } + // trace 0 (oldest) was evicted: promoting it flushes nothing + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'slow-a', durationMs: 2000 })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['slow-a']); + + // trace 1 survived intact + runSpan(processor, makeSpan({ traceId: traceIds[1], name: 'slow-t1', durationMs: 2000 })); + await processor.forceFlush(); + expect(batchExporter.spans).toHaveLength(1 + perTrace + 1); + expect(names(batchExporter)).toContain('t1-0'); + expect(names(batchExporter)[batchExporter.spans.length - 1]).toBe('slow-t1'); + }); + + it('cap eviction spends settled linger buffers before in-flight ones', async () => { + const traceIds = findTraceIds(22, false); + const { processor, batchExporter } = createProcessor(); + // oldest holder: in-flight trace A + startSpan(processor, makeSpan({ traceId: traceIds[0], name: 'hold-a' })); + for (let i = 0; i < 100; i++) { + runSpan(processor, makeSpan({ traceId: traceIds[0], name: `a${i}` })); + } + // trace S: buffered, then settled into its linger window + const holdS = makeSpan({ traceId: traceIds[1], name: 'hold-s' }); + startSpan(processor, holdS); + for (let i = 0; i < 100; i++) { + runSpan(processor, makeSpan({ traceId: traceIds[1], name: `s${i}` })); + } + processor.onEnd(holdS); + // fill to overflow with in-flight traces + const perTrace = (GLOBAL_CAP - 100) / 20; + for (let t = 2; t <= 21; t++) { + startSpan(processor, makeSpan({ traceId: traceIds[t], name: `hold-${t}` })); + for (let i = 0; i < perTrace; i++) { + runSpan(processor, makeSpan({ traceId: traceIds[t], name: `t${t}-${i}` })); + } + } + // S (settled) was evicted even though A is the older holder + runSpan(processor, makeSpan({ traceId: traceIds[1], name: 'slow-s', durationMs: 2000 })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['slow-s']); + + // A (in-flight) kept its buffer + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'slow-a', durationMs: 2000 })); + await processor.forceFlush(); + expect(names(batchExporter)).toContain('a0'); + expect(names(batchExporter)).toContain('a99'); + }); + + it('keeps the buffer of a live trace past the pending TTL', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + const start = Date.now(); + const traceIds = findTraceIds(2, false); + const { processor, batchExporter } = createProcessor(); + startSpan(processor, makeSpan({ traceId: traceIds[0], name: 'hold' })); + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'buffered' })); + + vi.setSystemTime(start + PENDING_TTL_MS + 60_000); + // any span end past the cleanup gate triggers the sweep + runSpan(processor, makeSpan({ traceId: traceIds[1], name: 'other' })); + + // the open 'hold' span keeps the trace live, so promotion still flushes + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'slow', durationMs: 2000 })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['buffered', 'slow']); + }); + + it('evicts leaked traces and their buffers after the live TTL', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + const start = Date.now(); + const traceIds = findTraceIds(2, false); + const { processor, batchExporter } = createProcessor(); + startSpan(processor, makeSpan({ traceId: traceIds[0], name: 'leaked' })); + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'buffered' })); + + vi.setSystemTime(start + LIVE_TTL_MS + 60_000); + runSpan(processor, makeSpan({ traceId: traceIds[1], name: 'other' })); + + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'slow', durationMs: 2000 })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['slow']); + }); + + it('expires untracked pending buffers after the pending TTL', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + const start = Date.now(); + const traceIds = findTraceIds(2, false); + const { processor, batchExporter } = createProcessor(); + // onEnd without onStart: the span predates processor attach, no live entry + processor.onEnd(makeSpan({ traceId: traceIds[0], name: 'orphan' })); + + vi.setSystemTime(start + PENDING_TTL_MS + 60_000); + runSpan(processor, makeSpan({ traceId: traceIds[1], name: 'other' })); + + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'slow', durationMs: 2000 })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['slow']); + }); + + it('expires exported marks after the exported TTL', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + const start = Date.now(); + const traceIds = findTraceIds(2, false); + const { processor, batchExporter } = createProcessor(); + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'slow', durationMs: 2000 })); + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'follow-1' })); + + vi.setSystemTime(start + EXPORTED_TTL_MS + 60_000); + runSpan(processor, makeSpan({ traceId: traceIds[1], name: 'other' })); + + // the exported mark expired: this span is buffered, then discarded when the trace settles + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'follow-2' })); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['slow', 'follow-1']); + }); + + it('caps promoted tombstones, evicting the oldest first', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + const start = Date.now(); + const traceIds = findTraceIds(TOMBSTONE_CAP + 2, false); + const { processor, batchExporter } = createProcessor(); + for (let i = 0; i <= TOMBSTONE_CAP; i++) { + runSpan(processor, makeSpan({ traceId: traceIds[i], name: `slow-${i}`, durationMs: 2000 })); + } + + vi.setSystemTime(start + 60_000); + runSpan(processor, makeSpan({ traceId: traceIds[TOMBSTONE_CAP + 1], name: 'other' })); + + // oldest tombstone evicted: its late span no longer follows the promotion + runSpan(processor, makeSpan({ traceId: traceIds[0], name: 'follow-oldest' })); + // a recent tombstone survives: its late span still exports + runSpan(processor, makeSpan({ traceId: traceIds[1], name: 'follow-recent' })); + await processor.forceFlush(); + expect(names(batchExporter)).not.toContain('follow-oldest'); + expect(names(batchExporter)).toContain('follow-recent'); + }); + + it('drains the pending buffer on shutdown', async () => { + const { processor, batchExporter } = createProcessor(); + const root = makeSpan({ name: 'root', kind: SpanKind.SERVER, parent: 'none' }); + startSpan(processor, root); + runSpan(processor, makeSpan({ name: 'in-flight' })); + await processor.shutdown(); + expect(names(batchExporter)).toEqual(['in-flight']); + }); +}); + +describe('integration with real SDK spans', () => { + it('promotes a slow trace in full and discards a fast one', async () => { + const { processor, batchExporter, priorityExporter } = createProcessor({ exportRatio: 0 }); + const provider = new BasicTracerProvider({ spanProcessors: [processor] }); + const tracer = provider.getTracer('test'); + const t0 = 1_700_000_000_000; + + // slow request: a fast child, a slow child, then the slow SERVER root + const root = tracer.startSpan('slow-root', { + kind: SpanKind.SERVER, + startTime: t0, + root: true, + }); + const ctx = trace.setSpan(context.active(), root); + const fastChild = tracer.startSpan('fast-child', { startTime: t0 }, ctx); + fastChild.end(t0 + 10); + const slowChild = tracer.startSpan('slow-child', { startTime: t0 }, ctx); + slowChild.end(t0 + 6000); + root.end(t0 + 10_600); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['fast-child', 'slow-child']); + expect(names(priorityExporter)).toEqual(['slow-root']); + + // fast request: child is buffered, then discarded when the last span ends + const root2 = tracer.startSpan('fast-root', { + kind: SpanKind.SERVER, + startTime: t0, + root: true, + }); + const ctx2 = trace.setSpan(context.active(), root2); + const quickChild = tracer.startSpan('quick-child', { startTime: t0 }, ctx2); + quickChild.end(t0 + 10); + root2.end(t0 + 100); + await processor.forceFlush(); + expect(names(batchExporter)).toEqual(['fast-child', 'slow-child']); + expect(names(priorityExporter)).toEqual(['slow-root', 'fast-root']); + + await provider.shutdown(); + }); +}); diff --git a/apps/nestjs-backend/src/tracing-span-export.ts b/apps/nestjs-backend/src/tracing-span-export.ts new file mode 100644 index 0000000000..31a4c008b6 --- /dev/null +++ b/apps/nestjs-backend/src/tracing-span-export.ts @@ -0,0 +1,403 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/** + * Smart span export with trace-coherent tail decision. + * + * Sampling stays at 100%; this module decides at export time which spans + * reach the backend: + * - slow / error / 5xx spans always export and promote their whole trace: + * buffered spans are flushed, later spans export directly + * - traces picked by the traceId hash (OTEL_EXPORT_RATIO) export in full + * - other traces export only priority spans (SERVER/route/handler, keeps APM + * stats accurate); the rest are buffered and discarded once the trace's + * live-span refcount drops to zero without a promotion + * + * Refcounting (onStart/onEnd) instead of watching for a parentless root span + * handles remote-parent entry spans and post-response async work uniformly. + * Per-trace bookkeeping lives in TraceStore below, which documents the record + * shapes and memory bounds. Priority spans batch on a short delay; shutdown + * drains undecided buffers (they belong to interrupted requests). + */ +import { SpanKind, SpanStatusCode } from '@opentelemetry/api'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import type { ReadableSpan, SpanExporter, SpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { ATTR_HTTP_RESPONSE_STATUS_CODE } from '@opentelemetry/semantic-conventions'; + +export const PER_TRACE_CAP = 512; +export const GLOBAL_CAP = 10_000; +export const PENDING_TTL_MS = 5 * 60 * 1000; +export const LIVE_TTL_MS = 30 * 60 * 1000; +export const EXPORTED_TTL_MS = 10 * 60 * 1000; +export const SETTLED_LINGER_MS = 10_000; +export const TOMBSTONE_CAP = 10_000; +const CLEANUP_INTERVAL_MS = 30_000; + +export const hashTraceId = (traceId: string): number => { + // FNV-1a hash for better distribution + let hash = 2166136261; + for (let i = 0; i < traceId.length; i++) { + hash ^= traceId.charCodeAt(i); + hash = (hash * 16777619) >>> 0; + } + return hash % 10000; +}; + +export const getTraceDecision = (traceId: string, exportRatio: number): boolean => + hashTraceId(traceId) < exportRatio * 10000; + +// Prisma emits ~11 spans per query. Keep only the spine +// (operation -> engine:query -> db_query) so the surviving db_query is never +// orphaned (renders as a "Missing Span"); drop the rest. +const PRISMA_SPINE_SPANS = new Set([ + 'prisma:client:operation', + 'prisma:engine:query', + 'prisma:engine:db_query', +]); + +export const isDroppedPrismaSpan = (span: ReadableSpan): boolean => + span.name.startsWith('prisma:') && !PRISMA_SPINE_SPANS.has(span.name); + +export const isPriorityTraceSpan = (span: ReadableSpan): boolean => { + const attributes = span.attributes; + return ( + span.kind === SpanKind.SERVER || + typeof attributes['teable.route.full'] === 'string' || + typeof attributes['http.route'] === 'string' || + (typeof attributes['nest.controller'] === 'string' && + typeof attributes['nest.handler'] === 'string') + ); +}; + +export const isSlowOrErrorSpan = (span: ReadableSpan, latencyThresholdMs: number): boolean => { + if (span.status.code === SpanStatusCode.ERROR) return true; + + const httpStatusCode = span.attributes[ATTR_HTTP_RESPONSE_STATUS_CODE]; + if (typeof httpStatusCode === 'number' && httpStatusCode >= 500) return true; + + const durationMs = span.duration[0] * 1000 + span.duration[1] / 1_000_000; + return durationMs > latencyThresholdMs; +}; + +export interface ISmartSpanProcessorOptions { + exportRatio: number; + latencyThresholdMs: number; + maxQueueSize: number; + maxExportBatchSize: number; + scheduledDelayMillis: number; + /** Short delay for priority spans; APM stats lag by at most this much. */ + priorityScheduledDelayMillis: number; + exportTimeoutMillis: number; +} + +interface ITraceState { + liveSpans: number; + spans: ReadableSpan[]; + promotedUntilMs: number; + settledAtMs: number; + lastTouchedMs: number; +} + +/** + * Per-trace bookkeeping. The `traces` Map is the single source of truth; a + * record is in exactly one shape, derived from its fields: + * - active (liveSpans > 0): buffer held for possible promotion; reclaimed as + * leaked only after LIVE_TTL_MS without span activity + * - settled (refcount reached zero unpromoted): buffer retained for + * SETTLED_LINGER_MS so fire-and-forget work that resumes the trace can + * still promote it complete + * - orphan (buffered spans that predate onStart tracking): reclaimed after + * PENDING_TTL_MS + * - tombstone (promoted and settled): kept until promotedUntilMs so late + * spans keep exporting; capped at TOMBSTONE_CAP + * + * Buffers are capped per trace (PER_TRACE_CAP) and globally (GLOBAL_CAP). + * `bufferHolders`/`settledHolders` are insertion-ordered views used only to + * pick eviction order in O(1); `drainBuffer` is their single removal point, + * so a stale index entry can at worst delay one eviction. + */ +class TraceStore { + private readonly traces = new Map(); + private readonly bufferHolders = new Set(); + private readonly settledHolders = new Set(); + private bufferedSpanCount = 0; + + getOrCreate(traceId: string, nowMs: number): ITraceState { + let trace = this.traces.get(traceId); + if (!trace) { + trace = { liveSpans: 0, spans: [], promotedUntilMs: 0, settledAtMs: 0, lastTouchedMs: nowMs }; + this.traces.set(traceId, trace); + } + return trace; + } + + /** Empties a trace's buffer and returns the spans. The only index-removal point. */ + drainBuffer(traceId: string, trace: ITraceState): ReadableSpan[] { + const spans = trace.spans; + this.bufferedSpanCount -= spans.length; + trace.spans = []; + this.bufferHolders.delete(traceId); + this.settledHolders.delete(traceId); + return spans; + } + + removeIfInert(traceId: string, trace: ITraceState, nowMs: number): void { + if (trace.liveSpans === 0 && trace.spans.length === 0 && trace.promotedUntilMs <= nowMs) { + this.traces.delete(traceId); + } + } + + recordStart(traceId: string, nowMs: number): void { + const trace = this.getOrCreate(traceId, nowMs); + trace.liveSpans++; + if (trace.settledAtMs !== 0) { + trace.settledAtMs = 0; + this.settledHolders.delete(traceId); + } + trace.lastTouchedMs = nowMs; + } + + settle(traceId: string, trace: ITraceState, nowMs: number): void { + // liveSpans 0 here means this span was never tracked by onStart + // (it predates processor attach); the orphan TTL owns its buffer. + if (trace.liveSpans === 0) return; + trace.liveSpans--; + if (trace.liveSpans > 0) { + trace.lastTouchedMs = nowMs; + return; + } + // Completed without promotion: keep the buffer through a short linger so + // fire-and-forget work that resumes the trace can still promote it whole. + trace.settledAtMs = nowMs; + if (trace.spans.length > 0) this.settledHolders.add(traceId); + } + + buffer(span: ReadableSpan, traceId: string, trace: ITraceState, nowMs: number): void { + if (trace.spans.length >= PER_TRACE_CAP) { + trace.spans.shift(); + this.bufferedSpanCount--; + } + trace.spans.push(span); + trace.lastTouchedMs = nowMs; + this.bufferedSpanCount++; + this.bufferHolders.add(traceId); + if (trace.settledAtMs > 0) this.settledHolders.add(traceId); + if (this.bufferedSpanCount > GLOBAL_CAP) this.evictOverCap(nowMs); + } + + private evictOverCap(nowMs: number): void { + // Settled linger buffers are already scheduled for discard: spend them + // first so cap pressure never evicts an in-flight trace's buffer early. + for (const traceId of this.settledHolders) { + if (this.bufferedSpanCount <= GLOBAL_CAP) break; + const trace = this.traces.get(traceId); + // A resumed trace stays indexed until its buffer drops; skip it here. + if (!trace || trace.settledAtMs === 0) { + this.settledHolders.delete(traceId); + continue; + } + this.drainBuffer(traceId, trace); + this.removeIfInert(traceId, trace, nowMs); + } + // Still over: evict whole oldest buffers (insertion order). + for (const traceId of this.bufferHolders) { + if (this.bufferedSpanCount <= GLOBAL_CAP) break; + const trace = this.traces.get(traceId); + if (!trace) { + this.bufferHolders.delete(traceId); + continue; + } + this.drainBuffer(traceId, trace); + this.removeIfInert(traceId, trace, nowMs); + } + } + + sweep(nowMs: number): void { + let tombstones = 0; + for (const [traceId, trace] of this.traces) { + if (this.reapIfExpired(traceId, trace, nowMs)) tombstones++; + } + if (tombstones > TOMBSTONE_CAP) this.evictTombstones(tombstones - TOMBSTONE_CAP, nowMs); + } + + /** Applies the shape's TTL to one record; returns true for a live tombstone. */ + private reapIfExpired(traceId: string, trace: ITraceState, nowMs: number): boolean { + if (trace.liveSpans > 0) { + // Presumed leaked (a span started but never ended) after LIVE_TTL_MS. + if (nowMs - trace.lastTouchedMs > LIVE_TTL_MS) { + this.drainBuffer(traceId, trace); + this.traces.delete(traceId); + } + return false; + } + if (trace.spans.length > 0) { + // Settled buffers linger briefly for async resumption; orphan buffers + // (spans that predate onStart tracking) wait out the pending TTL. + const sinceMs = trace.settledAtMs > 0 ? trace.settledAtMs : trace.lastTouchedMs; + const ttlMs = trace.settledAtMs > 0 ? SETTLED_LINGER_MS : PENDING_TTL_MS; + if (nowMs - sinceMs > ttlMs) { + this.drainBuffer(traceId, trace); + this.traces.delete(traceId); + } + return false; + } + if (trace.promotedUntilMs <= nowMs) { + this.traces.delete(traceId); + return false; + } + return true; + } + + // Evict oldest surplus tombstones; their late spans just fall back to + // the hash decision instead of following the promotion. + private evictTombstones(excess: number, nowMs: number): void { + for (const [traceId, trace] of this.traces) { + if (excess === 0) break; + if (trace.liveSpans === 0 && trace.spans.length === 0 && trace.promotedUntilMs > nowMs) { + this.traces.delete(traceId); + excess--; + } + } + } + + /** Empties every buffer and resets the store; returns the drained spans. */ + drainAll(): ReadableSpan[] { + const drained: ReadableSpan[] = []; + for (const trace of this.traces.values()) { + drained.push(...trace.spans); + } + this.traces.clear(); + this.bufferHolders.clear(); + this.settledHolders.clear(); + this.bufferedSpanCount = 0; + return drained; + } +} + +export const createSmartSpanProcessor = ( + batchExporter: SpanExporter, + priorityExporter: SpanExporter, + options: ISmartSpanProcessorOptions +): SpanProcessor => { + const { exportRatio, latencyThresholdMs } = options; + const batchProcessor = new BatchSpanProcessor(batchExporter, { + maxQueueSize: options.maxQueueSize, + maxExportBatchSize: options.maxExportBatchSize, + scheduledDelayMillis: options.scheduledDelayMillis, + exportTimeoutMillis: options.exportTimeoutMillis, + }); + const priorityProcessor = new BatchSpanProcessor(priorityExporter, { + // Priority spans are the APM baseline; 4x queue headroom means a slow or + // restarting collector drops detail spans well before request-rate data. + maxQueueSize: options.maxQueueSize * 4, + maxExportBatchSize: options.maxExportBatchSize, + scheduledDelayMillis: options.priorityScheduledDelayMillis, + exportTimeoutMillis: options.exportTimeoutMillis, + }); + + const route = (span: ReadableSpan): void => { + if (isPriorityTraceSpan(span)) { + priorityProcessor.onEnd(span); + } else { + batchProcessor.onEnd(span); + } + }; + + const forwardStart: SpanProcessor['onStart'] = (span, parentContext) => { + priorityProcessor.onStart(span, parentContext); + batchProcessor.onStart(span, parentContext); + }; + + const flushProcessors = (): Promise => + Promise.all([priorityProcessor.forceFlush(), batchProcessor.forceFlush()]).then( + () => undefined + ); + + const shutdownProcessors = (): Promise => + Promise.all([priorityProcessor.shutdown(), batchProcessor.shutdown()]).then(() => undefined); + + // Full export: every span routes directly, no lifecycle tracking needed. + if (exportRatio >= 1.0) { + return { + onStart: forwardStart, + onEnd: (span: ReadableSpan) => { + if (isDroppedPrismaSpan(span) && !isSlowOrErrorSpan(span, latencyThresholdMs)) return; + route(span); + }, + shutdown: shutdownProcessors, + forceFlush: flushProcessors, + }; + } + + const store = new TraceStore(); + let lastCleanupMs = Date.now(); + + const cleanup = (nowMs: number): void => { + if (nowMs - lastCleanupMs < CLEANUP_INTERVAL_MS) return; + lastCleanupMs = nowMs; + store.sweep(nowMs); + }; + + // Backstop for idle processes: onEnd also triggers cleanup, but with no + // traffic nothing would ever reclaim the store. unref'd so it cannot hold + // the process open. + const cleanupTimer = setInterval(() => cleanup(Date.now()), CLEANUP_INTERVAL_MS); + cleanupTimer.unref?.(); + + const promote = (traceId: string, trace: ITraceState, nowMs: number): void => { + trace.promotedUntilMs = nowMs + EXPORTED_TTL_MS; + for (const buffered of store.drainBuffer(traceId, trace)) { + batchProcessor.onEnd(buffered); + } + }; + + const decide = (span: ReadableSpan, trace: ITraceState, traceId: string, nowMs: number): void => { + if (isSlowOrErrorSpan(span, latencyThresholdMs)) { + promote(traceId, trace, nowMs); + route(span); + return; + } + + if (isDroppedPrismaSpan(span)) return; + + if (trace.promotedUntilMs > nowMs) { + route(span); + return; + } + + // Deterministic per traceId, so picked traces need no stored state. + if (getTraceDecision(traceId, exportRatio)) { + route(span); + return; + } + + // Priority spans are never buffered, so promotion cannot duplicate them. + if (isPriorityTraceSpan(span)) { + priorityProcessor.onEnd(span); + } else { + store.buffer(span, traceId, trace, nowMs); + } + }; + + return { + onStart: (span, parentContext) => { + forwardStart(span, parentContext); + store.recordStart(span.spanContext().traceId, Date.now()); + }, + onEnd: (span: ReadableSpan) => { + const nowMs = Date.now(); + cleanup(nowMs); + const traceId = span.spanContext().traceId; + const trace = store.getOrCreate(traceId, nowMs); + decide(span, trace, traceId, nowMs); + store.settle(traceId, trace, nowMs); + store.removeIfInert(traceId, trace, nowMs); + }, + shutdown: () => { + clearInterval(cleanupTimer); + for (const buffered of store.drainAll()) { + batchProcessor.onEnd(buffered); + } + return shutdownProcessors(); + }, + forceFlush: flushProcessors, + }; +}; diff --git a/apps/nestjs-backend/src/tracing.ts b/apps/nestjs-backend/src/tracing.ts index 01cad54561..c55d234c20 100644 --- a/apps/nestjs-backend/src/tracing.ts +++ b/apps/nestjs-backend/src/tracing.ts @@ -19,6 +19,7 @@ * | OTEL_BSP_MAX_QUEUE_SIZE | Trace BSP max queue size | 2048 | 2048 | * | OTEL_BSP_MAX_EXPORT_BATCH_SIZE | Trace BSP max export batch | 512 | 512 | * | OTEL_BSP_SCHEDULE_DELAY | Trace BSP delay (ms) | 5000 | 5000 | + * | OTEL_BSP_PRIORITY_SCHEDULE_DELAY | Priority-span BSP delay (ms) | 1000 | 1000 | * | OTEL_BSP_EXPORT_TIMEOUT | Trace BSP export timeout (ms) | 30000 | 30000 | * | OTEL_METRIC_EXPORT_INTERVAL_MS | Metrics export interval (ms) | 10000 | 60000 | * | BACKEND_SENTRY_DSN | Sentry DSN for error tracking | (disabled) | (disabled) | @@ -29,10 +30,11 @@ * - In development, traces and logs are enabled by default (localhost endpoint) * - In production, you must explicitly set OTEL_EXPORTER_OTLP_ENDPOINT to enable tracing * - Sampling rate is always 100%; OTEL_EXPORT_RATIO controls how many spans are sent to backend - * - Smart export always sends: errors, HTTP 5xx responses, and slow requests (regardless of ratio) + * - Smart export always sends: errors, HTTP 5xx responses, and slow requests (regardless of ratio), + * and promotes their whole trace so it arrives complete (see tracing-span-export.ts) */ import { Logger } from '@nestjs/common'; -import { metrics, SpanKind, SpanStatusCode } from '@opentelemetry/api'; +import { diag, DiagConsoleLogger, DiagLogLevel, metrics, SpanKind } from '@opentelemetry/api'; import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; @@ -46,20 +48,13 @@ import { PinoInstrumentation } from '@opentelemetry/instrumentation-pino'; import { RuntimeNodeInstrumentation } from '@opentelemetry/instrumentation-runtime-node'; import { resourceFromAttributes } from '@opentelemetry/resources'; import * as opentelemetry from '@opentelemetry/sdk-node'; -import { - BatchSpanProcessor, - NoopSpanProcessor, - SimpleSpanProcessor, -} from '@opentelemetry/sdk-trace-base'; +import { NoopSpanProcessor } from '@opentelemetry/sdk-trace-base'; import type { SpanProcessor } from '@opentelemetry/sdk-trace-base'; -import { - ATTR_HTTP_RESPONSE_STATUS_CODE, - ATTR_SERVICE_NAME, - ATTR_SERVICE_VERSION, -} from '@opentelemetry/semantic-conventions'; +import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'; import { PrismaInstrumentation } from '@prisma/instrumentation'; import { wrapContextManagerClass } from '@sentry/opentelemetry'; import { setTeableDbSpanAttributes, setTeableDbSpanAttributesFromSpan } from './tracing-db-context'; +import { createSmartSpanProcessor } from './tracing-span-export'; // Use webpack's special require that bypasses bundling, falling back to standard require // This is needed because webpack transforms import.meta.url and createRequire in ways @@ -76,6 +71,11 @@ const { AlwaysOnSampler } = opentelemetry.node; const otelLogger = new Logger('OpenTelemetry'); const isDevelopment = process.env.NODE_ENV !== 'production'; +// OTel SDK self-diagnostics (span queue drops, export failures) are no-op by +// default; surface WARN+ to stdout. Not routed through pino, so this reaches +// container logs only, never the OTLP log pipeline. +diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.WARN); + /** * Environment-specific default values * - undefined means the feature is disabled unless explicitly configured @@ -91,6 +91,7 @@ const ENV_DEFAULTS = { OTEL_BSP_MAX_QUEUE_SIZE: '2048', OTEL_BSP_MAX_EXPORT_BATCH_SIZE: '512', OTEL_BSP_SCHEDULE_DELAY: '5000', + OTEL_BSP_PRIORITY_SCHEDULE_DELAY: '1000', OTEL_BSP_EXPORT_TIMEOUT: '30000', OTEL_METRIC_EXPORT_INTERVAL_MS: '10000', }, @@ -104,6 +105,7 @@ const ENV_DEFAULTS = { OTEL_BSP_MAX_QUEUE_SIZE: '2048', OTEL_BSP_MAX_EXPORT_BATCH_SIZE: '512', OTEL_BSP_SCHEDULE_DELAY: '5000', + OTEL_BSP_PRIORITY_SCHEDULE_DELAY: '1000', OTEL_BSP_EXPORT_TIMEOUT: '30000', OTEL_METRIC_EXPORT_INTERVAL_MS: '60000', }, @@ -165,6 +167,11 @@ const traceBatchMaxExportBatchSize = Math.min( parseIntegerConfig('OTEL_BSP_MAX_EXPORT_BATCH_SIZE', 512, 1) ); const traceBatchScheduledDelayMillis = parseIntegerConfig('OTEL_BSP_SCHEDULE_DELAY', 5000, 0); +const tracePriorityScheduledDelayMillis = parseIntegerConfig( + 'OTEL_BSP_PRIORITY_SCHEDULE_DELAY', + 1000, + 0 +); const traceBatchExportTimeoutMillis = parseIntegerConfig('OTEL_BSP_EXPORT_TIMEOUT', 30000, 1); const metricExportIntervalMs = Math.max( 1000, @@ -215,97 +222,6 @@ if (metricsExporter) { }; } -// Smart export: deterministic decision based on traceId hash -// No cache needed - hash function is pure and fast -const hashTraceId = (traceId: string): number => { - // FNV-1a hash for better distribution - let hash = 2166136261; - for (let i = 0; i < traceId.length; i++) { - hash ^= traceId.charCodeAt(i); - hash = (hash * 16777619) >>> 0; - } - return hash % 10000; -}; - -const getTraceDecision = (traceId: string): boolean => hashTraceId(traceId) < exportRatio * 10000; - -// Prisma emits ~11 spans per query (operation, client:middleware/serialize, -// engine:query/connection/db_query/serialize/...). Keep only the spine -// (operation -> engine:query -> db_query) and drop the rest. The full spine is kept -// so the surviving db_query is never orphaned (which renders as a "Missing Span"). -const PRISMA_SPINE_SPANS = new Set([ - 'prisma:client:operation', - 'prisma:engine:query', - 'prisma:engine:db_query', -]); - -const isDroppedPrismaSpan = (span: opentelemetry.tracing.ReadableSpan): boolean => - span.name.startsWith('prisma:') && !PRISMA_SPINE_SPANS.has(span.name); - -const isPriorityTraceSpan = (span: opentelemetry.tracing.ReadableSpan): boolean => { - const attributes = span.attributes; - return ( - span.kind === SpanKind.SERVER || - typeof attributes['teable.route.full'] === 'string' || - typeof attributes['http.route'] === 'string' || - (typeof attributes['nest.controller'] === 'string' && - typeof attributes['nest.handler'] === 'string') - ); -}; - -const shouldExportSpan = (span: opentelemetry.tracing.ReadableSpan): boolean => { - // Always export errors - if (span.status.code === SpanStatusCode.ERROR) return true; - - // Always export HTTP errors (5xx) - const httpStatusCode = span.attributes[ATTR_HTTP_RESPONSE_STATUS_CODE]; - if (typeof httpStatusCode === 'number' && httpStatusCode >= 500) return true; - - // Always export slow requests - const durationMs = span.duration[0] * 1000 + span.duration[1] / 1_000_000; - if (durationMs > latencyThresholdMs) return true; - - if (isDroppedPrismaSpan(span)) return false; - - if (exportRatio >= 1.0) return true; - - // Consistent export decision based on traceId - all spans in same trace have same fate - return getTraceDecision(span.spanContext().traceId); -}; - -const createSmartSpanProcessor = ( - batchExporter: OTLPTraceExporter, - priorityExporter: OTLPTraceExporter -): SpanProcessor => { - const batchProcessor = new BatchSpanProcessor(batchExporter, { - maxQueueSize: traceBatchMaxQueueSize, - maxExportBatchSize: traceBatchMaxExportBatchSize, - scheduledDelayMillis: traceBatchScheduledDelayMillis, - exportTimeoutMillis: traceBatchExportTimeoutMillis, - }); - const priorityProcessor = new SimpleSpanProcessor(priorityExporter); - - return { - onStart: (span, parentContext) => { - priorityProcessor.onStart(span, parentContext); - batchProcessor.onStart(span, parentContext); - }, - onEnd: (span: opentelemetry.tracing.ReadableSpan) => { - if (isPriorityTraceSpan(span)) { - priorityProcessor.onEnd(span); - return; - } - if (shouldExportSpan(span)) batchProcessor.onEnd(span); - }, - shutdown: () => - Promise.all([priorityProcessor.shutdown(), batchProcessor.shutdown()]).then(() => undefined), - forceFlush: () => - Promise.all([priorityProcessor.forceFlush(), batchProcessor.forceFlush()]).then( - () => undefined - ), - }; -}; - // Track in-flight outbound HTTP requests by target host via SpanProcessor, // since instrumentation-http only records duration after completion. const httpClientActiveRequests = metrics @@ -363,7 +279,16 @@ const spanProcessors = [ ? [ createSmartSpanProcessor( traceExporter, - new OTLPTraceExporter(createExporterOptions(traceEndpoint)) + new OTLPTraceExporter(createExporterOptions(traceEndpoint)), + { + exportRatio, + latencyThresholdMs, + maxQueueSize: traceBatchMaxQueueSize, + maxExportBatchSize: traceBatchMaxExportBatchSize, + scheduledDelayMillis: traceBatchScheduledDelayMillis, + priorityScheduledDelayMillis: tracePriorityScheduledDelayMillis, + exportTimeoutMillis: traceBatchExportTimeoutMillis, + } ), ] : [new NoopSpanProcessor()]), diff --git a/apps/nestjs-backend/src/types/i18n.generated.ts b/apps/nestjs-backend/src/types/i18n.generated.ts index 14b65588d8..848b4a2da7 100644 --- a/apps/nestjs-backend/src/types/i18n.generated.ts +++ b/apps/nestjs-backend/src/types/i18n.generated.ts @@ -5543,6 +5543,15 @@ export type I18nTranslations = { "tableId": string; "automationId": string; "appId": string; + "searchVector": string; + "searchVectorFields": string; + "searchVectorStatus": { + "ready": string; + "configured": string; + "rebuildPending": string; + "stale": string; + "unknown": string; + }; }; }; "pluginPanel": { diff --git a/apps/nestjs-backend/src/utils/postgres-regex-escape.ts b/apps/nestjs-backend/src/utils/postgres-regex-escape.ts index 1fabef7bdd..378134df36 100644 --- a/apps/nestjs-backend/src/utils/postgres-regex-escape.ts +++ b/apps/nestjs-backend/src/utils/postgres-regex-escape.ts @@ -46,3 +46,26 @@ export function escapeJsonbRegex(input: string): string { return '\\\\' + match; }); } + +/** + * Escape a value so it is safe inside a JSONB path *string literal* (the part + * between the double quotes in `$[*] ? (@ == "...")`). + * + * This is the single-level escaping used when the whole jsonpath is passed to + * Postgres as a *bound parameter* (the driver handles the surrounding SQL + * string literal). Only backslash and the closing double quote need escaping; + * everything else — including single quotes — is inert because it never touches + * the SQL string. + */ +export function escapeJsonPathStringLiteral(input: string): string { + return String(input).replace(/\\/g, '\\\\').replace(/"/g, '\\"'); +} + +/** + * Escape a value for use as a `like_regex` pattern inside a JSONB path string + * literal, for the bound-parameter case: first neutralize regex metacharacters + * (so the value matches literally), then escape for the jsonpath string. + */ +export function escapeJsonPathRegexLiteral(input: string): string { + return escapeJsonPathStringLiteral(escapePostgresRegex(String(input))); +} diff --git a/apps/nestjs-backend/test/base-node.e2e-spec.ts b/apps/nestjs-backend/test/base-node.e2e-spec.ts index 1d9f255bd9..5365c52d75 100644 --- a/apps/nestjs-backend/test/base-node.e2e-spec.ts +++ b/apps/nestjs-backend/test/base-node.e2e-spec.ts @@ -271,7 +271,7 @@ describe('BaseNodeController (e2e) /api/base/:baseId/node', () => { expect(response.status).toBe(201); expect(response.headers['x-teable-v2']).toBe('true'); expect(response.headers['x-teable-v2-feature']).toBe('createTable'); - expect(response.headers['x-teable-v2-reason']).toBe('new_base'); + expect(['new_base', 'env_force_v2_all']).toContain(response.headers['x-teable-v2-reason']); nodesToCleanup.push(response.data.id); }); @@ -470,7 +470,7 @@ describe('BaseNodeController (e2e) /api/base/:baseId/node', () => { expect(response.status).toBe(201); expect(response.headers['x-teable-v2']).toBe('true'); expect(response.headers['x-teable-v2-feature']).toBe('createTable'); - expect(response.headers['x-teable-v2-reason']).toBe('new_base'); + expect(['new_base', 'env_force_v2_all']).toContain(response.headers['x-teable-v2-reason']); nodesToCleanup.push(response.data.id); @@ -759,7 +759,7 @@ describe('BaseNodeController (e2e) /api/base/:baseId/node', () => { expect(response.status).toBe(200); expect(response.headers['x-teable-v2']).toBe('true'); expect(response.headers['x-teable-v2-feature']).toBe('deleteTable'); - expect(response.headers['x-teable-v2-reason']).toBe('new_base'); + expect(['new_base', 'env_force_v2_all']).toContain(response.headers['x-teable-v2-reason']); const error = await getError(() => getBaseNode(baseId, table.data.id)); expect(error?.status).toBeGreaterThanOrEqual(400); @@ -1118,7 +1118,7 @@ describe('BaseNodeController (e2e) /api/base/:baseId/node', () => { expect(response.status).toBe(201); expect(response.headers['x-teable-v2']).toBe('true'); expect(response.headers['x-teable-v2-feature']).toBe('duplicateTable'); - expect(response.headers['x-teable-v2-reason']).toBe('new_base'); + expect(['new_base', 'env_force_v2_all']).toContain(response.headers['x-teable-v2-reason']); nodesToCleanup.push(response.data.id); expect(response.data.resourceMeta?.name).toBe('Duplicated Table Via Node Route'); diff --git a/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts b/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts index 051e51a241..3694e98072 100644 --- a/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts +++ b/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts @@ -909,7 +909,9 @@ describeByodbStorage('BYODB space storage placement (e2e)', () => { }); expect(records.data.records).toHaveLength(1); expect(records.headers[X_TEABLE_V2_HEADER]).toBe('true'); - expect(records.headers[X_TEABLE_V2_REASON_HEADER]).toBe('new_base'); + expect(['new_base', 'env_force_v2_all']).toContain( + records.headers[X_TEABLE_V2_REASON_HEADER] + ); const scopedDataPrisma = (await app .get(DataDbClientManager) diff --git a/apps/nestjs-backend/test/import-base.e2e-spec.ts b/apps/nestjs-backend/test/import-base.e2e-spec.ts index db0eec872c..e5e5470d1f 100644 --- a/apps/nestjs-backend/test/import-base.e2e-spec.ts +++ b/apps/nestjs-backend/test/import-base.e2e-spec.ts @@ -1034,7 +1034,7 @@ describe('OpenAPI BaseController for base import (e2e)', () => { const prisma = app.get(PrismaService); if (process.env.V2_COMPUTED_UPDATE_MODE === 'sync') { - const deadline = Date.now() + 15_000; + const deadline = Date.now() + 45_000; while (Date.now() < deadline) { const deferredTasks = await prisma.computedUpdateOutbox.findMany({ @@ -1064,7 +1064,7 @@ describe('OpenAPI BaseController for base import (e2e)', () => { throw new Error(`Timed out waiting for claimed computed tasks to drain for base ${baseId}`); } - const deadline = Date.now() + 15_000; + const deadline = Date.now() + 45_000; while (Date.now() < deadline) { const activeTaskCount = await prisma.computedUpdateOutbox.count({ where: { baseId } }); diff --git a/apps/nestjs-backend/test/oauth-server.e2e-spec.ts b/apps/nestjs-backend/test/oauth-server.e2e-spec.ts index 9aa1c390b6..b06d137eab 100644 --- a/apps/nestjs-backend/test/oauth-server.e2e-spec.ts +++ b/apps/nestjs-backend/test/oauth-server.e2e-spec.ts @@ -415,7 +415,50 @@ describe('OpenAPI OAuthController (e2e)', () => { ) ); expect(error?.status).toBe(403); - // base|read_all + // base|read_all must NOT be granted implicitly: this token only consented + // to table|read, so /base/access/all (requires base|read_all) is denied. + // (Regression guard for GHSA-c57x — previously read_all was auto-added.) + const baseListError = await getError(() => + anonymousAxios.get(`/base/access/all`, { + headers: { + Authorization: `${tokenRes.data.token_type} ${tokenRes.data.access_token}`, + }, + }) + ); + expect(baseListError?.status).toBe(403); + }); + + it('/api/oauth/access_token (POST) - base|read_all only granted when consented', async () => { + const oauthRes = await oauthCreate({ + ...oauthData, + scopes: ['base|read_all'], + }); + const { transactionID } = await getAuthorize(axios, oauthRes.data); + + const res = await decision(axios, transactionID!); + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + const secret = await generateOAuthSecret(oauthRes.data.clientId); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauthRes.data.clientId, + client_secret: secret.data.secret, + redirect_uri: oauthRes.data.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + + // With base|read_all consented, /base/access/all succeeds. const baseListRes = await anonymousAxios.get(`/base/access/all`, { headers: { Authorization: `${tokenRes.data.token_type} ${tokenRes.data.access_token}`, diff --git a/apps/nestjs-backend/test/performance.e2e-spec.ts b/apps/nestjs-backend/test/performance.e2e-spec.ts deleted file mode 100644 index 42398e44cb..0000000000 --- a/apps/nestjs-backend/test/performance.e2e-spec.ts +++ /dev/null @@ -1,159 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -/* eslint-disable sonarjs/no-duplicate-string */ -import { faker } from '@faker-js/faker'; -import type { INestApplication } from '@nestjs/common'; -import { Colors, FieldType, RatingIcon, Relationship } from '@teable/core'; -import { createRecords, createTable } from '@teable/openapi'; -import type { ITableFullVo } from '@teable/openapi'; -import { initApp, permanentDeleteTable } from './utils/init-app'; - -describe('OpenAPI RecordController (e2e)', () => { - let app: INestApplication; - const baseId = globalThis.testConfig.baseId; - const userId = globalThis.testConfig.userId; - beforeAll(async () => { - const appCtx = await initApp(); - app = appCtx.app; - }); - - afterAll(async () => { - await app.close(); - }); - - describe('create records performance', () => { - let table1: ITableFullVo; - let table2: ITableFullVo; - const batchSize = 1000; - - beforeEach(async () => { - table2 = await createTable(baseId, { - name: 'table2', - fields: [ - { - type: FieldType.SingleLineText, - name: 'Title', - }, - ], - records: [ - { - fields: { - Title: 'A1', - }, - }, - { - fields: { - Title: 'A2', - }, - }, - { - fields: { - Title: 'A3', - }, - }, - ], - }).then((res) => res.data); - - table1 = await createTable(baseId, { - name: 'table1', - fields: [ - { - type: FieldType.SingleLineText, - name: 'Title', - }, - { - type: FieldType.Number, - name: 'Count', - }, - { - type: FieldType.SingleSelect, - name: 'Status', - options: { - choices: [{ name: 'Not Started' }, { name: 'In Progress' }, { name: 'Completed' }], - }, - }, - { - type: FieldType.LongText, - name: 'Text', - }, - { - type: FieldType.MultipleSelect, - name: 'Tags', - options: { - choices: [ - { name: 'Tag 1' }, - { name: 'Tag 2' }, - { name: 'Tag 3' }, - { name: 'Tag 4' }, - { name: 'Tag 5' }, - ], - }, - }, - { - type: FieldType.User, - name: 'Member', - }, - { - type: FieldType.Date, - name: 'Date', - }, - { - type: FieldType.Rating, - name: 'Rating', - options: { - icon: RatingIcon.Star, - color: Colors.YellowBright, - max: 5, - }, - }, - { - type: FieldType.Link, - name: 'One-way Link', - options: { - relationship: Relationship.ManyOne, - foreignTableId: table2.id, - isOneWay: true, - }, - }, - { - type: FieldType.Link, - name: 'Two-way Link', - options: { - relationship: Relationship.ManyOne, - foreignTableId: table2.id, - }, - }, - ], - }).then((res) => res.data); - }); - - afterEach(async () => { - await permanentDeleteTable(baseId, table1.id); - await permanentDeleteTable(baseId, table2.id); - }); - - it('batch create records', { timeout: 10000 }, async () => { - const { data } = await createRecords(table1.id, { - typecast: true, - records: Array.from({ length: batchSize }, () => ({ - fields: { - Title: faker.lorem.sentence(), - Count: faker.number.int({ min: 1, max: 100 }), - Status: faker.helpers.arrayElement(['Not Started', 'In Progress', 'Completed']), - Text: faker.lorem.paragraph(), - Tags: faker.helpers.arrayElements(['Tag 1', 'Tag 2', 'Tag 3', 'Tag 4', 'Tag 5'], { - min: 1, - max: 5, - }), - Member: userId, - Date: faker.date.recent().toISOString(), - Rating: faker.number.int({ min: 0, max: 5 }), - 'One-way Link': faker.helpers.arrayElement(['A1', 'A2', 'A3']), - 'Two-way Link': faker.helpers.arrayElement(['A1', 'A2', 'A3']), - }, - })), - }); - - expect(data.records).toHaveLength(batchSize); - }); - }); -}); diff --git a/apps/nestjs-backend/test/table-trash.e2e-spec.ts b/apps/nestjs-backend/test/table-trash.e2e-spec.ts index 0b73ea1dd2..caf5b920af 100644 --- a/apps/nestjs-backend/test/table-trash.e2e-spec.ts +++ b/apps/nestjs-backend/test/table-trash.e2e-spec.ts @@ -206,7 +206,8 @@ describe('Trash (e2e)', () => { await awaitWithFieldDeleteSync(async () => deleteFields(tableId, deletedFieldIds)); - const result = await getTrashItems({ resourceId: tableId, resourceType: ResourceType.Table }); + // Wait like other field-delete cases: trash projection can lag slightly after sync. + const result = await waitForTableTrashItems(tableId, 1); expect(result.data.trashItems.length).toBe(1); expect((result.data.trashItems[0] as ITableTrashItemVo).resourceIds).toEqual(deletedFieldIds); diff --git a/apps/nestjs-backend/test/trash.e2e-spec.ts b/apps/nestjs-backend/test/trash.e2e-spec.ts index 72ecf990ee..f2893af616 100644 --- a/apps/nestjs-backend/test/trash.e2e-spec.ts +++ b/apps/nestjs-backend/test/trash.e2e-spec.ts @@ -211,7 +211,7 @@ describe('Trash (e2e)', () => { expect(restored.status).toEqual(201); expect(restored.headers['x-teable-v2']).toBe('true'); expect(restored.headers['x-teable-v2-feature']).toBe('restoreTable'); - expect(restored.headers['x-teable-v2-reason']).toBe('new_base'); + expect(restored.headers['x-teable-v2-reason']).toBe('env_force_v2_all'); }); }); diff --git a/apps/nestjs-backend/test/user-last-visit.e2e-spec.ts b/apps/nestjs-backend/test/user-last-visit.e2e-spec.ts index f96bcd91a6..f3371b5122 100644 --- a/apps/nestjs-backend/test/user-last-visit.e2e-spec.ts +++ b/apps/nestjs-backend/test/user-last-visit.e2e-spec.ts @@ -195,6 +195,11 @@ describe('OpenAPI OAuthController (e2e)', () => { }); it('should get last visit list base', async () => { + const prisma = app.get(PrismaService); + await prisma.userLastVisit.deleteMany({ + where: { resourceType: LastVisitResourceType.Base, resourceId: base.id }, + }); + // eslint-disable-next-line @typescript-eslint/naming-convention const base_21: ICreateBaseVo[] = []; @@ -229,7 +234,6 @@ describe('OpenAPI OAuthController (e2e)', () => { expect(res2.data.list.length).toEqual(0); - const prisma = app.get(PrismaService); const userLastVisit = await prisma.userLastVisit.findMany({ where: { parentResourceId: base_21[0].spaceId, diff --git a/apps/nestjs-backend/test/utils/action-trigger.ts b/apps/nestjs-backend/test/utils/action-trigger.ts index df6b26d761..3232d5d1b7 100644 --- a/apps/nestjs-backend/test/utils/action-trigger.ts +++ b/apps/nestjs-backend/test/utils/action-trigger.ts @@ -39,8 +39,10 @@ export const collectActionTriggers = async (params: { port, tableId, act, - idleMs = 300, - timeoutMs = 5000, + // Parallel e2e workers contend for CPU; presence events can trail `act` by + // well over 300ms, and finishing early records an empty batch. + idleMs = 1500, + timeoutMs = 8000, until, } = params; diff --git a/apps/nestjs-backend/test/utils/e2e-shared.ts b/apps/nestjs-backend/test/utils/e2e-shared.ts new file mode 100644 index 0000000000..02cbe99ccb --- /dev/null +++ b/apps/nestjs-backend/test/utils/e2e-shared.ts @@ -0,0 +1,440 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/** + * Shared-app-per-worker support for the e2e suites. + * + * The e2e configs run with `isolate: false` so each vitest worker process executes + * many spec files sequentially. Booting the full Nest AppModule for every file used + * to dominate CI time (~24s per file), so the first boot in a worker is cached here + * and reused by later files. Correctness guards: + * + * - Spec files that mutate process.env BEFORE calling initApp (boot-time config) + * automatically fall back to a private, really-closable app: initApp compares the + * current env against the worker's baseline fingerprint and boots fresh on any + * difference. The custom e2e runner restores the baseline env after every file. + * - `close()` on the shared app is a no-op (156 spec files call it in afterAll); the + * worker process teardown reclaims resources. Fresh apps keep a real close() that + * also restores the axios singleton to the shared app. + * + * Parallel file execution is made safe by giving every worker its own database, + * cloned from the freshly seeded template DB in globalSetup (CREATE DATABASE ... + * TEMPLATE). Workers rewrite PRISMA_DATABASE_URL before anything connects. + */ +import os from 'node:os'; +import type { INestApplication } from '@nestjs/common'; + +export interface ISharedBundle { + app: INestApplication; + appUrl: string; + cookie: string; + sessionID: string; +} + +interface ISharedEntry { + bundle: ISharedBundle; + proxied: ISharedBundle; +} + +interface ISharedState { + // Boot promises, stored synchronously: concurrent initApp calls (the eagerly + // booting setup file races the first spec file — vitest does not await setup + // promises) join the same in-flight boot instead of double-booting. + registry: Map>; + resolved: Map; + baselineEnv?: Record; + originalDbUrl?: string; + originalCacheRedisUri?: string; + originalPerfCacheUri?: string; + /** First shared app to finish booting — the worker's canonical axios target. */ + primaryKey?: string; + /** Interceptor-list lengths right after the primary shared app registered its own. */ + axiosSnapshot?: { request: number; response: number }; + /** Private apps whose spec never called close(); reaped after each file. */ + strayFreshApps: Set<() => Promise>; + /** Async cleanup from the runner's per-file reset; awaited by the next initApp. */ + pendingMaintenance?: Promise; +} + +/** + * Env keys that never force a private app and are not reset between files. + * PORT/STORAGE_PREFIX are NOT ignored: the shared boot merges its values into the + * baseline, and restoring them after a private-app file matters — the server + * builds attachment URLs from STORAGE_PREFIX at request time, so a dead private + * app's leftover value would break later files. + */ +const IGNORED_ENV_KEYS = new Set(['VITEST_POOL_ID', 'VITEST_WORKER_ID', 'NODE_OPTIONS']); + +function state(): ISharedState { + const g = globalThis as any; + g.__teableE2eShared ??= { + registry: new Map(), + resolved: new Map(), + strayFreshApps: new Set(), + } satisfies ISharedState; + return g.__teableE2eShared; +} + +/** + * Strict opt-in: the e2e vitest configs set E2E_SHARED_APP=1. Other configs that + * reuse initApp (vitest-ai, bench) keep the classic app-per-file behavior. + */ +export function sharedAppEnabled(): boolean { + return process.env.E2E_SHARED_APP === '1'; +} + +// Redis db per worker must stay within redis' 16 default databases, clear of the +// dev defaults (0/1) and the dedicated BullMQ e2e db (14). +const MAX_E2E_WORKERS = 12; + +export function resolveE2eMaxWorkers(): number { + if (process.env.E2E_FILE_PARALLELISM === '0') return 1; + const explicit = Number(process.env.E2E_MAX_WORKERS || 0); + if (explicit > 0) return Math.min(explicit, MAX_E2E_WORKERS); + const cpus = os.availableParallelism?.() ?? os.cpus().length; + // Hosted CI runners are small (2 vCPU / 7GB); each worker holds a full Nest app. + if (process.env.CI) return 3; + return Math.max(2, Math.min(8, Math.floor(cpus / 2))); +} + +/** + * Capture the post-setup env as the reference for fingerprinting and per-file + * restore. Setup files re-run for every test file, so only the first capture + * (right after the shared app booted) counts. + */ +export function captureBaselineEnv(): void { + state().baselineEnv ??= { ...process.env }; +} + +export function restoreBaselineEnv(): void { + const baseline = state().baselineEnv; + if (!baseline) return; + const restored: string[] = []; + for (const key of Object.keys(process.env)) { + if (IGNORED_ENV_KEYS.has(key)) continue; + if (!(key in baseline)) { + delete process.env[key]; + restored.push(key); + } + } + for (const [key, value] of Object.entries(baseline)) { + if (IGNORED_ENV_KEYS.has(key)) continue; + if (process.env[key] !== value) { + process.env[key] = value; + restored.push(key); + } + } + if (restored.length > 0 && process.env.E2E_PROBE) { + // eslint-disable-next-line no-console + console.log(`[e2e-shared] restored env after file: ${restored.join(', ')}`); + } +} + +function envDiffFromBaseline(): string[] { + const baseline = state().baselineEnv; + if (!baseline) return []; // eager boot happens before the baseline is captured + const keys = new Set([...Object.keys(baseline), ...Object.keys(process.env)]); + const diff: string[] = []; + for (const key of keys) { + if (IGNORED_ENV_KEYS.has(key)) continue; + if ((baseline[key] ?? undefined) !== (process.env[key] ?? undefined)) { + diff.push(key); + } + } + return diff; +} + +export interface IBootResult { + bundle: ISharedBundle; + cookieInterceptorId: number; +} + +/** + * Resolve an app for the calling spec file: reuse this worker's shared app, boot + * it if it doesn't exist yet, or fall back to a private app when the caller's env + * diverges from the worker baseline (spec customized process.env before initApp). + */ +let privateBootCounter = 0; + +/** + * Boot a private app under its own BullMQ queue prefix so the worker's shared + * app (same redis db) can't consume the private app's jobs, or vice versa. + */ +export async function bootWithPrivateQueues(boot: () => Promise): Promise { + const previous = process.env.BACKEND_QUEUE_PREFIX; + process.env.BACKEND_QUEUE_PREFIX = `bullp${process.env.VITEST_POOL_ID ?? 0}x${++privateBootCounter}`; + try { + return await boot(); + } finally { + if (previous === undefined) { + delete process.env.BACKEND_QUEUE_PREFIX; + } else { + process.env.BACKEND_QUEUE_PREFIX = previous; + } + } +} + +export function setPendingMaintenance(promise: Promise): void { + state().pendingMaintenance = promise.catch(() => undefined); +} + +export async function acquireApp( + cacheKey: string, + boot: () => Promise, + restoreAxios: (bootResult: IBootResult) => void, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + axios?: any +): Promise { + // Let the previous file's async cleanup (cache value flush) settle before this + // file starts priming caches. + const pending = state().pendingMaintenance; + if (pending) { + state().pendingMaintenance = undefined; + await pending; + } + const envDiff = sharedAppEnabled() ? envDiffFromBaseline() : []; + if (!sharedAppEnabled() || envDiff.length > 0) { + if (envDiff.length > 0) { + // eslint-disable-next-line no-console + console.log(`[e2e-shared] private app for "${cacheKey}", env differs: ${envDiff.join(', ')}`); + } + const bootResult = await bootWithPrivateQueues(boot); + return wrapFreshApp(bootResult.bundle, () => restoreAxios(bootResult)); + } + + const st = state(); + let entryPromise = st.registry.get(cacheKey); + if (!entryPromise) { + // Files run sequentially inside a worker, so nothing else executes test code + // while this boot is in flight: any env delta across the boot is a boot + // artifact (e.g. SSL_CERT_FILE) — absorb it into the baseline so later files + // aren't misclassified as env-customized. + const preBootEnv = { ...process.env }; + entryPromise = boot().then(({ bundle }) => { + const baseline = st.baselineEnv; + if (baseline) { + const keys = new Set([...Object.keys(preBootEnv), ...Object.keys(process.env)]); + for (const key of keys) { + if ((preBootEnv[key] ?? undefined) !== (process.env[key] ?? undefined)) { + baseline[key] = process.env[key]; + } + } + } + const entry: ISharedEntry = { + bundle, + proxied: { ...bundle, app: closelessApp(bundle.app) }, + }; + st.resolved.set(cacheKey, entry); + if (!st.primaryKey && axios) { + st.primaryKey = cacheKey; + st.axiosSnapshot = { + request: axios.interceptors.request.handlers.length, + response: axios.interceptors.response.handlers.length, + }; + } + return entry; + }); + st.registry.set(cacheKey, entryPromise); + } + const entry = await entryPromise; + // Reusing a secondary shared app (e.g. the EE-edition app while CLOUD is the + // worker primary): point the axios singleton at it — booting did this, reuse + // must too. The runner resets back to the primary after the file. + if (axios) { + axios.defaults.baseURL = entry.bundle.appUrl + '/api'; + } + return entry.proxied; +} + +/* --------------------------- axios singleton hygiene --------------------------- */ + +/** + * The openapi package's axios singleton is worker-global. Under the old + * app-per-file model every file started from a fresh process; here the runner + * resets the singleton to the shared app's state after each file so a spec that + * booted a private app (or added interceptors) can't poison the next file. + * The axios instance is passed in by the caller (worker-side modules only). + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resetAxiosToSharedApp(axios: any): void { + const st = state(); + if (!st.primaryKey || !st.axiosSnapshot) return; + const entry = st.resolved.get(st.primaryKey); + if (!entry) return; + axios.defaults.baseURL = entry.bundle.appUrl + '/api'; + for (let id = st.axiosSnapshot.request; id < axios.interceptors.request.handlers.length; id++) { + axios.interceptors.request.eject(id); + } + for (let id = st.axiosSnapshot.response; id < axios.interceptors.response.handlers.length; id++) { + axios.interceptors.response.eject(id); + } +} + +function closelessApp(app: INestApplication): INestApplication { + return new Proxy(app, { + get(target, prop) { + if (prop === 'close') { + return async () => undefined; + } + const value = Reflect.get(target, prop) as unknown; + return typeof value === 'function' ? (value as any).bind(target) : value; + }, + }); +} + +export function getSharedBundle(cacheKey: string): ISharedBundle | undefined { + return state().resolved.get(cacheKey)?.bundle; +} + +export function getAllSharedBundles(): ISharedBundle[] { + return [...state().resolved.values()].map((entry) => entry.bundle); +} + +/** + * Wrap a privately-booted app so its close() also restores the axios singleton + * (baseURL + cookie interceptor) back to the shared app of this worker. + */ +export function wrapFreshApp(bundle: ISharedBundle, restoreAxios: () => void): ISharedBundle { + const app = bundle.app; + const realClose = app.close.bind(app); + let closed = false; + const close = async () => { + if (closed) return; + closed = true; + state().strayFreshApps.delete(close); + restoreAxios(); + await realClose(); + }; + state().strayFreshApps.add(close); + const proxied = new Proxy(app, { + get(target, prop) { + if (prop === 'close') return close; + const value = Reflect.get(target, prop) as unknown; + return typeof value === 'function' ? (value as any).bind(target) : value; + }, + }); + return { ...bundle, app: proxied }; +} + +/** + * Close private apps whose spec file finished without calling close(). Left + * running they would keep consuming queue events on this worker's DB/redis. + */ +export function reapStrayFreshApps(): void { + for (const close of [...state().strayFreshApps]) { + void close().catch(() => undefined); + } +} + +/* ------------------------------- worker databases ------------------------------- */ + +export function workerDbEnabled(): boolean { + return process.env.E2E_WORKER_DB !== '0'; +} + +export function workerDatabaseUrl(url: string, poolId: number): string { + const parsed = new URL(url); + const dbName = parsed.pathname.replace(/^\//, ''); + parsed.pathname = `/${dbName}_w${poolId}`; + // Keep the original connection_limit: concurrency-heavy specs need the full + // pool. The CI postgres runs with max_connections=500 to absorb N workers. + return parsed.toString(); +} + +/** + * Point PRISMA_DATABASE_URL at this worker's database clone. Must run in the worker + * setup file before anything opens a connection. Setup files re-run for every test + * file, so always derive from the original template URL (idempotent). + */ +export function applyWorkerDatabaseEnv(): void { + if (!workerDbEnabled()) return; + const poolId = Number(process.env.VITEST_POOL_ID || 0); + if (!poolId || resolveE2eMaxWorkers() <= 1) return; + + const url = (state().originalDbUrl ??= process.env.PRISMA_DATABASE_URL); + if (url) { + process.env.PRISMA_DATABASE_URL = workerDatabaseUrl(url, poolId); + } + + // The redis cache holds sessions and permission caches keyed by the shared seed + // user; concurrent workers on one redis db would clobber each other (e.g. the + // auth specs invalidating every worker's session). One redis db per worker. + const cacheUri = (state().originalCacheRedisUri ??= process.env.BACKEND_CACHE_REDIS_URI); + if (cacheUri && process.env.BACKEND_CACHE_PROVIDER === 'redis') { + const parsed = new URL(cacheUri); + parsed.pathname = `/${poolId}`; + process.env.BACKEND_CACHE_REDIS_URI = parsed.toString(); + } + + // Same isolation for the performance cache: its keys are global strings (e.g. + // the instance setting containing the canary config), so a shared redis db + // would let workers read each other's settings. + const perfUri = (state().originalPerfCacheUri ??= process.env.BACKEND_PERFORMANCE_CACHE); + if (perfUri) { + const parsed = new URL(perfUri); + parsed.pathname = `/${poolId}`; + process.env.BACKEND_PERFORMANCE_CACHE = parsed.toString(); + } +} + +/** + * globalSetup helper: clone the seeded template database once per worker. + * Runs in the vitest main process before any worker spawns. + */ +export async function provisionWorkerDatabases(): Promise { + if (!workerDbEnabled()) return; + const url = process.env.PRISMA_DATABASE_URL; + const workers = resolveE2eMaxWorkers(); + if (!url || workers <= 1) return; + + const parsed = new URL(url); + const template = parsed.pathname.replace(/^\//, ''); + const admin = new URL(url); + admin.pathname = '/postgres'; + admin.search = ''; + + const { default: pg } = await import('pg'); + const client = new pg.Client({ connectionString: admin.toString() }); + await client.connect(); + try { + await client.query( + `select pg_terminate_backend(pid) from pg_stat_activity + where datname like $1 and pid <> pg_backend_pid()`, + [`${template}%`] + ); + for (let poolId = 1; poolId <= workers; poolId++) { + const dbName = `${template}_w${poolId}`; + await client.query(`drop database if exists "${dbName}"`); + await client.query(`create database "${dbName}" template "${template}"`); + } + // eslint-disable-next-line no-console + console.log(`[e2e] provisioned ${workers} worker databases from template "${template}"`); + } finally { + await client.end(); + } + + await flushWorkerRedisDbs(workers); +} + +/** + * Redis outlives a run on dev boxes; cached values (sessions, the 1-day-TTL + * instance settings in the performance cache) from a previous run would poison + * freshly cloned worker databases. Flush each worker's redis db up front. + */ +async function flushWorkerRedisDbs(workers: number): Promise { + const uri = process.env.BACKEND_CACHE_REDIS_URI ?? process.env.BACKEND_PERFORMANCE_CACHE; + if (!uri) return; + try { + const { default: redisCtor } = await import('ioredis'); + const redis = new redisCtor(uri, { lazyConnect: true, maxRetriesPerRequest: 1 }); + await redis.connect(); + for (let poolId = 1; poolId <= workers; poolId++) { + await redis.select(poolId); + await redis.flushdb(); + } + redis.disconnect(); + // eslint-disable-next-line no-console + console.log(`[e2e] flushed redis dbs 1..${workers}`); + } catch (error) { + // eslint-disable-next-line no-console + console.warn(`[e2e] redis flush skipped: ${(error as Error).message}`); + } +} diff --git a/apps/nestjs-backend/test/utils/e2e-test-runner.ts b/apps/nestjs-backend/test/utils/e2e-test-runner.ts new file mode 100644 index 0000000000..bca0d81ede --- /dev/null +++ b/apps/nestjs-backend/test/utils/e2e-test-runner.ts @@ -0,0 +1,72 @@ +import { axios } from '@teable/openapi'; +import { VitestTestRunner } from 'vitest/runners'; +import type { IBaseConfig } from '../../src/configs/base.config'; +import { baseConfig } from '../../src/configs/base.config'; +import { PerformanceCacheService } from '../../src/performance-cache'; +import { + getAllSharedBundles, + reapStrayFreshApps, + resetAxiosToSharedApp, + restoreBaselineEnv, + setPendingMaintenance, +} from './e2e-shared'; + +/** + * E2e runner for the shared-app worker model (`isolate: false`). + * + * Workers execute many spec files in one process, so env mutations made by a spec + * (runtime feature flags like FORCE_V2_ALL) would leak into the next file. After + * each file finishes, reset process.env to the baseline captured right after the + * worker booted its shared app. + */ +export default class E2eTestRunner extends VitestTestRunner { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + onCollectStart(file: any): void { + // Vitest may collect a file in a different worker than the one that runs it; + // collection imports the module, so import-time env mutations would otherwise + // linger in the collecting worker. Start every file from the baseline. + restoreBaselineEnv(); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const name = String(file?.filepath ?? '') + .split('/') + .pop(); + if (process.env.E2E_PROBE) { + // eslint-disable-next-line no-console + console.log(`[e2e-probe] file-start pool=${process.env.VITEST_POOL_ID} ${name}`); + } + super.onCollectStart(file); + } + + onAfterRunFiles(): void { + super.onAfterRunFiles(); + restoreBaselineEnv(); + reapStrayFreshApps(); + resetAxiosToSharedApp(axios); + // Cache-stats specs assert absolute hit/miss counts, and some specs flip + // boot-time config flags (e.g. recordHistoryDisabled) on the shared app. + // Start each file from the same state a freshly booted app would have. + const cleanups: Promise[] = []; + for (const bundle of getAllSharedBundles()) { + try { + const perfCache = bundle.app.get(PerformanceCacheService, { strict: false }); + perfCache?.resetTypeStats?.(); + // Values too: under app-per-file every file got a new session id, so + // sid-scoped cache entries were never warm at file start. The flush is + // async; the next file's initApp awaits it via setPendingMaintenance. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const cleared = (perfCache as any)?._clear?.(); + if (cleared) cleanups.push(cleared.catch(() => undefined)); + const config = bundle.app.get(baseConfig.KEY, { strict: false }); + if (config) { + config.recordHistoryDisabled = true; + config.storagePrefix = bundle.appUrl; + } + } catch { + // app may be mid-shutdown; per-file state reset is best-effort + } + } + if (cleanups.length > 0) { + setPendingMaintenance(Promise.all(cleanups)); + } + } +} diff --git a/apps/nestjs-backend/test/utils/init-app.ts b/apps/nestjs-backend/test/utils/init-app.ts index c01d4269b1..0ef5cabbf7 100644 --- a/apps/nestjs-backend/test/utils/init-app.ts +++ b/apps/nestjs-backend/test/utils/init-app.ts @@ -2,6 +2,7 @@ import type { INestApplication } from '@nestjs/common'; import { ValidationPipe } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { EventEmitter2 } from '@nestjs/event-emitter'; import { WsAdapter } from '@nestjs/platform-ws'; import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; @@ -78,12 +79,35 @@ import { GlobalExceptionFilter } from '../../src/filter/global-exception.filter' import type { IClsStore } from '../../src/types/cls'; import { WsGateway } from '../../src/ws/ws.gateway'; import { DevWsGateway } from '../../src/ws/ws.gateway.dev'; +import { acquireApp, getSharedBundle } from './e2e-shared'; import { TestingLogger } from './testing-logger'; export async function initApp() { // eslint-disable-next-line @typescript-eslint/no-misused-promises if (globalThis.initApp) return await globalThis.initApp(); + const cacheKey = 'community:default'; + // Private apps (env customized by the spec, or sharing disabled) keep a real + // close() that also restores the axios singleton to this worker's shared app. + return acquireApp( + cacheKey, + bootApp, + ({ cookieInterceptorId }) => { + axios.interceptors.request.eject(cookieInterceptorId); + const shared = getSharedBundle(cacheKey); + if (shared) { + axios.defaults.baseURL = shared.appUrl + '/api'; + } + }, + axios + ); +} + +async function bootApp() { + if (process.env.E2E_PROBE) { + // eslint-disable-next-line no-console + console.log(`[e2e-probe] BOOT community pool=${process.env.VITEST_POOL_ID}`); + } const moduleFixture: TestingModule = await Test.createTestingModule({ imports: [AppModule, BaseSqlExecutorModule], }) @@ -93,6 +117,12 @@ export async function initApp() { return; }, }) + // EventEmitterModule.forRoot() is evaluated once per process, so every test + // app would otherwise share one EventEmitter2: closing any app wipes all + // listeners (onApplicationShutdown -> removeAllListeners) and events double + // fire across coexisting apps. Give each app its own emitter. + .overrideProvider(EventEmitter2) + .useValue(new EventEmitter2({ wildcard: true, delimiter: '.' })) .overrideProvider(DevWsGateway) .useClass(WsGateway) .compile(); @@ -132,8 +162,13 @@ export async function initApp() { await getCookie(globalThis.testConfig.email, globalThis.testConfig.password) ).cookie.join(';'); - axios.interceptors.request.use((config) => { - config.headers.Cookie = cookie; + const cookieInterceptorId = axios.interceptors.request.use((config) => { + // Never attach the shared session to signin/signup: passport regenerates the + // session attached to a login request, which would destroy this cookie's sid + // and break every later spec file sharing the app. + if (!/\/auth\/(?:signin|signup)\b/.test(config.url ?? '')) { + config.headers.Cookie = cookie; + } return config; }); @@ -147,7 +182,7 @@ export async function initApp() { console.log('> Test Current System Time:', now.toString()); const sessionHandleService = app.get(SessionHandleService); - return { + const bundle = { app, appUrl: url, cookie, @@ -157,6 +192,7 @@ export async function initApp() { // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any), }; + return { bundle, cookieInterceptorId }; } /** diff --git a/apps/nestjs-backend/vitest-e2e.config.ts b/apps/nestjs-backend/vitest-e2e.config.ts index cf3f329cc9..0a02c7d72c 100644 --- a/apps/nestjs-backend/vitest-e2e.config.ts +++ b/apps/nestjs-backend/vitest-e2e.config.ts @@ -1,6 +1,7 @@ import swc from 'unplugin-swc'; import tsconfigPaths from 'vite-tsconfig-paths'; import { configDefaults, defineConfig } from 'vitest/config'; +import { resolveE2eMaxWorkers } from './test/utils/e2e-shared'; // Set timezone to UTC for deterministic datetime test results // This must be set before any datetime operations @@ -14,8 +15,15 @@ if (!process.env.CONDITIONAL_QUERY_DEFAULT_LIMIT) { process.env.CONDITIONAL_QUERY_DEFAULT_LIMIT = process.env.CONDITIONAL_QUERY_MAX_LIMIT; } +// Shared-app worker model (see test/utils/e2e-shared.ts): workers reuse one Nest +// app across spec files and run files in parallel against per-worker databases. +process.env.E2E_SHARED_APP ??= '1'; +process.env.E2E_WORKER_DB ??= '1'; +const e2eMaxWorkers = resolveE2eMaxWorkers(); + const timeout = process.env.CI ? 60000 : 10000; -const testFiles = ['**/test/**/*.{e2e-test,e2e-spec}.{js,ts}']; +// Anchored at test/ so the glob never has to crawl node_modules. +const testFiles = ['test/**/*.{e2e-test,e2e-spec}.{js,ts}']; export default defineConfig({ resolve: { @@ -43,11 +51,15 @@ export default defineConfig({ globals: true, environment: 'node', setupFiles: './vitest-e2e.setup.ts', + globalSetup: './vitest-e2e.global-setup.ts', + runner: './test/utils/e2e-test-runner.ts', testTimeout: timeout, hookTimeout: timeout, passWithNoTests: true, - pool: 'threads', - fileParallelism: false, + pool: 'forks', + isolate: process.env.E2E_ISOLATE === '1', + fileParallelism: e2eMaxWorkers > 1, + maxWorkers: e2eMaxWorkers, coverage: { provider: 'v8', reportsDirectory: './coverage/e2e', diff --git a/apps/nestjs-backend/vitest-e2e.global-setup.ts b/apps/nestjs-backend/vitest-e2e.global-setup.ts new file mode 100644 index 0000000000..7ecaf76bce --- /dev/null +++ b/apps/nestjs-backend/vitest-e2e.global-setup.ts @@ -0,0 +1,31 @@ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import dotenv from 'dotenv-flow'; +import { buildSync } from 'esbuild'; +import { provisionWorkerDatabases } from './test/utils/e2e-shared'; + +const dirname = path.dirname(fileURLToPath(import.meta.url)); + +/** + * Runs once in the vitest main process, after `pre-test-e2e` seeded the template + * database and before any worker spawns: + * - compile the worker bundle once so parallel workers don't race in buildSync + * - clone the template database once per worker so spec files can run in + * parallel without sharing state + */ +export default async function globalSetup() { + // Workers load this via their setup file; the main process needs it here so + // provisionWorkerDatabases can see the redis URIs to flush. + dotenv.config({ path: '../nextjs-app', node_env: process.env.NODE_ENV || 'test' }); + + buildSync({ + entryPoints: [path.join(dirname, 'src/worker/**.ts')], + outdir: path.join(dirname, 'dist/worker'), + bundle: true, + platform: 'node', + target: 'node20', + }); + process.env.E2E_WORKER_PREBUILT = '1'; + + await provisionWorkerDatabases(); +} diff --git a/apps/nestjs-backend/vitest-e2e.setup.ts b/apps/nestjs-backend/vitest-e2e.setup.ts index 97d0c99932..5b70f23767 100644 --- a/apps/nestjs-backend/vitest-e2e.setup.ts +++ b/apps/nestjs-backend/vitest-e2e.setup.ts @@ -1,4 +1,5 @@ import { Buffer as NodeBuffer } from 'node:buffer'; +import http from 'node:http'; import { createRequire } from 'node:module'; import path from 'path'; import type { INestApplication } from '@nestjs/common'; @@ -6,6 +7,11 @@ import { DriverClient, parseDsn } from '@teable/core'; import dotenv from 'dotenv-flow'; import { buildSync } from 'esbuild'; +// Node >=19 enables keep-alive on the global http agent; the app servers close +// idle sockets after ~5s and a reuse racing that close surfaces as ECONNRESET. +// Tests favor determinism over connection reuse. +http.globalAgent = new http.Agent({ keepAlive: false }); + const require = createRequire(import.meta.url); const bufferModule = require('buffer') as Record; bufferModule['SlowBuffer'] ??= bufferModule['Buffer'] ?? NodeBuffer; @@ -99,6 +105,9 @@ async function setup() { process.env.CONDITIONAL_QUERY_DEFAULT_LIMIT = process.env.CONDITIONAL_QUERY_MAX_LIMIT; } + const { applyWorkerDatabaseEnv, captureBaselineEnv } = await import('./test/utils/e2e-shared'); + applyWorkerDatabaseEnv(); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const databaseUrl = process.env.PRISMA_DATABASE_URL!; @@ -107,7 +116,16 @@ async function setup() { console.log('driver: ', driver); globalThis.testConfig.driver = driver; - compileWorkerFile(); + // globalSetup pre-builds the worker bundle for the e2e configs; other configs + // (bench) still compile here, where files run serially. + if (process.env.E2E_WORKER_PREBUILT !== '1') { + compileWorkerFile(); + } + + // Fingerprint the clean pre-boot env. The first spec file's initApp boots the + // worker's shared app (no eager boot here: vitest does not await this setup + // promise, and an in-flight boot would race the first file's env fingerprint). + captureBaselineEnv(); } export default setup(); diff --git a/apps/nextjs-app/package.json b/apps/nextjs-app/package.json index 4524c8197a..38422d5997 100644 --- a/apps/nextjs-app/package.json +++ b/apps/nextjs-app/package.json @@ -139,6 +139,7 @@ "date-fns-tz": "3.2.0", "dayjs": "1.11.10", "decimal.js-light": "2.5.1", + "dompurify": "3.3.3", "echarts": "5.5.0", "emoji-mart": "5.5.2", "eventsource-parser": "1.1.2", diff --git a/apps/nextjs-app/src/components/Metrics.tsx b/apps/nextjs-app/src/components/Metrics.tsx index 45cc04f806..2ad34bc0a4 100644 --- a/apps/nextjs-app/src/components/Metrics.tsx +++ b/apps/nextjs-app/src/components/Metrics.tsx @@ -1,24 +1,93 @@ +import { ANONYMOUS_USER_ID } from '@teable/core'; import Script from 'next/script'; -import { useEffect, useState } from 'react'; +import { useEffect, useRef, useState } from 'react'; import { syncMarketingAttributionFromUrl } from '@/lib/marketing-attribution'; const GOOGLE_LINKER_DOMAINS = ['teable.ai', 'app.teable.ai']; const POSTHOG_DEFAULT_HOST = 'https://us.i.posthog.com'; const INTERNAL_EMAIL_REGEX = /@teable\.(?:io|ai|cn)$/i; +// Dispatched (on window) by the init snippet's `loaded` callback once the real +// posthog-js SDK (array.js) has replaced the stub. Identity-STATE reads +// (_isIdentified / get_property) are only answerable after this: the stub has +// no _isIdentified at all, and its get_property merely queues the call and +// returns undefined. +const POSTHOG_LOADED_EVENT = 'teable:posthog-loaded'; interface IPostHog { init: (key: string, config: Record) => void; capture: (event: string, properties?: Record) => void; identify: (distinctId: string, properties?: Record) => void; reset: () => void; + /** + * Real SDK only — absent on the pre-load stub, which is how we detect that + * array.js has loaded. Reads the persisted `$user_state` (covers both + * persistence and sessionPersistence stores). + */ + // eslint-disable-next-line @typescript-eslint/naming-convention + _isIdentified?: () => boolean; + /** Fallback identity-state read: persisted super property lookup. */ + get_property?: (propertyName: string) => unknown; } +const isPosthogSdkLoaded = (posthog?: IPostHog): boolean => + // eslint-disable-next-line no-underscore-dangle + !!posthog && typeof posthog._isIdentified === 'function'; + +// Real user ids are NOT required to carry the usr prefix (EE admin/API-created +// users can have custom ids) — only empty values and the reserved word +// 'anonymous' (case-insensitive, matching PostHog's reserved distinct_id +// semantics) are excluded. +// +// Shared by EVERY analytics tool in this file (Clarity, Umami, GA, PostHog): +// signed-out visitors on ensureLogin pages (/auth/login, /auth/signup via +// re-export, /invite, ...) get pageProps.user set to the backend's +// ANONYMOUS_USER, which carries a non-empty name ('Anonymous') and email +// ('anonymous@system.teable.ai') — so a truthy user or email is NOT proof of +// a signed-in account. Every identity call below must pass this gate. +const isIdentifiableUserId = (userId?: string): userId is string => + !!userId && userId.trim().toLowerCase() !== ANONYMOUS_USER_ID; + +// Newest page user id, mirrored into module scope by the PostHog component so +// handlePosthogLoaded can read it synchronously: the init snippet renders only +// once, so a React closure captured there would go stale. +let latestPageUserId: string | undefined; + +/** + * Runs SYNCHRONOUSLY inside posthog's `loaded` config callback — i.e. BEFORE + * the initial $pageview. posthog-js `_loaded()` invokes `config.loaded(this)` + * first and only then schedules `_captureInitialPageview()` via + * `setTimeout(..., 1)` ("this happens after loaded so a user can call identify + * or any other things before the pageview fires" — + * packages/browser/src/posthog-core.ts); `capture_pageview: 'history_change'` + * is truthy and goes through that same deferred path. So cutting a stale + * identity here guarantees the initial pageview of an anonymous load can never + * be attributed to the previously identified user. + */ +const handlePosthogLoaded = (posthog: IPostHog) => { + if (isIdentifiableUserId(latestPageUserId)) { + // A real user is on the page — the identify effect owns this case. + return; + } + // eslint-disable-next-line no-underscore-dangle + if (typeof posthog._isIdentified === 'function' && posthog._isIdentified()) { + posthog.reset(); + } +}; + declare global { interface Window { gtag?: (command: string, targetId: string | Date, config?: Record) => void; dataLayer?: unknown[]; posthog?: IPostHog; + /** Clarity queue/API — defined synchronously by the init snippet. */ + clarity?: (command: string, ...args: string[]) => void; + /** + * Hook invoked by the posthog init snippet's `loaded` callback with the + * real SDK instance, before the event dispatch and the initial $pageview. + */ + // eslint-disable-next-line @typescript-eslint/naming-convention + __teablePosthogOnLoaded?: (posthog: IPostHog) => void; } } @@ -33,32 +102,45 @@ export const MicrosoftClarity = ({ email?: string; }; }) => { + const userId = user?.id; + const userEmail = user?.email; + const [isClarityReady, setIsClarityReady] = useState(false); + + // Identify from an effect, not a one-shot inline script: login goes through + // a client-side router.push, which keeps this component mounted and would + // never re-run a script tag — the session would stay unidentified until a + // full page load. The effect re-runs whenever the user id changes (initial + // load, login, account switch alike). Clarity has no reset API, so + // anonymous/invalid users simply skip identification and stay on Clarity's + // own per-device session id. + useEffect(() => { + if (!isClarityReady || !isIdentifiableUserId(userId)) { + return; + } + window.clarity?.('identify', userEmail || userId); + }, [isClarityReady, userId, userEmail]); + if (!clarityId) { return null; } return ( - <> -