From ef3554d95681d5bd6033836a15c0916bd499457a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 02:15:55 +0000 Subject: [PATCH] [sync] T4086 Avoid computed updates through unavailable tables T4086 Synced from teableio/teable-ee@99a5c55 Co-authored-by: Aries X Co-authored-by: Bieber Co-authored-by: Boris Co-authored-by: Jocky-Teable Co-authored-by: Jun Lu Co-authored-by: Pengap Co-authored-by: Uno Co-authored-by: nichenqin --- apps/nestjs-backend/package.json | 1 + .../src/configs/env.validation.schema.spec.ts | 5 +- .../src/configs/env.validation.schema.ts | 1 - .../src/configs/threshold.config.ts | 2 +- .../db-provider/duplicate-table/abstract.ts | 6 +- .../duplicate-query.postgres.ts | 21 +- .../distributed-lock.module.ts | 12 + .../distributed-lock.service.spec.ts | 100 ++ .../distributed-lock.service.ts | 83 + .../src/distributed-lock/index.ts | 2 + .../listeners/record-history.listener.ts | 21 +- .../access-token/access-token.service.spec.ts | 81 +- .../access-token/access-token.service.ts | 37 +- .../aggregation/aggregation.service.spec.ts | 51 + .../aggregation/aggregation.service.ts | 52 +- .../attachments/attachments.controller.ts | 13 +- .../base-share/base-share-open.controller.ts | 58 +- .../features/base-share/base-share.module.ts | 2 + .../base-sql-executor.service.ts | 280 +--- .../src/features/base-sql-executor/utils.ts | 2 +- .../src/features/base/base-controller.spec.ts | 55 + .../base/base-duplicate-v2.service.ts | 25 + .../base/base-duplicate.service.spec.ts | 559 +++++++ .../features/base/base-duplicate.service.ts | 1013 ++++++++++-- .../features/base/base-export-v2.service.ts | 1002 ++++++++++++ .../src/features/base/base-export.service.ts | 289 +++- .../base-import-csv.processor.spec.ts | 92 ++ .../base-import-csv.processor.ts | 125 +- .../base-import-junction.processor.ts | 57 +- .../features/base/base-import.service.spec.ts | 155 +- .../src/features/base/base-import.service.ts | 244 ++- .../base/base-query/base-query.service.ts | 9 +- .../src/features/base/base.controller.ts | 161 +- .../src/features/base/base.module.ts | 6 + .../src/features/base/base.service.spec.ts | 76 + .../src/features/base/base.service.ts | 417 ++++- .../base/cross-space-detection.util.spec.ts | 437 +++++ .../base/cross-space-detection.util.ts | 118 ++ .../features/base/db-connection.service.ts | 50 +- .../nestjs-backend/src/features/base/utils.ts | 17 +- .../src/features/calculation/batch.service.ts | 178 +- .../field-calculation.service.spec.ts | 7 +- .../calculation/field-calculation.service.ts | 30 +- .../src/features/calculation/link.service.ts | 91 +- .../calculation/system-field.service.ts | 12 +- .../src/features/canary/canary.service.ts | 4 +- .../collaborator/collaborator.service.ts | 71 +- .../database-view/database-view.service.ts | 14 +- .../field-converting-link.service.ts | 23 +- .../field-converting.service.ts | 60 +- .../field-calculate/field-creating.service.ts | 86 +- .../field-supplement.service.ts | 203 ++- .../field-duplicate.service.ts | 424 +++-- .../src/features/field/field.service.spec.ts | 28 + .../src/features/field/field.service.ts | 108 +- .../field-open-api-v2.service.spec.ts | 10 + .../open-api/field-open-api-v2.service.ts | 105 +- .../field/open-api/field-open-api.service.ts | 533 ++++-- .../src/features/graph/graph.service.ts | 29 +- .../features/health/health.controller.test.ts | 9 +- .../src/features/health/health.controller.ts | 9 +- .../open-api/import-open-api-v2.service.ts | 4 +- .../features/integrity/foreign-key.service.ts | 24 +- .../integrity/integrity-v2.service.spec.ts | 1 + .../integrity/integrity-v2.service.ts | 19 +- .../features/integrity/link-field.service.ts | 19 +- .../integrity/link-integrity.service.ts | 34 +- .../integrity/unique-index.service.ts | 10 +- .../notification/notification.service.ts | 167 +- .../oauth/oauth-app-init.service.spec.ts | 63 + .../features/oauth/oauth-app-init.service.ts | 57 + .../src/features/oauth/oauth.module.ts | 4 + .../src/features/oauth/oauth.service.ts | 47 +- .../computed-dependency-collector.service.ts | 32 +- .../services/computed-orchestrator.service.ts | 8 +- .../services/link-cascade-resolver.ts | 19 +- .../record-computed-update.service.ts | 16 +- .../record-open-api-v2.service.spec.ts | 47 +- .../open-api/record-open-api-v2.service.ts | 133 +- .../open-api/record-open-api.service.spec.ts | 14 +- .../open-api/record-open-api.service.ts | 9 +- .../record-modify/record-create.service.ts | 2 +- .../record-modify.shared.service.ts | 6 +- .../features/record/record-query.service.ts | 10 +- .../features/record/record.service.spec.ts | 72 +- .../src/features/record/record.service.ts | 231 ++- .../open-api/admin-open-api.controller.ts | 15 +- .../setting/open-api/admin-open-api.module.ts | 2 + .../open-api/admin-open-api.service.ts | 25 +- .../src/features/share/guard/auth.guard.ts | 5 +- .../src/features/share/share.service.ts | 25 +- .../space/data-db-baseline.service.ts | 60 +- .../space/data-db-binding.service.spec.ts | 326 +++- .../features/space/data-db-binding.service.ts | 282 +++- .../features/space/data-db-internal-schema.ts | 34 + .../space/data-db-migration.service.spec.ts | 259 +++ .../space/data-db-migration.service.ts | 317 ++++ .../space/data-db-preflight.service.spec.ts | 159 +- .../space/data-db-preflight.service.ts | 195 ++- .../src/features/space/space.service.ts | 10 +- .../table-open-api-v2.service.spec.ts | 99 +- .../open-api/table-open-api-v2.service.ts | 113 +- .../open-api/table-open-api.controller.ts | 25 + .../open-api/table-open-api.service.spec.ts | 195 ++- .../table/open-api/table-open-api.service.ts | 119 +- ...able-mutation-cache-invalidator.service.ts | 5 +- .../features/table/table-duplicate.service.ts | 316 +++- .../src/features/table/table-index.service.ts | 45 +- .../src/features/table/table.service.spec.ts | 52 + .../src/features/table/table.service.ts | 43 +- .../listener/table-trash.listener.spec.ts | 25 +- .../trash/listener/table-trash.listener.ts | 64 +- .../src/features/trash/trash.controller.ts | 3 +- .../src/features/trash/trash.service.ts | 112 +- .../features/trash/v2-record-trash.service.ts | 2 +- .../trash/v2-table-trash.service.spec.ts | 36 +- .../features/trash/v2-table-trash.service.ts | 53 +- .../undo-redo/open-api/undo-redo.service.ts | 8 +- .../operations/delete-fields.operation.ts | 24 +- .../operations/delete-records.operation.ts | 67 +- .../operations/delete-trash-routing.spec.ts | 32 +- .../operations/delete-view.operation.ts | 19 +- .../stack/undo-redo-operation.service.ts | 12 +- .../src/features/user/user.service.ts | 17 + .../features/v2/v2-container.service.spec.ts | 62 +- .../src/features/v2/v2-container.service.ts | 66 +- .../v2/v2-execution-context.factory.ts | 5 +- .../v2/v2-field-delete-compat.service.spec.ts | 59 +- .../v2/v2-field-delete-compat.service.ts | 5 +- .../features/v2/v2-record-history.service.ts | 9 +- ...v2-schema-operation-runner.service.spec.ts | 68 +- .../v2/v2-schema-operation-runner.service.ts | 45 + .../v2/v2-view-compat.service.spec.ts | 141 +- .../src/features/v2/v2-view-compat.service.ts | 76 +- .../src/features/v2/v2.controller.ts | 16 +- .../src/features/view/constant.ts | 6 +- .../src/features/view/model/form-view.dto.ts | 6 +- .../view/open-api/view-open-api-v2.service.ts | 4 +- .../view/open-api/view-open-api.service.ts | 45 +- .../view-data-safety-limit.service.spec.ts | 159 ++ .../view/view-data-safety-limit.service.ts | 178 ++ .../src/features/view/view.module.ts | 3 +- .../src/features/view/view.service.ts | 73 +- .../filter/global-exception.filter.spec.ts | 155 +- .../src/filter/global-exception.filter.ts | 126 +- .../src/global/byodb-routing.guard.spec.ts | 55 + .../data-db-client-manager.service.spec.ts | 270 +++- .../global/data-db-client-manager.service.ts | 300 +++- .../data-db-runtime-cache.service.spec.ts | 55 + .../global/data-db-runtime-cache.service.ts | 137 ++ .../src/global/data-db-runtime-error.spec.ts | 66 + .../src/global/data-db-runtime-error.ts | 207 +++ .../global/database-router.service.spec.ts | 89 + .../src/global/database-router.service.ts | 200 ++- .../src/global/global.module.ts | 6 + .../readonly/record-readonly.service.ts | 26 +- .../src/tracing-db-context.spec.ts | 94 ++ apps/nestjs-backend/src/tracing-db-context.ts | 149 ++ apps/nestjs-backend/src/tracing.ts | 22 + apps/nestjs-backend/src/types/cls.ts | 13 +- .../src/types/i18n.generated.ts | 175 +- .../test/attachment.e2e-spec.ts | 35 + .../test/base-duplicate.e2e-spec.ts | 799 ++++++++- .../test/base-sql-executor.e2e-spec.ts | 6 +- apps/nestjs-backend/test/base.e2e-spec.ts | 228 ++- .../byodb-space-storage-placement.e2e-spec.ts | 1431 +++++++++++++++++ .../caces/view-default-share-meta.ts | 6 +- .../test/dual-db-split.e2e-spec.ts | 4 +- .../test/field-duplicate.e2e-spec.ts | 61 + .../test/large-table-operations.e2e-spec.ts | 32 +- .../test/oauth-server.e2e-spec.ts | 46 + apps/nestjs-backend/test/oauth.e2e-spec.ts | 52 + apps/nestjs-backend/test/record.e2e-spec.ts | 8 +- .../test/space-owner-limit.e2e-spec.ts | 205 --- .../test/table-trash.e2e-spec.ts | 14 +- .../v2-schema-operation-runner.e2e-spec.ts | 166 +- apps/nestjs-backend/test/view.e2e-spec.ts | 6 + apps/nextjs-app/src/components/Metrics.tsx | 61 +- apps/nextjs-app/src/components/google-ads.tsx | 33 +- .../src/features/app/base-node/TablePage.tsx | 2 + .../src/features/app/base-node/helper.spec.ts | 53 + .../src/features/app/base-node/helper.ts | 27 +- .../base/base-side-bar/BaseNodeMore.tsx | 74 +- .../base/duplicate/DuplicateBaseModal.tsx | 165 +- .../duplicate/duplicateBaseProgress.spec.ts | 72 + .../base/duplicate/duplicateBaseProgress.ts | 99 ++ .../components/IntegrityV2Components.tsx | 2 +- .../app/blocks/share/view/ShareViewPage.tsx | 1 + .../app/blocks/space/NoSpacesPlaceholder.tsx | 34 +- .../space/component/BaseActionTrigger.tsx | 217 ++- .../data-db/ByodbSpaceCreateSection.spec.tsx | 152 ++ .../space/data-db/ByodbSpaceCreateSection.tsx | 215 +++ .../data-db/create-space-data-db.spec.ts | 41 + .../space/data-db/create-space-data-db.ts | 38 + .../app/blocks/space/data-db/index.ts | 3 + .../space/data-db/useByodbSpaceCreate.tsx | 89 + .../blocks/space/space-side-bar/SpaceList.tsx | 34 +- .../space/space-side-bar/SpaceSwitcher.tsx | 35 +- .../app/blocks/table-list/TableOperation.tsx | 68 +- .../blocks/trash/components/TableTrash.tsx | 2 +- .../view/form/components/FormEditorMain.tsx | 2 +- .../view/form/components/FormPreviewer.tsx | 2 +- .../blocks/view/grid/components/FieldMenu.tsx | 93 +- .../grid/components/SelectionStatistic.tsx | 85 +- .../view/grid/hooks/useSelectionOperation.ts | 12 +- .../blocks/view/grid/utils/selection.spec.ts | 48 + .../app/blocks/view/grid/utils/selection.ts | 20 + .../blocks/view/tool-bar/CalendarToolBar.tsx | 5 +- .../blocks/view/tool-bar/GalleryToolBar.tsx | 5 +- .../app/blocks/view/tool-bar/GridToolBar.tsx | 22 +- .../blocks/view/tool-bar/KanbanToolBar.tsx | 5 +- .../components/CalendarViewOperators.tsx | 30 +- .../components/GalleryViewOperators.tsx | 38 +- .../tool-bar/components/GridViewOperators.tsx | 5 +- .../components/KanbanViewOperators.tsx | 22 +- .../components/ScrollableToolbarGroup.tsx | 83 + .../components/ToolBarAddRecordButton.tsx | 25 + .../blocks/view/tool-bar/components/index.ts | 1 + .../FieldSetting.issue-t4599.spec.tsx | 77 + .../field-setting/FieldSetting.spec.tsx | 184 ++- .../components/field-setting/FieldSetting.tsx | 18 +- .../options/LinkOptions/SelectTable.tsx | 17 +- .../notifications/NotificationIcon.tsx | 3 +- .../notifications/NotificationItem.tsx | 6 +- .../notifications/NotificationsManage.tsx | 64 +- .../LinkNotification.tsx | 2 +- .../src/features/app/hooks/useBaseUsage.ts | 4 +- .../src/features/app/utils/download-url.ts | 32 + .../src/features/auth/components/SignForm.tsx | 14 +- apps/nextjs-app/src/lib/database-url.spec.ts | 6 - apps/nextjs-app/src/lib/database-url.ts | 1 - apps/nextjs-app/src/lib/server-env.ts | 1 + apps/nextjs-app/src/lib/withEnv.ts | 1 + .../common-i18n/src/locales/de/common.json | 8 +- packages/common-i18n/src/locales/de/sdk.json | 34 +- .../common-i18n/src/locales/de/space.json | 11 +- .../common-i18n/src/locales/de/table.json | 9 + .../common-i18n/src/locales/en/common.json | 9 +- packages/common-i18n/src/locales/en/sdk.json | 34 +- .../common-i18n/src/locales/en/space.json | 91 +- .../common-i18n/src/locales/en/table.json | 54 + .../common-i18n/src/locales/es/common.json | 8 +- packages/common-i18n/src/locales/es/sdk.json | 34 +- .../common-i18n/src/locales/es/space.json | 11 +- .../common-i18n/src/locales/es/table.json | 9 + .../common-i18n/src/locales/fr/common.json | 8 +- packages/common-i18n/src/locales/fr/sdk.json | 34 +- .../common-i18n/src/locales/fr/space.json | 11 +- .../common-i18n/src/locales/fr/table.json | 9 + .../common-i18n/src/locales/it/common.json | 8 +- packages/common-i18n/src/locales/it/sdk.json | 34 +- .../common-i18n/src/locales/it/space.json | 11 +- .../common-i18n/src/locales/it/table.json | 9 + .../common-i18n/src/locales/ja/common.json | 8 +- packages/common-i18n/src/locales/ja/sdk.json | 34 +- .../common-i18n/src/locales/ja/space.json | 11 +- .../common-i18n/src/locales/ja/table.json | 9 + .../common-i18n/src/locales/ru/common.json | 8 +- packages/common-i18n/src/locales/ru/sdk.json | 34 +- .../common-i18n/src/locales/ru/space.json | 11 +- .../common-i18n/src/locales/ru/table.json | 9 + .../common-i18n/src/locales/tr/common.json | 8 +- packages/common-i18n/src/locales/tr/sdk.json | 34 +- .../common-i18n/src/locales/tr/space.json | 11 +- .../common-i18n/src/locales/tr/table.json | 9 + .../common-i18n/src/locales/uk/common.json | 8 +- packages/common-i18n/src/locales/uk/sdk.json | 34 +- .../common-i18n/src/locales/uk/space.json | 11 +- .../common-i18n/src/locales/uk/table.json | 9 + .../common-i18n/src/locales/zh/common.json | 9 +- packages/common-i18n/src/locales/zh/sdk.json | 34 +- .../common-i18n/src/locales/zh/space.json | 90 +- .../common-i18n/src/locales/zh/table.json | 54 + packages/core/src/auth/oauth.ts | 28 + packages/core/src/formula/visitor.spec.ts | 15 + packages/core/src/formula/visitor.ts | 24 + .../models/notification/notification.enum.ts | 1 + packages/core/src/models/view/view.schema.ts | 11 +- packages/db-data-prisma/package.json | 2 +- .../migration.sql | 13 +- packages/db-data-prisma/prisma/schema.prisma | 2 +- .../scripts/run-prisma-command.mjs | 5 +- packages/db-data-prisma/src/database-url.ts | 19 +- .../migration.sql | 13 + .../migration.sql | 16 + .../prisma/postgres/schema.prisma | 1 + packages/db-main-prisma/src/database-url.ts | 11 +- packages/openapi/src/admin/setting/get.ts | 9 +- .../openapi/src/admin/setting/key.enum.ts | 1 - packages/openapi/src/admin/setting/update.ts | 28 +- .../src/base/cross-space-affected-field.ts | 9 + packages/openapi/src/base/duplicate-check.ts | 53 + packages/openapi/src/base/duplicate.ts | 159 ++ packages/openapi/src/base/export.ts | 165 ++ packages/openapi/src/base/import.ts | 2 +- packages/openapi/src/base/index.ts | 3 + packages/openapi/src/base/move-check.ts | 46 + packages/openapi/src/base/move.ts | 20 +- packages/openapi/src/field/duplicate-check.ts | 46 + packages/openapi/src/field/index.ts | 1 + packages/openapi/src/notification/index.ts | 1 + .../notification/send-admin-notification.ts | 26 + packages/openapi/src/space/create.ts | 3 +- packages/openapi/src/space/data-db.spec.ts | 24 + packages/openapi/src/space/data-db.ts | 105 ++ packages/openapi/src/table/duplicate-check.ts | 46 + packages/openapi/src/table/index.ts | 1 + packages/openapi/src/trash/restore.ts | 9 +- .../CollaboratorWithHoverCard.tsx | 27 +- .../components/expand-record/Modal.spec.tsx | 33 + .../src/components/expand-record/Modal.tsx | 3 +- .../component/FilterUserSelect.tsx | 82 +- .../component/base/BaseMultipleSelect.tsx | 2 +- .../component/base/BaseSingleSelect.tsx | 2 +- .../hooks/use-grid-async-records.spec.tsx | 79 + .../hooks/use-grid-async-records.ts | 19 + .../hooks/use-grid-columns.tsx | 5 +- .../sdk/src/context/app/queryClient.spec.ts | 76 + packages/sdk/src/context/app/queryClient.tsx | 29 +- .../use-instances/use-instances.spec.tsx | 76 + .../src/context/use-instances/useInstances.ts | 87 +- packages/ui-lib/src/shadcn/global.shadcn.css | 4 +- packages/ui-lib/src/shadcn/ui/sheet.tsx | 4 +- .../PostgresSchemaOperationRepository.spec.ts | 5 +- .../PostgresSchemaOperationRepository.ts | 7 +- .../PostgresTableRepository.helpers.spec.ts | 10 +- .../repositories/PostgresTableRepository.ts | 27 +- .../PostgresTableRowLimitPlugin.ts | 3 +- .../visitors/TableWhereVisitor.spec.ts | 22 +- .../visitors/TableWhereVisitor.ts | 4 + .../commands/CreateRecordHandler.db.spec.ts | 108 ++ .../ComputedBeforeImageFromChanges.ts | 77 + .../ComputedFieldBackfillService.spec.ts | 86 +- .../computed/ComputedFieldBackfillService.ts | 186 ++- .../record/computed/ComputedFieldUpdater.ts | 132 +- .../record/computed/ComputedUpdatePlanner.ts | 27 +- .../record/computed/FieldDependencyGraph.ts | 105 +- .../computed/UpdateFromSelectBuilder.ts | 57 +- .../ComputedBeforeImageFromChanges.spec.ts | 93 ++ .../__tests__/ComputedFieldUpdater.spec.ts | 133 ++ .../__tests__/ComputedUpdatePlanner.spec.ts | 35 + .../FieldDependencyGraph.pglite.spec.ts | 39 + .../__tests__/UpdateFromSelectBuilder.spec.ts | 81 +- .../computed/outbox/ComputedUpdateOutbox.ts | 25 + .../computed/outbox/IComputedUpdateOutbox.ts | 24 + .../strategies/HybridWithOutboxStrategy.ts | 21 +- .../ComputedUpdatePollingService.spec.ts | 37 + .../worker/ComputedUpdatePollingService.ts | 16 +- .../worker/ComputedUpdateWorker.spec.ts | 143 +- .../computed/worker/ComputedUpdateWorker.ts | 210 ++- .../ComputedFieldSelectExpressionVisitor.ts | 11 + .../ComputedTableRecordQueryBuilder.spec.ts | 275 +++- .../ComputedTableRecordQueryBuilder.ts | 424 ++++- .../computed/FieldReferenceSqlVisitor.ts | 13 +- .../computed/SameTableBatchQueryBuilder.ts | 33 +- .../update/BatchUpdateSqlBuilder.ts | 64 +- ...sTableRecordQueryRepository.pglite.spec.ts | 57 +- .../PostgresTableRecordQueryRepository.ts | 19 +- ...stgresTableRecordRepository.delete.spec.ts | 6 +- .../PostgresTableRecordRepository.ts | 68 +- ...stgresTableRecordRepository.update.spec.ts | 141 +- .../PostgresTableSchemaRepository.spec.ts | 131 +- .../PostgresTableSchemaRepository.ts | 100 +- .../src/schema/rules/checker/SchemaChecker.ts | 10 +- .../src/schema/rules/core/ISchemaRule.ts | 9 +- .../rules/core/SchemaStatementAccessPolicy.ts | 81 + .../src/schema/rules/core/index.ts | 7 + .../schema/rules/field/ColumnExistsRule.ts | 8 +- .../rules/field/ColumnUniqueConstraintRule.ts | 385 ++++- .../rules/field/FieldSchemaRulesFactory.ts | 14 +- .../src/schema/rules/field/FkColumnRule.ts | 9 +- .../src/schema/rules/field/ForeignKeyRule.ts | 215 ++- .../rules/field/GeneratedColumnMetaRule.ts | 14 +- .../schema/rules/field/JunctionTableRule.ts | 410 ++++- .../rules/field/LinkSymmetricFieldRule.ts | 204 ++- .../schema/rules/field/LinkValueColumnRule.ts | 8 +- .../rules/field/NotNullConstraintRule.ts | 118 +- .../src/schema/rules/field/OrderColumnRule.ts | 8 +- .../rules/field/SchemaRules.pglite.spec.ts | 676 +++++++- .../rules/field/SelectOptionsMetaRule.ts | 94 +- .../src/schema/rules/field/UniqueIndexRule.ts | 276 +++- .../schema/rules/helpers/StatementBuilders.ts | 216 +-- .../src/schema/rules/helpers/index.ts | 2 + .../src/schema/rules/index.ts | 4 + .../schema/rules/planner/SchemaRulePlanner.ts | 5 + .../schema/rules/repairer/SchemaRepairer.ts | 5 +- .../schema/rules/table/SystemTableRules.ts | 95 +- .../visitors/FieldTypeConversionVisitor.ts | 157 +- ...tgresTableSchemaFieldCreateVisitor.spec.ts | 5 + .../PostgresTableSchemaFieldCreateVisitor.ts | 42 +- .../visitors/TableSchemaUpdateVisitor.ts | 56 +- .../FieldTypeConversionVisitor.spec.ts | 53 +- .../src/shared/db.spec.ts | 31 +- .../src/shared/db.ts | 16 +- .../src/shared/undoCapture.ts | 10 +- .../src/shared/undoCaptureGlobalsSql.ts | 12 +- .../src/benchmarkTableDataSafetyLimits.ts | 8 + .../benchmark-node/src/create-table.bench.ts | 5 +- .../src/get-table-by-id.bench.ts | 5 +- packages/v2/container-node/src/index.ts | 3 +- .../RecordsBatchCreatedRealtimeProjection.ts | 2 + .../projections/runRealtimeTasks.ts | 1 + .../DotTeaExportFieldNormalizer.spec.ts | 140 ++ .../services/DotTeaExportFieldNormalizer.ts | 119 ++ .../FieldUpdateSideEffectService.spec.ts | 18 +- .../services/FieldUpdateSideEffectService.ts | 4 +- .../SchemaOperationRunnerService.spec.ts | 15 +- .../services/SchemaOperationRunnerService.ts | 23 +- .../services/TableCreationService.spec.ts | 37 +- .../services/TableCreationService.ts | 4 +- ...ableDataSafetyLimitFieldOperationPlugin.ts | 16 +- ...ataSafetyLimitTableOperationPlugin.spec.ts | 69 +- ...ableDataSafetyLimitTableOperationPlugin.ts | 114 +- ...DataSafetyLimitViewOperationPlugin.spec.ts | 218 +++ ...TableDataSafetyLimitViewOperationPlugin.ts | 204 +++ ...bleSchemaOperationLifecycleService.spec.ts | 32 + .../TableSchemaOperationLifecycleService.ts | 23 + .../TableSchemaOperationRepairHandler.spec.ts | 58 + .../TableSchemaOperationRepairHandler.ts | 16 +- .../services/TableUpdateFlow.spec.ts | 30 +- .../application/services/TableUpdateFlow.ts | 62 +- .../services/ViewOperationPluginRunner.ts | 161 ++ .../src/commands/CreateRecordsHandler.spec.ts | 45 + .../core/src/commands/CreateTableHandler.ts | 1 + .../core/src/commands/CreateTablesHandler.ts | 1 + .../src/commands/DeleteTableHandler.spec.ts | 12 +- .../core/src/commands/DeleteTableHandler.ts | 44 +- .../src/commands/DuplicateBaseCommand.spec.ts | 26 + .../core/src/commands/DuplicateBaseCommand.ts | 111 ++ .../src/commands/DuplicateBaseHandler.spec.ts | 80 + .../core/src/commands/DuplicateBaseHandler.ts | 507 ++++++ .../src/commands/DuplicateTableHandler.ts | 1 + .../v2/core/src/commands/ImportCsvHandler.ts | 1 + .../ImportDotTeaStructureHandler.spec.ts | 73 +- .../commands/ImportDotTeaStructureHandler.ts | 29 +- .../src/commands/ImportRecordsHandler.spec.ts | 4 +- .../core/src/commands/ImportRecordsHandler.ts | 3 +- .../v2/core/src/di/registerCoreServices.ts | 22 + .../src/di/registerViewOperationPlugin.ts | 68 + .../domain/shared/TableDataSafetyLimits.ts | 2 +- .../src/domain/table/IdValueObjects.spec.ts | 7 + packages/v2/core/src/domain/table/Table.ts | 23 +- .../core/src/domain/table/fields/FieldId.ts | 5 +- .../types/ConditionalLookupField.spec.ts | 57 + .../table/fields/types/FieldCondition.ts | 119 +- .../types/SelectFieldOptionWriteConfig.ts | 31 + .../UpdateMultipleSelectOptionsSpec.ts | 10 +- .../UpdateSingleSelectOptionsSpec.ts | 10 +- packages/v2/core/src/index.ts | 7 + .../v2/core/src/ports/TableOperationPlugin.ts | 4 + packages/v2/core/src/ports/TableRepository.ts | 2 +- .../core/src/ports/TableSchemaRepository.ts | 5 +- .../v2/core/src/ports/ViewOperationPlugin.ts | 92 ++ packages/v2/core/src/ports/tokens.ts | 2 + .../e2e/src/computed-optimization.e2e.spec.ts | 99 ++ ...nditionalFieldDirtyPropagation.e2e.spec.ts | 110 ++ packages/v2/e2e/src/deleteTable.e2e.spec.ts | 467 +++++- .../longText/conversion/to-date.spec.ts | 21 +- .../src/FormulaSqlPgExpressionBuilder.ts | 11 +- .../src/LookupArrayNormalization.spec.ts | 7 + .../v2/formula-sql-pg/src/PgSqlHelpers.ts | 8 +- .../__snapshots__/ArrayFunctions.spec.ts.snap | 68 +- .../BinaryOperators.spec.ts.snap | 48 +- .../__snapshots__/DateFunctions.spec.ts.snap | 400 ++--- .../IfBranchNormalization.spec.ts.snap | 4 +- .../LogicalFunctions.spec.ts.snap | 88 +- .../NumericFunctions.spec.ts.snap | 150 +- .../__snapshots__/TextFunctions.spec.ts.snap | 84 +- packages/v2/formula-sql-pg/vitest.config.ts | 1 + packages/v2/postgres-schema/src/v1/types.ts | 10 + .../components/SharePopover.tsx | 2 +- pnpm-lock.yaml | 15 +- scripts/db-migrate.mjs | 2 +- 473 files changed, 29367 insertions(+), 4385 deletions(-) create mode 100644 apps/nestjs-backend/src/distributed-lock/distributed-lock.module.ts create mode 100644 apps/nestjs-backend/src/distributed-lock/distributed-lock.service.spec.ts create mode 100644 apps/nestjs-backend/src/distributed-lock/distributed-lock.service.ts create mode 100644 apps/nestjs-backend/src/distributed-lock/index.ts create mode 100644 apps/nestjs-backend/src/features/base/base-controller.spec.ts create mode 100644 apps/nestjs-backend/src/features/base/base-duplicate-v2.service.ts create mode 100644 apps/nestjs-backend/src/features/base/base-export-v2.service.ts create mode 100644 apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts create mode 100644 apps/nestjs-backend/src/features/base/cross-space-detection.util.spec.ts create mode 100644 apps/nestjs-backend/src/features/base/cross-space-detection.util.ts create mode 100644 apps/nestjs-backend/src/features/oauth/oauth-app-init.service.spec.ts create mode 100644 apps/nestjs-backend/src/features/oauth/oauth-app-init.service.ts create mode 100644 apps/nestjs-backend/src/features/space/data-db-internal-schema.ts create mode 100644 apps/nestjs-backend/src/features/space/data-db-migration.service.spec.ts create mode 100644 apps/nestjs-backend/src/features/space/data-db-migration.service.ts create mode 100644 apps/nestjs-backend/src/features/view/view-data-safety-limit.service.spec.ts create mode 100644 apps/nestjs-backend/src/features/view/view-data-safety-limit.service.ts create mode 100644 apps/nestjs-backend/src/global/byodb-routing.guard.spec.ts create mode 100644 apps/nestjs-backend/src/global/data-db-runtime-cache.service.spec.ts create mode 100644 apps/nestjs-backend/src/global/data-db-runtime-cache.service.ts create mode 100644 apps/nestjs-backend/src/global/data-db-runtime-error.spec.ts create mode 100644 apps/nestjs-backend/src/global/data-db-runtime-error.ts create mode 100644 apps/nestjs-backend/src/global/database-router.service.spec.ts create mode 100644 apps/nestjs-backend/src/tracing-db-context.spec.ts create mode 100644 apps/nestjs-backend/src/tracing-db-context.ts create mode 100644 apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts delete mode 100644 apps/nestjs-backend/test/space-owner-limit.e2e-spec.ts create mode 100644 apps/nextjs-app/src/features/app/base-node/helper.spec.ts create mode 100644 apps/nextjs-app/src/features/app/blocks/base/duplicate/duplicateBaseProgress.spec.ts create mode 100644 apps/nextjs-app/src/features/app/blocks/base/duplicate/duplicateBaseProgress.ts create mode 100644 apps/nextjs-app/src/features/app/blocks/space/data-db/ByodbSpaceCreateSection.spec.tsx create mode 100644 apps/nextjs-app/src/features/app/blocks/space/data-db/ByodbSpaceCreateSection.tsx create mode 100644 apps/nextjs-app/src/features/app/blocks/space/data-db/create-space-data-db.spec.ts create mode 100644 apps/nextjs-app/src/features/app/blocks/space/data-db/create-space-data-db.ts create mode 100644 apps/nextjs-app/src/features/app/blocks/space/data-db/index.ts create mode 100644 apps/nextjs-app/src/features/app/blocks/space/data-db/useByodbSpaceCreate.tsx create mode 100644 apps/nextjs-app/src/features/app/blocks/view/tool-bar/components/ScrollableToolbarGroup.tsx create mode 100644 apps/nextjs-app/src/features/app/blocks/view/tool-bar/components/ToolBarAddRecordButton.tsx create mode 100644 apps/nextjs-app/src/features/app/components/field-setting/FieldSetting.issue-t4599.spec.tsx create mode 100644 apps/nextjs-app/src/features/app/utils/download-url.ts create mode 100644 packages/db-main-prisma/prisma/postgres/migrations/20260507075100_add_data_db_internal_schema/migration.sql create mode 100644 packages/db-main-prisma/prisma/postgres/migrations/20260521000000_add_computed_outbox_claim_indexes/migration.sql create mode 100644 packages/openapi/src/base/cross-space-affected-field.ts create mode 100644 packages/openapi/src/base/duplicate-check.ts create mode 100644 packages/openapi/src/base/move-check.ts create mode 100644 packages/openapi/src/field/duplicate-check.ts create mode 100644 packages/openapi/src/notification/send-admin-notification.ts create mode 100644 packages/openapi/src/table/duplicate-check.ts create mode 100644 packages/sdk/src/components/expand-record/Modal.spec.tsx create mode 100644 packages/sdk/src/components/grid-enhancements/hooks/use-grid-async-records.spec.tsx create mode 100644 packages/v2/adapter-table-repository-postgres/src/record/computed/ComputedBeforeImageFromChanges.ts create mode 100644 packages/v2/adapter-table-repository-postgres/src/record/computed/__tests__/ComputedBeforeImageFromChanges.spec.ts create mode 100644 packages/v2/adapter-table-repository-postgres/src/schema/rules/core/SchemaStatementAccessPolicy.ts create mode 100644 packages/v2/benchmark-node/src/benchmarkTableDataSafetyLimits.ts create mode 100644 packages/v2/core/src/application/services/DotTeaExportFieldNormalizer.spec.ts create mode 100644 packages/v2/core/src/application/services/DotTeaExportFieldNormalizer.ts create mode 100644 packages/v2/core/src/application/services/TableDataSafetyLimitViewOperationPlugin.spec.ts create mode 100644 packages/v2/core/src/application/services/TableDataSafetyLimitViewOperationPlugin.ts create mode 100644 packages/v2/core/src/application/services/ViewOperationPluginRunner.ts create mode 100644 packages/v2/core/src/commands/DuplicateBaseCommand.spec.ts create mode 100644 packages/v2/core/src/commands/DuplicateBaseCommand.ts create mode 100644 packages/v2/core/src/commands/DuplicateBaseHandler.spec.ts create mode 100644 packages/v2/core/src/commands/DuplicateBaseHandler.ts create mode 100644 packages/v2/core/src/di/registerViewOperationPlugin.ts create mode 100644 packages/v2/core/src/ports/ViewOperationPlugin.ts diff --git a/apps/nestjs-backend/package.json b/apps/nestjs-backend/package.json index 6759115455..9d69112059 100644 --- a/apps/nestjs-backend/package.json +++ b/apps/nestjs-backend/package.json @@ -194,6 +194,7 @@ "@teable/v2-contract-http-openapi": "workspace:*", "@teable/v2-core": "workspace:*", "@teable/v2-di": "workspace:*", + "@teable/v2-dottea": "workspace:*", "@teable/v2-import": "workspace:*", "@valibot/to-json-schema": "1.3.0", "ai": "6.0.169", diff --git a/apps/nestjs-backend/src/configs/env.validation.schema.spec.ts b/apps/nestjs-backend/src/configs/env.validation.schema.spec.ts index 37fe17fecd..34828c88e1 100644 --- a/apps/nestjs-backend/src/configs/env.validation.schema.spec.ts +++ b/apps/nestjs-backend/src/configs/env.validation.schema.spec.ts @@ -18,19 +18,16 @@ describe('envValidationSchema', () => { expect(value.PRISMA_DATABASE_URL).toContain('/teable'); }); - it('accepts split meta/data env without the legacy alias', () => { + it('accepts split meta env without the legacy alias', () => { const { error, value } = envValidationSchema.validate( createEnv({ PRISMA_META_DATABASE_URL: 'postgresql://teable:teable@127.0.0.1:5432/teable-meta?schema=public', - PRISMA_DATA_DATABASE_URL: - 'postgresql://teable:teable@127.0.0.1:5432/teable-data?schema=public', }) ); expect(error).toBeUndefined(); expect(value.PRISMA_META_DATABASE_URL).toContain('/teable-meta'); - expect(value.PRISMA_DATA_DATABASE_URL).toContain('/teable-data'); }); it('accepts DATABASE_URL as the last-resort meta fallback', () => { diff --git a/apps/nestjs-backend/src/configs/env.validation.schema.ts b/apps/nestjs-backend/src/configs/env.validation.schema.ts index acbadb547c..f17902d90e 100644 --- a/apps/nestjs-backend/src/configs/env.validation.schema.ts +++ b/apps/nestjs-backend/src/configs/env.validation.schema.ts @@ -15,7 +15,6 @@ export const envValidationSchema = Joi.object({ // database_url PRISMA_DATABASE_URL: Joi.string(), PRISMA_META_DATABASE_URL: Joi.string(), - PRISMA_DATA_DATABASE_URL: Joi.string(), DATABASE_URL: Joi.string(), STORAGE_PREFIX: Joi.string().uri().optional(), diff --git a/apps/nestjs-backend/src/configs/threshold.config.ts b/apps/nestjs-backend/src/configs/threshold.config.ts index 7b9d15f351..83a550b690 100644 --- a/apps/nestjs-backend/src/configs/threshold.config.ts +++ b/apps/nestjs-backend/src/configs/threshold.config.ts @@ -36,7 +36,7 @@ export const thresholdConfig = registerAs('threshold', () => ({ jitter: Number(process.env.BACKEND_DB_DEADLOCK_JITTER ?? 1.0), }, baseNodeMaxFolderDepth: Number(process.env.BASE_NODE_MAX_FOLDER_DEPTH ?? 2), - maxOwnedSpaceCount: Number(process.env.MAX_SPACE_OWNER_COUNT ?? 10), + maxFreeOwnedSpaceCount: Number(process.env.MAX_FREE_SPACE_OWNER_COUNT ?? 2), changeEmailSendCodeMailRate: Number(process.env.BACKEND_CHANGE_EMAIL_SEND_CODE_MAIL_RATE ?? 30), resetPasswordSendMailRate: Number(process.env.BACKEND_RESET_PASSWORD_SEND_MAIL_RATE ?? 30), signupVerificationSendCodeMailRate: Number( diff --git a/apps/nestjs-backend/src/db-provider/duplicate-table/abstract.ts b/apps/nestjs-backend/src/db-provider/duplicate-table/abstract.ts index 97a0f4a275..393de53e42 100644 --- a/apps/nestjs-backend/src/db-provider/duplicate-table/abstract.ts +++ b/apps/nestjs-backend/src/db-provider/duplicate-table/abstract.ts @@ -8,6 +8,10 @@ export abstract class DuplicateTableQueryAbstract { targetTable: string, newColumns: string[], oldColumns: string[], - crossBaseLinkDbFieldNames: { dbFieldName: string; isMultipleCellValue: boolean }[] + crossBaseLinkDbFieldNames: { dbFieldName: string; isMultipleCellValue: boolean }[], + range?: { + minAutoNumberExclusive?: number; + maxAutoNumberInclusive?: number; + } ): Knex.QueryBuilder; } diff --git a/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-query.postgres.ts b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-query.postgres.ts index d3a7426e74..1916be6bf1 100644 --- a/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-query.postgres.ts +++ b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-query.postgres.ts @@ -13,7 +13,11 @@ export class DuplicateTableQueryPostgres extends DuplicateTableQueryAbstract { targetTable: string, newColumns: string[], oldColumns: string[], - crossBaseLinkDbFieldNames: { dbFieldName: string; isMultipleCellValue: boolean }[] + crossBaseLinkDbFieldNames: { dbFieldName: string; isMultipleCellValue: boolean }[], + range?: { + minAutoNumberExclusive?: number; + maxAutoNumberInclusive?: number; + } ) { const newColumnList = newColumns.map((col) => `"${col}"`).join(', '); const oldColumnList = oldColumns @@ -37,9 +41,20 @@ export class DuplicateTableQueryPostgres extends DuplicateTableQueryAbstract { return `"${col}"`; }) .join(', '); + const whereClauses: string[] = []; + const whereBindings: unknown[] = []; + if (range?.minAutoNumberExclusive != null) { + whereClauses.push('"__auto_number" > ?'); + whereBindings.push(range.minAutoNumberExclusive); + } + if (range?.maxAutoNumberInclusive != null) { + whereClauses.push('"__auto_number" <= ?'); + whereBindings.push(range.maxAutoNumberInclusive); + } + const whereSql = whereClauses.length ? ` WHERE ${whereClauses.join(' AND ')}` : ''; return this.knex.raw( - `INSERT INTO ?? (${newColumnList}) SELECT ${oldColumnList} FROM ?? ORDER BY __auto_number`, - [targetTable, sourceTable] + `INSERT INTO ?? (${newColumnList}) SELECT ${oldColumnList} FROM ??${whereSql} ORDER BY "__auto_number"`, + [targetTable, sourceTable, ...whereBindings] ); } } diff --git a/apps/nestjs-backend/src/distributed-lock/distributed-lock.module.ts b/apps/nestjs-backend/src/distributed-lock/distributed-lock.module.ts new file mode 100644 index 0000000000..a5fc16cfd1 --- /dev/null +++ b/apps/nestjs-backend/src/distributed-lock/distributed-lock.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { DistributedLockService } from './distributed-lock.service'; + +/** + * Provides {@link DistributedLockService}. Import it into any feature module + * that needs to guard startup seeding or other once-per-deployment work. + */ +@Module({ + providers: [DistributedLockService], + exports: [DistributedLockService], +}) +export class DistributedLockModule {} diff --git a/apps/nestjs-backend/src/distributed-lock/distributed-lock.service.spec.ts b/apps/nestjs-backend/src/distributed-lock/distributed-lock.service.spec.ts new file mode 100644 index 0000000000..48948c3e85 --- /dev/null +++ b/apps/nestjs-backend/src/distributed-lock/distributed-lock.service.spec.ts @@ -0,0 +1,100 @@ +import type { ConfigService } from '@nestjs/config'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { CacheService } from '../cache/cache.service'; +import { DistributedLockService } from './distributed-lock.service'; + +describe('DistributedLockService', () => { + const cache = { setnx: vi.fn(), get: vi.fn(), del: vi.fn() }; + const config = { get: vi.fn() }; + const newService = () => + new DistributedLockService( + cache as unknown as CacheService, + config as unknown as ConfigService + ); + + afterEach(() => { + vi.clearAllMocks(); + }); + + describe('with Redis', () => { + const useRedis = () => config.get.mockReturnValue({ provider: 'redis' }); + + it('runs the task when the lock is acquired', async () => { + useRedis(); + cache.setnx.mockResolvedValue(true); + const task = vi.fn().mockResolvedValue(undefined); + + const ran = await newService().runExclusive('seed', 60, task); + + expect(ran).toBe(true); + expect(cache.setnx).toHaveBeenCalledWith('lock:seed', expect.any(String), 60); + expect(task).toHaveBeenCalledOnce(); + }); + + it('skips the task when another instance holds the lock', async () => { + useRedis(); + cache.setnx.mockResolvedValue(false); + const task = vi.fn(); + + const ran = await newService().runExclusive('seed', 60, task); + + expect(ran).toBe(false); + expect(task).not.toHaveBeenCalled(); + }); + + it('releases the lock it owns after the task', async () => { + useRedis(); + cache.setnx.mockResolvedValue(true); + // Mirror Redis: `get` returns the value `setnx` stored. + cache.get.mockImplementation(async () => cache.setnx.mock.calls[0]?.[1]); + + await newService().runExclusive('seed', 60, vi.fn().mockResolvedValue(undefined)); + + expect(cache.del).toHaveBeenCalledWith('lock:seed'); + }); + + it('releases the lock even when the task throws', async () => { + useRedis(); + cache.setnx.mockResolvedValue(true); + cache.get.mockImplementation(async () => cache.setnx.mock.calls[0]?.[1]); + const task = vi.fn().mockRejectedValue(new Error('boom')); + + await expect(newService().runExclusive('seed', 60, task)).rejects.toThrow('boom'); + expect(cache.del).toHaveBeenCalledWith('lock:seed'); + }); + + it('does not release a lock owned by another instance', async () => { + useRedis(); + cache.setnx.mockResolvedValue(true); + cache.get.mockResolvedValue('another-instance'); + + await newService().runExclusive('seed', 60, vi.fn().mockResolvedValue(undefined)); + + expect(cache.del).not.toHaveBeenCalled(); + }); + + it('runs the task anyway when acquiring the lock errors', async () => { + useRedis(); + cache.setnx.mockRejectedValue(new Error('redis down')); + const task = vi.fn().mockResolvedValue(undefined); + + const ran = await newService().runExclusive('seed', 60, task); + + expect(ran).toBe(true); + expect(task).toHaveBeenCalledOnce(); + }); + }); + + describe('without Redis', () => { + it('runs the task without acquiring a lock', async () => { + config.get.mockReturnValue({ provider: 'memory' }); + const task = vi.fn().mockResolvedValue(undefined); + + const ran = await newService().runExclusive('seed', 60, task); + + expect(ran).toBe(true); + expect(cache.setnx).not.toHaveBeenCalled(); + expect(task).toHaveBeenCalledOnce(); + }); + }); +}); diff --git a/apps/nestjs-backend/src/distributed-lock/distributed-lock.service.ts b/apps/nestjs-backend/src/distributed-lock/distributed-lock.service.ts new file mode 100644 index 0000000000..8194b4d59b --- /dev/null +++ b/apps/nestjs-backend/src/distributed-lock/distributed-lock.service.ts @@ -0,0 +1,83 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { CacheService } from '../cache/cache.service'; +import type { ICacheConfig } from '../configs/cache.config'; + +/** + * Best-effort distributed lock backed by Redis (`SET NX`). + * + * Lets a caller run a critical section on exactly one instance across a + * multi-pod deployment. Without Redis there is no shared store, so the lock + * degrades to a no-op and every instance proceeds — callers must therefore + * keep the guarded work idempotent. + */ +@Injectable() +export class DistributedLockService { + private readonly logger = new Logger(DistributedLockService.name); + + /** Unique per process — identifies the locks this instance owns. */ + private readonly owner = `${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`; + + constructor( + private readonly cacheService: CacheService, + private readonly configService: ConfigService + ) {} + + /** + * Run `task` while holding the lock named `name`, so only one instance runs + * it at a time. If another instance holds the lock, `task` is skipped. The + * lock is released afterwards and also auto-expires after `ttlSeconds`. + * + * @returns `true` if `task` ran, `false` if it was skipped. + */ + async runExclusive( + name: string, + ttlSeconds: number, + task: () => Promise + ): Promise { + const key = `lock:${name}` as const; + + if (!(await this.acquire(key, ttlSeconds))) { + this.logger.debug(`Lock "${name}" held by another instance, skipping`); + return false; + } + + try { + await task(); + } finally { + await this.release(key); + } + return true; + } + + private get usesRedis(): boolean { + return this.configService.get('cache')?.provider === 'redis'; + } + + private async acquire(key: `lock:${string}`, ttlSeconds: number): Promise { + // No Redis — no shared store to lock against; let the caller proceed. + if (!this.usesRedis) { + return true; + } + try { + return await this.cacheService.setnx(key, this.owner, ttlSeconds); + } catch (error) { + this.logger.warn(`Failed to acquire lock "${key}", proceeding anyway`, error); + return true; + } + } + + private async release(key: `lock:${string}`): Promise { + if (!this.usesRedis) { + return; + } + try { + // Only release a lock this instance still owns. + if ((await this.cacheService.get(key)) === this.owner) { + await this.cacheService.del(key); + } + } catch (error) { + this.logger.warn(`Failed to release lock "${key}"`, error); + } + } +} diff --git a/apps/nestjs-backend/src/distributed-lock/index.ts b/apps/nestjs-backend/src/distributed-lock/index.ts new file mode 100644 index 0000000000..7173f51dab --- /dev/null +++ b/apps/nestjs-backend/src/distributed-lock/index.ts @@ -0,0 +1,2 @@ +export { DistributedLockModule } from './distributed-lock.module'; +export { DistributedLockService } from './distributed-lock.service'; diff --git a/apps/nestjs-backend/src/event-emitter/listeners/record-history.listener.ts b/apps/nestjs-backend/src/event-emitter/listeners/record-history.listener.ts index 4ccc0afa82..6a05805db3 100644 --- a/apps/nestjs-backend/src/event-emitter/listeners/record-history.listener.ts +++ b/apps/nestjs-backend/src/event-emitter/listeners/record-history.listener.ts @@ -3,15 +3,12 @@ import { Injectable } from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; import type { ISelectFieldOptions } from '@teable/core'; import { FieldType, generateRecordHistoryId } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { Field } from '@teable/db-main-prisma'; -import { Knex } from 'knex'; import { isEqual, isObject, isString } from 'lodash'; -import { InjectModel } from 'nest-knexjs'; import { BaseConfig, IBaseConfig } from '../../configs/base.config'; import { DataLoaderService } from '../../features/data-loader/data-loader.service'; import { rawField2FieldObj } from '../../features/field/model/factory'; -import { DATA_KNEX } from '../../global/knex/knex.module'; +import { DatabaseRouter } from '../../global/database-router.service'; import { EventEmitterService } from '../event-emitter.service'; import { Events, RecordUpdateEvent } from '../events'; @@ -21,10 +18,9 @@ const SELECT_FIELD_TYPE_SET = new Set([FieldType.SingleSelect, FieldType.Multipl @Injectable() export class RecordHistoryListener { constructor( - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly eventEmitterService: EventEmitterService, @BaseConfig() private readonly baseConfig: IBaseConfig, - @InjectModel(DATA_KNEX) private readonly knex: Knex, private readonly dataLoaderService: DataLoaderService ) {} @@ -129,9 +125,16 @@ export class RecordHistoryListener { }); if (recordHistoryList.length) { - const query = this.knex.insert(recordHistoryList).into('record_history').toQuery(); - - await this.dataPrismaService.txClient().$executeRawUnsafe(query); + const dataKnex = await this.databaseRouter.dataKnexForTable(tableId); + const dataDbUrl = await this.databaseRouter.getDataDatabaseUrlForTable(tableId); + const dataDbInternalSchema = new URL(dataDbUrl).searchParams.get('schema') || 'public'; + const query = dataKnex + .withSchema(dataDbInternalSchema) + .insert(recordHistoryList) + .into('record_history') + .toQuery(); + + await this.databaseRouter.executeDataPrismaForTable(tableId, query); } } diff --git a/apps/nestjs-backend/src/features/access-token/access-token.service.spec.ts b/apps/nestjs-backend/src/features/access-token/access-token.service.spec.ts index b2de1034a3..176576d21c 100644 --- a/apps/nestjs-backend/src/features/access-token/access-token.service.spec.ts +++ b/apps/nestjs-backend/src/features/access-token/access-token.service.spec.ts @@ -5,6 +5,7 @@ import { Test } from '@nestjs/testing'; import { PrismaService } from '@teable/db-main-prisma'; import { mockDeep, mockReset } from 'vitest-mock-extended'; import { GlobalModule } from '../../global/global.module'; +import { PerformanceCacheService } from '../../performance-cache'; import { AccessTokenModel } from '../model/access-token'; import { AccessTokenModule } from './access-token.module'; import { AccessTokenService } from './access-token.service'; @@ -13,6 +14,7 @@ describe('AccessTokenService', () => { let accessTokenService: AccessTokenService; const prismaService = mockDeep(); const accessTokenModel = mockDeep(); + const performanceCacheService = mockDeep(); beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ @@ -22,6 +24,8 @@ describe('AccessTokenService', () => { .useValue(prismaService) .overrideProvider(AccessTokenModel) .useValue(accessTokenModel) + .overrideProvider(PerformanceCacheService) + .useValue(performanceCacheService) .compile(); accessTokenService = module.get(AccessTokenService); @@ -38,6 +42,7 @@ describe('AccessTokenService', () => { afterEach(() => { vitest.resetAllMocks(); mockReset(prismaService); + mockReset(performanceCacheService); }); it('should be defined', () => { @@ -57,6 +62,7 @@ describe('AccessTokenService', () => { sign, expiredTime, } as any); + prismaService.accessToken.updateMany.mockResolvedValue({ count: 1 } as any); // Call the validate method const result = await accessTokenService.validate({ accessTokenId, sign }); @@ -65,11 +71,72 @@ describe('AccessTokenService', () => { expect(result.userId).toEqual('user123'); expect(result.accessTokenId).toEqual(accessTokenId); - // Validate that accessToken.update was called with the correct arguments - expect(prismaService.txClient().accessToken.update).toHaveBeenCalledWith({ - where: { id: accessTokenId }, + // Validate that accessToken.updateMany was called with a throttled lastUsedTime update. + expect(prismaService.txClient().accessToken.updateMany).toHaveBeenCalledWith({ + where: { + id: accessTokenId, + OR: [{ lastUsedTime: null }, { lastUsedTime: { lt: expect.any(Date) } }], + }, data: { lastUsedTime: expect.any(String) }, // It updates lastUsedTime to current time }); + expect(performanceCacheService.del).not.toHaveBeenCalled(); + expect(prismaService.accessToken.findUnique).not.toHaveBeenCalled(); + }); + + it('should throw UnauthorizedException when cached token was deleted before lastUsedTime update', async () => { + const accessTokenId = '123'; + const sign = 'SIGN'; + + accessTokenModel.getAccessTokenRawById.mockResolvedValue({ + userId: 'user123', + id: accessTokenId, + sign, + expiredTime: new Date(Date.now() + 2000).toISOString(), + } as any); + prismaService.accessToken.updateMany.mockResolvedValue({ count: 0 } as any); + prismaService.accessToken.findUnique.mockResolvedValue(null); + + await expect(accessTokenService.validate({ accessTokenId, sign })).rejects.toThrowError( + new UnauthorizedException('token not found') + ); + expect(performanceCacheService.del).toHaveBeenCalled(); + }); + + it('should keep validating when lastUsedTime was refreshed by another request', async () => { + const accessTokenId = '123'; + const sign = 'SIGN'; + + accessTokenModel.getAccessTokenRawById.mockResolvedValue({ + userId: 'user123', + id: accessTokenId, + sign, + expiredTime: new Date(Date.now() + 2000).toISOString(), + } as any); + prismaService.accessToken.updateMany.mockResolvedValue({ count: 0 } as any); + prismaService.accessToken.findUnique.mockResolvedValue({ id: accessTokenId } as any); + + const result = await accessTokenService.validate({ accessTokenId, sign }); + + expect(result).toEqual({ userId: 'user123', accessTokenId }); + expect(performanceCacheService.del).not.toHaveBeenCalled(); + }); + + it('skips lastUsedTime update when it was refreshed recently', async () => { + const accessTokenId = '123'; + const sign = 'SIGN'; + + accessTokenModel.getAccessTokenRawById.mockResolvedValue({ + userId: 'user123', + id: accessTokenId, + sign, + expiredTime: new Date(Date.now() + 2000).toISOString(), + lastUsedTime: new Date().toISOString(), + } as any); + + const result = await accessTokenService.validate({ accessTokenId, sign }); + + expect(result.userId).toEqual('user123'); + expect(prismaService.txClient().accessToken.updateMany).not.toHaveBeenCalled(); }); it('should throw UnauthorizedException for invalid sign', async () => { @@ -90,8 +157,8 @@ describe('AccessTokenService', () => { new UnauthorizedException('sign error') ); - // Ensure accessToken.update is not called in this case - expect(prismaService.txClient().accessToken.update).not.toHaveBeenCalled(); + // Ensure accessToken.updateMany is not called in this case + expect(prismaService.txClient().accessToken.updateMany).not.toHaveBeenCalled(); }); it('should throw UnauthorizedException for expired token', async () => { @@ -113,8 +180,8 @@ describe('AccessTokenService', () => { new UnauthorizedException('token expired') ); - // Ensure accessToken.update is not called in this case - expect(prismaService.txClient().accessToken.update).not.toHaveBeenCalled(); + // Ensure accessToken.updateMany is not called in this case + expect(prismaService.txClient().accessToken.updateMany).not.toHaveBeenCalled(); }); }); }); diff --git a/apps/nestjs-backend/src/features/access-token/access-token.service.ts b/apps/nestjs-backend/src/features/access-token/access-token.service.ts index a8461cef62..515681b1b6 100644 --- a/apps/nestjs-backend/src/features/access-token/access-token.service.ts +++ b/apps/nestjs-backend/src/features/access-token/access-token.service.ts @@ -14,6 +14,16 @@ import type { IClsStore } from '../../types/cls'; import { AccessTokenModel } from '../model/access-token'; import { getAccessToken } from './access-token.encryptor'; +const lastUsedTimeUpdateIntervalMs = 5 * 60 * 1000; + +const shouldUpdateLastUsedTime = ( + lastUsedTime: Date | string | null | undefined, + now: Date +): boolean => { + if (!lastUsedTime) return true; + return now.getTime() - new Date(lastUsedTime).getTime() >= lastUsedTimeUpdateIntervalMs; +}; + @Injectable() export class AccessTokenService { constructor( @@ -74,10 +84,29 @@ export class AccessTokenService { ) { throw new UnauthorizedException('token expired'); } - await this.prismaService.accessToken.update({ - where: { id: accessTokenId }, - data: { lastUsedTime: new Date().toISOString() }, - }); + const now = new Date(); + if (shouldUpdateLastUsedTime(accessTokenEntity.lastUsedTime, now)) { + const updated = await this.prismaService.accessToken.updateMany({ + where: { + id: accessTokenId, + OR: [ + { lastUsedTime: null }, + { lastUsedTime: { lt: new Date(now.getTime() - lastUsedTimeUpdateIntervalMs) } }, + ], + }, + data: { lastUsedTime: now.toISOString() }, + }); + if (updated.count === 0) { + const currentToken = await this.prismaService.accessToken.findUnique({ + where: { id: accessTokenId }, + select: { id: true }, + }); + if (!currentToken) { + await this.performanceCacheService.del(generateAccessTokenCacheKey(accessTokenId)); + throw new UnauthorizedException('token not found'); + } + } + } return { userId: accessTokenEntity.userId, diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.service.spec.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.service.spec.ts index f2d68948f7..23632dc9d5 100644 --- a/apps/nestjs-backend/src/features/aggregation/aggregation.service.spec.ts +++ b/apps/nestjs-backend/src/features/aggregation/aggregation.service.spec.ts @@ -18,4 +18,55 @@ describe('AggregateService', () => { it('should be defined', () => { expect(service).toBeDefined(); }); + + it('should execute row count SQL through the table scoped data database client', async () => { + const queryRaw = vi.fn().mockResolvedValue([{ count: 7 }]); + const databaseRouter = { + queryDataPrismaForTable: queryRaw, + }; + const recordPermissionService = { + wrapView: vi.fn().mockResolvedValue({ + builder: {}, + }), + }; + const recordQueryBuilder = { + createRecordAggregateBuilder: vi.fn().mockResolvedValue({ + qb: { + toQuery: () => 'SELECT COUNT(*)::int AS count FROM "bse1"."tbl1"', + }, + alias: 'tbl1', + selectionMap: {}, + }), + }; + const service = new AggregationService( + {} as never, + {} as never, + {} as never, + databaseRouter as never, + { queryBuilder: vi.fn().mockReturnValue({}) } as never, + {} as never, + {} as never, + { get: vi.fn().mockReturnValue('usr1') } as never, + recordPermissionService as never, + recordQueryBuilder as never + ); + + const serviceInternals = service as unknown as { + fetchStatisticsParams: () => Promise; + getDbTableName: () => Promise; + }; + vi.spyOn(serviceInternals, 'fetchStatisticsParams').mockResolvedValue({ + statisticsData: {}, + fieldInstanceMap: {}, + }); + vi.spyOn(serviceInternals, 'getDbTableName').mockResolvedValue('bse1.tbl1'); + + const result = await service.performRowCount('tbl1', { viewId: 'viw1' }); + + expect(result.rowCount).toBe(7); + expect(queryRaw).toHaveBeenCalledWith( + 'tbl1', + 'SELECT COUNT(*)::int AS count FROM "bse1"."tbl1"' + ); + }); }); diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts index 5e971ca7c6..c6f1d8ec10 100644 --- a/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts +++ b/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts @@ -12,7 +12,6 @@ import { ViewType, } from '@teable/core'; import type { IGridColumnMeta, IFilter, IGroup, ISortItem } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { StatisticsFunc } from '@teable/openapi'; @@ -41,6 +40,8 @@ import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.confi import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import type { IDataPrismaQueryExecutor } from '../../global/database-router.service'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import type { IClsStore } from '../../types/cls'; import { convertValueToStringify, string2Hash } from '../../utils'; @@ -65,6 +66,7 @@ type IStatisticsData = { // so this is undefined unless the caller is paginating. sort?: ISortItem[]; }; + /** * Version 2 implementation of the aggregation service * This is a placeholder implementation that will be developed in the future @@ -77,7 +79,7 @@ export class AggregationService implements IAggregationService { private readonly recordService: RecordService, private readonly tableIndexService: TableIndexService, private readonly prisma: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @InjectModel(DATA_KNEX) private readonly knex: Knex, @InjectDbProvider() private readonly dbProvider: IDbProvider, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, @@ -85,6 +87,22 @@ export class AggregationService implements IAggregationService { private readonly recordPermissionService: RecordPermissionService, @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder ) {} + + private async queryDataPrisma( + tableId: string, + query: string, + ...values: unknown[] + ): Promise { + return await this.databaseRouter.queryDataPrismaForTable(tableId, query, ...values); + } + + private async withDataPrismaTransaction( + tableId: string, + fn: (prisma: IDataPrismaQueryExecutor) => Promise + ): Promise { + return await this.databaseRouter.dataPrismaTransactionForTable(tableId, fn); + } + /** * Perform aggregation operations on table data * @param params - Parameters for aggregation including tableId, field IDs, view settings, and search @@ -127,7 +145,7 @@ export class AggregationService implements IAggregationService { const isPaginated = take !== undefined; const baseSort = isPaginated ? [...(groupBy ?? []), ...(resolvedSort ?? [])] : undefined; const defaultOrderField = isPaginated - ? await this.recordService.getBasicOrderIndexField(dbTableName, withView?.viewId) + ? await this.recordService.getBasicOrderIndexField(tableId, dbTableName, withView?.viewId) : undefined; const rawAggregationData = await this.handleAggregation({ @@ -325,7 +343,7 @@ export class AggregationService implements IAggregationService { const aggSql = qb.toQuery(); this.logger.debug('handleAggregation aggSql: %s', aggSql); - return this.dataPrismaService.$queryRawUnsafe<{ [field: string]: unknown }[]>(aggSql); + return this.queryDataPrisma<{ [field: string]: unknown }[]>(tableId, aggSql); } /** * Perform grouped aggregation operations @@ -621,7 +639,7 @@ export class AggregationService implements IAggregationService { const rawQuery = qb.toQuery(); this.logger.debug('handleRowCount raw query: %s', rawQuery); - return await this.dataPrismaService.$queryRawUnsafe<{ count: number }[]>(rawQuery); + return await this.queryDataPrisma<{ count: number }[]>(tableId, rawQuery); } private async fetchStatisticsParams(params: { @@ -874,7 +892,7 @@ export class AggregationService implements IAggregationService { const sql = queryBuilder.toQuery(); - const result = await this.dataPrismaService.$queryRawUnsafe<{ count: number }[] | null>(sql); + const result = await this.queryDataPrisma<{ count: number }[] | null>(tableId, sql); return { count: result ? Number(result[0]?.count) : 0, @@ -941,7 +959,11 @@ export class AggregationService implements IAggregationService { Object.values(fieldInstanceMap).map((f) => [f.id, `"${f.dbFieldName}"`]) ); - const basicSortIndex = await this.recordService.getBasicOrderIndexField(dbTableName, viewId); + const basicSortIndex = await this.recordService.getBasicOrderIndexField( + tableId, + dbTableName, + viewId + ); const filterQuery = (qb: Knex.QueryBuilder) => { this.dbProvider @@ -993,7 +1015,7 @@ export class AggregationService implements IAggregationService { this.logger.debug('getRecordIndexBySearchOrder sql: %s', sql); try { - return await this.dataPrismaService.$tx(async (prisma) => { + return await this.withDataPrismaTransaction(tableId, async (prisma) => { const result = await prisma.$queryRawUnsafe<{ __id: string; fieldId: string }[]>(sql); // no result found @@ -1035,9 +1057,7 @@ export class AggregationService implements IAggregationService { this.logger.debug('getRecordIndexBySearchOrder indexSql: %s', indexSql); const indexResult = // eslint-disable-next-line @typescript-eslint/naming-convention - await this.dataPrismaService.$queryRawUnsafe<{ row_num: number; __id: string }[]>( - indexSql - ); + await prisma.$queryRawUnsafe<{ row_num: number; __id: string }[]>(indexSql); if (indexResult?.length === 0) { return null; @@ -1102,7 +1122,7 @@ export class AggregationService implements IAggregationService { this.logger.debug('getRecordIndex sql: %s', sql); // eslint-disable-next-line @typescript-eslint/naming-convention - const result = await this.dataPrismaService.$queryRawUnsafe<{ row_num: number }[]>(sql); + const result = await this.queryDataPrisma<{ row_num: number }[]>(tableId, sql); if (!result?.length) { return null; @@ -1227,11 +1247,9 @@ export class AggregationService implements IAggregationService { endField: endField as DateFieldDto, dbTableName: viewCte || dbTableName, }); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe< - { date: Date | string; count: number; ids: string[] | string }[] - >(queryBuilder.toQuery()); + const result = await this.queryDataPrisma< + { date: Date | string; count: number; ids: string[] | string }[] + >(tableId, queryBuilder.toQuery()); const countMap = result.reduce( (map, item) => { diff --git a/apps/nestjs-backend/src/features/attachments/attachments.controller.ts b/apps/nestjs-backend/src/features/attachments/attachments.controller.ts index ae6b05b19d..b677459f73 100644 --- a/apps/nestjs-backend/src/features/attachments/attachments.controller.ts +++ b/apps/nestjs-backend/src/features/attachments/attachments.controller.ts @@ -46,12 +46,21 @@ export class AttachmentsController { @Query('token') token: string, @Query('response-content-disposition') responseContentDisposition?: string ) { + const headers: Record = {}; + headers['Cross-Origin-Resource-Policy'] = 'unsafe-none'; + headers['Content-Security-Policy'] = ''; + const hasCache = this.attachmentsService.localFileConditionalCaching(path, req.headers, res); if (hasCache) { + res.set(headers); res.status(304); return; } - const { fileStream, headers } = await this.attachmentsService.readLocalFile(path, token); + const { fileStream, headers: fileHeaders } = await this.attachmentsService.readLocalFile( + path, + token + ); + Object.assign(headers, fileHeaders); if (responseContentDisposition) { const fileNameMatch = responseContentDisposition.match(/filename\*=UTF-8''([^;]+)/) || @@ -64,8 +73,6 @@ export class AttachmentsController { headers['Content-Disposition'] = responseContentDisposition; } } - headers['Cross-Origin-Resource-Policy'] = 'unsafe-none'; - headers['Content-Security-Policy'] = ''; res.set(headers); return new StreamableFile(fileStream); } diff --git a/apps/nestjs-backend/src/features/base-share/base-share-open.controller.ts b/apps/nestjs-backend/src/features/base-share/base-share-open.controller.ts index ee843dc74f..be2d3f2105 100644 --- a/apps/nestjs-backend/src/features/base-share/base-share-open.controller.ts +++ b/apps/nestjs-backend/src/features/base-share/base-share-open.controller.ts @@ -1,4 +1,14 @@ -import { Body, Controller, Get, HttpCode, Post, Res, UseGuards, Request } from '@nestjs/common'; +import { + Body, + Controller, + Get, + HttpCode, + Post, + Res, + UseGuards, + Request, + UseInterceptors, +} from '@nestjs/common'; import { HttpErrorCode } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { @@ -10,7 +20,9 @@ import { type ICopyBaseShareVo, } from '@teable/openapi'; import { Response } from 'express'; +import { ClsService } from 'nestjs-cls'; import { CustomHttpException } from '../../custom.exception'; +import type { IClsStore } from '../../types/cls'; import { ZodValidationPipe } from '../../zod.validation.pipe'; import { AllowAnonymous } from '../auth/decorators/allow-anonymous.decorator'; import { Permissions } from '../auth/decorators/permissions.decorator'; @@ -19,6 +31,10 @@ import { ResourceMeta } from '../auth/decorators/resource_meta.decorator'; import { PermissionGuard } from '../auth/guard/permission.guard'; import { PermissionService } from '../auth/permission.service'; import { BaseDuplicateService } from '../base/base-duplicate.service'; +import { BaseDuplicateV2Service } from '../base/base-duplicate-v2.service'; +import { UseV2Feature } from '../canary/decorators/use-v2-feature.decorator'; +import { V2FeatureGuard } from '../canary/guards/v2-feature.guard'; +import { V2IndicatorInterceptor } from '../canary/interceptors/v2-indicator.interceptor'; import type { IBaseShareInfo } from './base-share-auth.service'; import { BaseShareAuthService } from './base-share-auth.service'; import { BaseShareAuthLocalGuard } from './guard/base-share-auth-local.guard'; @@ -30,7 +46,9 @@ export class BaseShareOpenController { private readonly baseShareAuthService: BaseShareAuthService, private readonly prismaService: PrismaService, private readonly baseDuplicateService: BaseDuplicateService, - private readonly permissionService: PermissionService + private readonly baseDuplicateV2Service: BaseDuplicateV2Service, + private readonly permissionService: PermissionService, + private readonly cls: ClsService ) {} @HttpCode(200) @@ -148,7 +166,9 @@ export class BaseShareOpenController { } @HttpCode(200) - @UseGuards(BaseShareAuthGuard, PermissionGuard) + @UseV2Feature('duplicateBase') + @UseGuards(BaseShareAuthGuard, V2FeatureGuard, PermissionGuard) + @UseInterceptors(V2IndicatorInterceptor) @Permissions('base|create') @ResourceMeta('spaceId', 'body') @Post('/:shareId/base/copy') @@ -205,21 +225,27 @@ export class BaseShareOpenController { nodes = [nodeId]; } - // Copy the base using BaseDuplicateService // allowCrossBase = false to disconnect cross-base links // duplicateMode = CopyShareBase to handle node relationships correctly - const { base, recordsLength } = await this.baseDuplicateService.duplicateBase( - { - fromBaseId, - spaceId, - name, - withRecords, - nodes, - baseId: targetBaseId, - }, - false, // allowCrossBase = false - BaseDuplicateMode.CopyShareBase - ); + const duplicateRo = { + fromBaseId, + spaceId, + name, + withRecords, + nodes, + baseId: targetBaseId, + }; + const { base, recordsLength } = this.cls.get('useV2') + ? await this.baseDuplicateV2Service.duplicateBase( + duplicateRo, + false, + BaseDuplicateMode.CopyShareBase + ) + : await this.baseDuplicateService.duplicateBase( + duplicateRo, + false, + BaseDuplicateMode.CopyShareBase + ); // Emit audit log for share base copy await this.baseDuplicateService.emitShareBaseCopyAuditLog( diff --git a/apps/nestjs-backend/src/features/base-share/base-share.module.ts b/apps/nestjs-backend/src/features/base-share/base-share.module.ts index 786e68d670..8066cf647c 100644 --- a/apps/nestjs-backend/src/features/base-share/base-share.module.ts +++ b/apps/nestjs-backend/src/features/base-share/base-share.module.ts @@ -4,6 +4,7 @@ import { authConfig } from '../../configs/auth.config'; import { AuthModule } from '../auth/auth.module'; import { PermissionModule } from '../auth/permission.module'; import { BaseModule } from '../base/base.module'; +import { CanaryModule } from '../canary'; import { FieldModule } from '../field/field.module'; import { ViewModule } from '../view/view.module'; import { BaseShareAuthService } from './base-share-auth.service'; @@ -19,6 +20,7 @@ import { BaseShareJwtStrategy } from './strategies/jwt.strategy'; AuthModule, PermissionModule, BaseModule, + CanaryModule, FieldModule, ViewModule, JwtModule.registerAsync({ diff --git a/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts index 41afa3a79d..d8f21106e4 100644 --- a/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts +++ b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts @@ -2,26 +2,24 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import type { IDsn } from '@teable/core'; import { DriverClient, HttpErrorCode, parseDsn } from '@teable/core'; -import { Prisma, PrismaService, PrismaClient, getDatabaseUrl } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; +import { Prisma, PrismaService, getDatabaseUrl } from '@teable/db-main-prisma'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; import { CustomHttpException } from '../../custom.exception'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex'; -import { BASE_READ_ONLY_ROLE_PREFIX, BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME } from './const'; +import { BASE_READ_ONLY_ROLE_PREFIX } from './const'; import { checkTableAccess, validateRoleOperations } from './utils'; @Injectable() export class BaseSqlExecutorService { - private db?: PrismaClient; private readonly dsn: IDsn; readonly driver: DriverClient; - private hasPgReadAllDataRole?: boolean; private readonly logger = new Logger(BaseSqlExecutorService.name); constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly configService: ConfigService, @InjectModel(DATA_KNEX) private readonly knex: Knex ) { @@ -32,214 +30,93 @@ export class BaseSqlExecutorService { private getDatabaseUrl() { return ( this.configService.get('PRISMA_DATABASE_URL_FOR_SQL_EXECUTOR') || - this.configService.get('PRISMA_DATA_DATABASE_URL') || - getDatabaseUrl('data') + getDatabaseUrl('meta') ); } - private getDisablePreSqlExecutorCheck() { - return this.configService.get('DISABLE_PRE_SQL_EXECUTOR_CHECK') === 'true'; - } - - private async getReadOnlyDatabaseConnectionConfig(): Promise { - if (!this.hasPgReadAllDataRole) { - return; - } - const isExistReadOnlyRole = await this.roleExits(BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME); - if (!isExistReadOnlyRole) { - await this.dataPrismaService.$tx(async (prisma) => { - try { - await prisma.$executeRawUnsafe( - this.knex - .raw( - `CREATE ROLE ?? WITH LOGIN PASSWORD ? NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION`, - [BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME, this.dsn.pass] - ) - .toQuery() - ); - await prisma.$executeRawUnsafe( - this.knex - .raw(`GRANT pg_read_all_data TO ??`, [BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME]) - .toQuery() - ); - } catch (error) { - if ( - error instanceof Prisma.PrismaClientKnownRequestError && - (error?.meta?.code === '42710' || - error?.meta?.code === '23505' || - error?.meta?.code === 'XX000') - ) { - this.logger.warn( - `read only role ${BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME} already exists or concurrent update detected, error code: ${error?.meta?.code}` - ); - return; - } - throw error; - } - }); - } - return `postgresql://${BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME}:${this.dsn.pass}@${this.dsn.host}:${this.dsn.port}/${this.dsn.db}${ - this.dsn.params - ? `?${Object.entries(this.dsn.params) - .map(([key, value]) => `${key}=${value}`) - .join('&')}` - : '' - }`; - } - - async onModuleInit() { - if (this.driver !== DriverClient.Pg) { - return; - } - if (this.getDisablePreSqlExecutorCheck()) { - return; - } - // if pg_read_all_data role not exist, no need to create read only role - this.hasPgReadAllDataRole = await this.roleExits('pg_read_all_data'); - if (!this.hasPgReadAllDataRole) { - return; - } - this.db = await this.createConnection(); - } - - async onModuleDestroy() { - await this.db?.$disconnect(); - } - - private async createConnection(): Promise { - if (this.db) { - return this.db; - } - const connectionConfig = await this.getReadOnlyDatabaseConnectionConfig(); - if (!connectionConfig) { - return; - } - const connection = new PrismaClient({ - datasources: { - db: { - url: connectionConfig, - }, - }, - }); - await connection.$connect(); - - // validate connection - try { - await connection.$queryRawUnsafe('SELECT 1'); - return connection; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } catch (error: any) { - await connection.$disconnect(); - throw new CustomHttpException( - `database connection failed: ${error.message}`, - HttpErrorCode.VALIDATION_ERROR, - { - localization: { - i18nKey: 'httpErrors.baseSqlExecutor.databaseConnectionFailed', - context: { - message: error.message, - }, - }, - } - ); - } - } - private getReadOnlyRoleName(baseId: string) { return `${BASE_READ_ONLY_ROLE_PREFIX}${baseId}`; } + private async dataPrismaForBase(baseId: string) { + return await this.databaseRouter.dataPrismaExecutorForBase(baseId); + } + async createReadOnlyRole(baseId: string) { const roleName = this.getReadOnlyRoleName(baseId); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe( - this.knex - .raw( - `CREATE ROLE ?? WITH NOLOGIN NOSUPERUSER NOINHERIT NOCREATEDB NOCREATEROLE NOREPLICATION`, - [roleName] - ) - .toQuery() - ); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe( - this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() - ); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe( - this.knex.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() - ); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe( - this.knex - .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [ - baseId, - roleName, - ]) - .toQuery() - ); + const dataPrisma = await this.dataPrismaForBase(baseId); + await dataPrisma.$executeRawUnsafe( + this.knex + .raw( + `CREATE ROLE ?? WITH NOLOGIN NOSUPERUSER NOINHERIT NOCREATEDB NOCREATEROLE NOREPLICATION`, + [roleName] + ) + .toQuery() + ); + await dataPrisma.$executeRawUnsafe( + this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() + ); + await dataPrisma.$executeRawUnsafe( + this.knex.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() + ); + await dataPrisma.$executeRawUnsafe( + this.knex + .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [ + baseId, + roleName, + ]) + .toQuery() + ); } async dropReadOnlyRole(baseId: string) { const roleName = this.getReadOnlyRoleName(baseId); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe( - this.knex.raw(`REVOKE USAGE ON SCHEMA ?? FROM ??`, [baseId, roleName]).toQuery() - ); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe( - this.knex - .raw(`REVOKE SELECT ON ALL TABLES IN SCHEMA ?? FROM ??`, [baseId, roleName]) - .toQuery() - ); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe( - this.knex - .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? REVOKE ALL ON TABLES FROM ??`, [ - baseId, - roleName, - ]) - .toQuery() - ); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe(this.knex.raw(`DROP ROLE IF EXISTS ??`, [roleName]).toQuery()); + const dataPrisma = await this.dataPrismaForBase(baseId); + await dataPrisma.$executeRawUnsafe( + this.knex.raw(`REVOKE USAGE ON SCHEMA ?? FROM ??`, [baseId, roleName]).toQuery() + ); + await dataPrisma.$executeRawUnsafe( + this.knex + .raw(`REVOKE SELECT ON ALL TABLES IN SCHEMA ?? FROM ??`, [baseId, roleName]) + .toQuery() + ); + await dataPrisma.$executeRawUnsafe( + this.knex + .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? REVOKE ALL ON TABLES FROM ??`, [ + baseId, + roleName, + ]) + .toQuery() + ); + await dataPrisma.$executeRawUnsafe( + this.knex.raw(`DROP ROLE IF EXISTS ??`, [roleName]).toQuery() + ); } async grantReadOnlyRole(baseId: string) { const roleName = this.getReadOnlyRoleName(baseId); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe( - this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() - ); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe( - this.knex.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() - ); - await this.dataPrismaService - .txClient() - .$executeRawUnsafe( - this.knex - .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [ - baseId, - roleName, - ]) - .toQuery() - ); + const dataPrisma = await this.dataPrismaForBase(baseId); + await dataPrisma.$executeRawUnsafe( + this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() + ); + await dataPrisma.$executeRawUnsafe( + this.knex.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() + ); + await dataPrisma.$executeRawUnsafe( + this.knex + .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [ + baseId, + roleName, + ]) + .toQuery() + ); } - private async roleExits(role: string): Promise { - const roleExists = await this.dataPrismaService.$queryRaw< - { count: bigint }[] - >`SELECT count(*) FROM pg_roles WHERE rolname=${role}`; + private async roleExits(role: string, baseId?: string): Promise { + const dataPrisma = baseId ? await this.dataPrismaForBase(baseId) : this.prismaService; + const roleExists = await dataPrisma.$queryRawUnsafe<{ count: bigint }[]>( + this.knex.raw('SELECT count(*) FROM pg_roles WHERE rolname = ?', [role]).toQuery() + ); return Boolean(roleExists[0].count); } @@ -248,7 +125,7 @@ export class BaseSqlExecutorService { return; } const roleName = this.getReadOnlyRoleName(baseId); - if (!(await this.roleExits(roleName))) { + if (!(await this.roleExits(roleName, baseId))) { try { await this.createReadOnlyRole(baseId); } catch (error) { @@ -279,8 +156,11 @@ export class BaseSqlExecutorService { await prisma.$executeRawUnsafe(this.knex.raw(`RESET ROLE`).toQuery()); } - private async readonlyExecuteSql(sql: string) { - return this.db?.$queryRawUnsafe(sql); + private async readonlyExecuteSql(baseId: string, sql: string) { + return this.databaseRouter.dataPrismaTransactionForBase(baseId, async (prisma) => { + await prisma.$executeRawUnsafe('SET TRANSACTION READ ONLY'); + return await prisma.$queryRawUnsafe(sql); + }); } /** @@ -318,7 +198,7 @@ export class BaseSqlExecutorService { }); // 3. read only role check table access, only pg and pg version > 14 support try { - await this.readonlyExecuteSql(sql); + await this.readonlyExecuteSql(baseId, sql); // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { throw new CustomHttpException( @@ -346,7 +226,7 @@ export class BaseSqlExecutorService { ) { await this.safeCheckSql(baseId, sql, opts); await this.roleCheckAndCreate(baseId); - return this.dataPrismaService.$tx(async (prisma) => { + return this.databaseRouter.dataPrismaTransactionForBase(baseId, async (prisma) => { try { await this.setRole(prisma, baseId); return await prisma.$queryRawUnsafe(sql); diff --git a/apps/nestjs-backend/src/features/base-sql-executor/utils.ts b/apps/nestjs-backend/src/features/base-sql-executor/utils.ts index 5266615464..a56e1922e1 100644 --- a/apps/nestjs-backend/src/features/base-sql-executor/utils.ts +++ b/apps/nestjs-backend/src/features/base-sql-executor/utils.ts @@ -102,7 +102,7 @@ export const checkTableAccess = ( const message = invalidTableNames.length > 0 ? `Table ${invalidTableNames.map((n: string) => `'${n}'`).join(', ')} not found. ` + - `dbTableName from get-tables-meta is already \`schema.table\` (e.g. \`bseXXX.tblYYY\`); ` + + `dbTableName from table get is already \`schema.table\` (e.g. \`bseXXX.tblYYY\`); ` + `use it in SQL as \`FROM "bseXXX"."tblYYY"\`.` : String(whiteListError?.message ?? whiteListError); diff --git a/apps/nestjs-backend/src/features/base/base-controller.spec.ts b/apps/nestjs-backend/src/features/base/base-controller.spec.ts new file mode 100644 index 0000000000..fc63df549d --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-controller.spec.ts @@ -0,0 +1,55 @@ +import { BaseController } from './base.controller'; + +const createController = (v2Reason: string | undefined) => { + const baseService = { + getBaseById: vi.fn().mockResolvedValue({ + id: 'bseTest', + v2Status: v2Reason ? { useV2: true, reason: v2Reason } : undefined, + }), + }; + const baseExportService = { + exportBaseZip: vi.fn().mockResolvedValue('v1-export'), + }; + const baseExportV2Service = { + exportBaseZip: vi.fn().mockResolvedValue('v2-export'), + }; + + return { + controller: new BaseController( + baseService as never, + baseExportService as never, + baseExportV2Service as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never + ), + baseService, + baseExportService, + baseExportV2Service, + }; +}; + +describe('BaseController', () => { + describe('exportBase', () => { + it('uses the v2 exporter for v2-created bases', async () => { + const { controller, baseExportService, baseExportV2Service } = createController('new_base'); + + await expect(controller.exportBase('bseTest')).resolves.toBe('v2-export'); + + expect(baseExportV2Service.exportBaseZip).toHaveBeenCalledWith('bseTest', true); + expect(baseExportService.exportBaseZip).not.toHaveBeenCalled(); + }); + + it('keeps rollout-only v2 decisions on the legacy exporter', async () => { + const { controller, baseExportService, baseExportV2Service } = + createController('space_feature'); + + await expect(controller.exportBase('bseTest', '0')).resolves.toBe('v1-export'); + + expect(baseExportService.exportBaseZip).toHaveBeenCalledWith('bseTest', false); + expect(baseExportV2Service.exportBaseZip).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/base/base-duplicate-v2.service.ts b/apps/nestjs-backend/src/features/base/base-duplicate-v2.service.ts new file mode 100644 index 0000000000..e9ce99de92 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-duplicate-v2.service.ts @@ -0,0 +1,25 @@ +import { Injectable } from '@nestjs/common'; +import { BaseDuplicateMode, type IDuplicateBaseRo } from '@teable/openapi'; +import type { BaseImportProgressCallback } from './base-import.service'; +import { BaseDuplicateService } from './base-duplicate.service'; + +type IDuplicateBaseV2Result = Awaited>; + +@Injectable() +export class BaseDuplicateV2Service { + constructor(private readonly baseDuplicateService: BaseDuplicateService) {} + + async duplicateBase( + duplicateBaseRo: IDuplicateBaseRo, + allowCrossBase: boolean = true, + duplicateMode: BaseDuplicateMode = BaseDuplicateMode.Normal, + onProgress?: BaseImportProgressCallback + ): Promise { + return await this.baseDuplicateService.duplicateBaseV2( + duplicateBaseRo, + allowCrossBase, + duplicateMode, + onProgress + ); + } +} diff --git a/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts b/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts index 5ac1154753..701645cae8 100644 --- a/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts +++ b/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts @@ -1,8 +1,12 @@ import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; +import { FieldType } from '@teable/core'; +import { BaseDuplicateMode } from '@teable/openapi'; import { GlobalModule } from '../../global/global.module'; import { BaseDuplicateService } from './base-duplicate.service'; +import type { BaseImportProgressCallback, IBaseImportProgress } from './base-import.service'; import { BaseModule } from './base.module'; +import type { ILinkFieldTableMap } from './utils'; describe('BaseDuplicateService', () => { let service: BaseDuplicateService; @@ -19,3 +23,558 @@ describe('BaseDuplicateService', () => { expect(service).toBeDefined(); }); }); + +describe('BaseDuplicateService duplicateBaseV2', () => { + type IServiceArgs = ConstructorParameters; + const duplicateBaseName = 'Duplicated base'; + const sourceTableName = 'Source table'; + const sourceDbTableName = 'bseSource.tblSource'; + + type IDuplicateServiceInternals = { + buildDuplicateStructureConfig: (...args: unknown[]) => Promise; + getDisconnectedLinkFieldTableMap: (...args: unknown[]) => Promise; + getDisconnectedLinkFieldIds: (...args: unknown[]) => Promise; + normalizeDuplicateStructureForV2: (structure: unknown) => unknown; + createDuplicateBaseSource: (...args: unknown[]) => { + records(tableId: string): AsyncIterable<{ fields: Record }>; + }; + duplicateTableData: (...args: unknown[]) => Promise; + duplicateAttachments: (...args: unknown[]) => Promise; + duplicateLinkJunction: (...args: unknown[]) => Promise; + }; + + it('should create the v2 execution context from the space data container', async () => { + const spaceId = 'spcTarget'; + const targetBaseId = 'bseTarget'; + const tableIdMap = { tblSource: 'tblTarget' }; + const context = { requestId: 'ctx' }; + const db = { dialect: 'pg' }; + const structure = { + name: duplicateBaseName, + icon: undefined, + tables: [{ id: 'tblSource', name: sourceTableName, fields: [], views: [] }], + }; + const source = { + structure, + records: async function* () { + yield undefined as never; + }, + }; + const commandBus = { + execute: vi.fn().mockResolvedValue({ + isErr: () => false, + value: (async function* () { + yield { + id: 'done', + baseId: targetBaseId, + tableIdMap, + fieldIdMap: {}, + viewIdMap: {}, + recordsLength: 0, + }; + })(), + }), + }; + const container = { + resolve: vi.fn().mockReturnValueOnce(commandBus).mockReturnValueOnce(db), + }; + const baseImportService = { + createBaseV2: vi.fn().mockResolvedValue({ id: targetBaseId }), + restoreBaseExtrasV2: vi.fn().mockResolvedValue(undefined), + }; + const v2ContainerService = { + getContainerForSpace: vi.fn().mockResolvedValue(container), + }; + const v2ContextFactory = { + createContext: vi.fn().mockResolvedValue(context), + }; + + const service = new BaseDuplicateService( + { + txClient: vi.fn().mockReturnValue({ + base: { update: vi.fn() }, + }), + } as unknown as IServiceArgs[0], + {} as IServiceArgs[1], + {} as IServiceArgs[2], + baseImportService as unknown as IServiceArgs[3], + {} as IServiceArgs[4], + {} as IServiceArgs[5], + {} as IServiceArgs[6], + { get: vi.fn().mockReturnValue('usrTest') } as unknown as IServiceArgs[7], + {} as IServiceArgs[8], + {} as IServiceArgs[9], + v2ContainerService as unknown as IServiceArgs[10], + v2ContextFactory as unknown as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + + const sourceDbTableNameByTableId = { tblSource: sourceDbTableName }; + + vi.spyOn(internals, 'buildDuplicateStructureConfig').mockResolvedValue({ + structure, + sourceDbTableNameByTableId, + }); + vi.spyOn(internals, 'getDisconnectedLinkFieldTableMap').mockResolvedValue({}); + vi.spyOn(internals, 'getDisconnectedLinkFieldIds').mockResolvedValue([]); + vi.spyOn(internals, 'normalizeDuplicateStructureForV2').mockReturnValue(structure); + vi.spyOn(internals, 'createDuplicateBaseSource').mockReturnValue(source); + + await service.duplicateBaseV2({ + fromBaseId: 'bseSource', + spaceId, + name: duplicateBaseName, + withRecords: false, + }); + + expect(v2ContainerService.getContainerForSpace).toHaveBeenCalledWith(spaceId); + expect(v2ContextFactory.createContext).toHaveBeenCalledWith(container); + expect(internals.createDuplicateBaseSource).toHaveBeenCalledWith( + 'bseSource', + structure, + {}, + sourceDbTableNameByTableId + ); + expect(commandBus.execute).toHaveBeenCalledWith(context, expect.any(Object)); + }); + + it('should create v2 structure first and copy records with raw table duplication', async () => { + const spaceId = 'spcTarget'; + const targetBaseId = 'bseTarget'; + const tableIdMap = { tblSource: 'tblTarget' }; + const fieldIdMap = { fldLink: 'fldTargetLink' }; + const viewIdMap = { viwSource: 'viwTarget' }; + const context = { requestId: 'ctx' }; + const db = { dialect: 'pg' }; + const structure = { + name: duplicateBaseName, + icon: undefined, + tables: [{ id: 'tblSource', name: sourceTableName, fields: [], views: [] }], + }; + const source = { + structure, + records: async function* () { + yield undefined as never; + }, + }; + const commandBus = { + execute: vi.fn().mockResolvedValue({ + isErr: () => false, + value: (async function* () { + yield { + id: 'done', + baseId: targetBaseId, + tableIdMap, + fieldIdMap, + viewIdMap, + recordsLength: 0, + }; + })(), + }), + }; + const container = { + resolve: vi.fn().mockReturnValueOnce(commandBus).mockReturnValueOnce(db), + }; + const baseUpdate = vi.fn(); + const baseImportService = { + createBaseV2: vi.fn().mockResolvedValue({ id: targetBaseId }), + restoreBaseExtrasV2: vi.fn().mockResolvedValue(undefined), + }; + const persistedComputedBackfillService = { + recomputeForTables: vi.fn().mockResolvedValue(undefined), + }; + const dataDbClientManager = { + getDataDatabaseForBase: vi.fn().mockResolvedValue({ cacheKey: 'same-db' }), + }; + const v2ContainerService = { + getContainerForSpace: vi.fn().mockResolvedValue(container), + }; + const v2ContextFactory = { + createContext: vi.fn().mockResolvedValue(context), + }; + + const service = new BaseDuplicateService( + { + txClient: vi.fn().mockReturnValue({ + base: { update: baseUpdate }, + }), + } as unknown as IServiceArgs[0], + {} as IServiceArgs[1], + {} as IServiceArgs[2], + baseImportService as unknown as IServiceArgs[3], + {} as IServiceArgs[4], + {} as IServiceArgs[5], + persistedComputedBackfillService as unknown as IServiceArgs[6], + { get: vi.fn().mockReturnValue('usrTest') } as unknown as IServiceArgs[7], + {} as IServiceArgs[8], + dataDbClientManager as unknown as IServiceArgs[9], + v2ContainerService as unknown as IServiceArgs[10], + v2ContextFactory as unknown as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + const mergedLinkFieldTableMap = { + tblSource: [{ dbFieldName: 'fldLink', selfKeyName: '__id', isMultipleCellValue: true }], + }; + + vi.spyOn(internals, 'buildDuplicateStructureConfig').mockResolvedValue({ + structure, + sourceDbTableNameByTableId: { tblSource: sourceDbTableName }, + }); + vi.spyOn(internals, 'getDisconnectedLinkFieldTableMap').mockResolvedValue( + mergedLinkFieldTableMap + ); + vi.spyOn(internals, 'getDisconnectedLinkFieldIds').mockResolvedValue(['fldDisconnected']); + vi.spyOn(internals, 'normalizeDuplicateStructureForV2').mockReturnValue(structure); + vi.spyOn(internals, 'createDuplicateBaseSource').mockReturnValue(source); + vi.spyOn(internals, 'duplicateTableData').mockResolvedValue(12); + vi.spyOn(internals, 'duplicateAttachments').mockResolvedValue(undefined); + vi.spyOn(internals, 'duplicateLinkJunction').mockResolvedValue(undefined); + + const result = await service.duplicateBaseV2({ + fromBaseId: 'bseSource', + spaceId, + name: duplicateBaseName, + withRecords: true, + }); + + const executedCommand = commandBus.execute.mock.calls[0]?.[1] as { withRecords: boolean }; + expect(executedCommand.withRecords).toBe(false); + expect(internals.duplicateTableData).toHaveBeenCalledWith( + targetBaseId, + tableIdMap, + fieldIdMap, + viewIdMap, + mergedLinkFieldTableMap, + undefined + ); + expect(internals.duplicateLinkJunction).toHaveBeenCalledWith( + targetBaseId, + tableIdMap, + fieldIdMap, + true, + ['fldDisconnected'] + ); + expect(persistedComputedBackfillService.recomputeForTables).toHaveBeenCalledWith(['tblTarget']); + expect(result.recordsLength).toBe(12); + }); + + it('should forward real row totals through duplicateBaseV2 progress events', async () => { + const spaceId = 'spcTarget'; + const targetBaseId = 'bseTarget'; + const tableIdMap = { tblSource: 'tblTarget' }; + const fieldIdMap = { fldText: 'fldTargetText' }; + const viewIdMap = { viwSource: 'viwTarget' }; + const context = { requestId: 'ctx' }; + const db = { dialect: 'pg' }; + const structure = { + name: duplicateBaseName, + icon: undefined, + tables: [{ id: 'tblSource', name: sourceTableName, fields: [], views: [] }], + }; + const source = { + structure, + records: async function* () { + yield undefined as never; + }, + }; + const commandBus = { + execute: vi.fn().mockResolvedValue({ + isErr: () => false, + value: (async function* () { + yield { + id: 'done', + baseId: targetBaseId, + tableIdMap, + fieldIdMap, + viewIdMap, + recordsLength: 0, + }; + })(), + }), + }; + const container = { + resolve: vi.fn().mockReturnValueOnce(commandBus).mockReturnValueOnce(db), + }; + const baseImportService = { + createBaseV2: vi.fn().mockResolvedValue({ id: targetBaseId }), + restoreBaseExtrasV2: vi.fn().mockResolvedValue(undefined), + }; + const persistedComputedBackfillService = { + recomputeForTables: vi.fn().mockResolvedValue(undefined), + }; + const dataDbClientManager = { + getDataDatabaseForBase: vi.fn().mockResolvedValue({ cacheKey: 'same-db' }), + }; + const v2ContainerService = { + getContainerForSpace: vi.fn().mockResolvedValue(container), + }; + const v2ContextFactory = { + createContext: vi.fn().mockResolvedValue(context), + }; + + const service = new BaseDuplicateService( + { + txClient: vi.fn().mockReturnValue({ + base: { update: vi.fn() }, + }), + } as unknown as IServiceArgs[0], + {} as IServiceArgs[1], + {} as IServiceArgs[2], + baseImportService as unknown as IServiceArgs[3], + {} as IServiceArgs[4], + {} as IServiceArgs[5], + persistedComputedBackfillService as unknown as IServiceArgs[6], + { get: vi.fn().mockReturnValue('usrTest') } as unknown as IServiceArgs[7], + {} as IServiceArgs[8], + dataDbClientManager as unknown as IServiceArgs[9], + v2ContainerService as unknown as IServiceArgs[10], + v2ContextFactory as unknown as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + const progressEvents: unknown[] = []; + + vi.spyOn(internals, 'buildDuplicateStructureConfig').mockResolvedValue({ + structure, + sourceDbTableNameByTableId: { tblSource: sourceDbTableName }, + }); + vi.spyOn(internals, 'getDisconnectedLinkFieldTableMap').mockResolvedValue({}); + vi.spyOn(internals, 'getDisconnectedLinkFieldIds').mockResolvedValue([]); + vi.spyOn(internals, 'normalizeDuplicateStructureForV2').mockReturnValue(structure); + vi.spyOn(internals, 'createDuplicateBaseSource').mockReturnValue(source); + vi.spyOn(internals, 'duplicateTableData').mockImplementation(async (...args: unknown[]) => { + const onProgress = args[5] as BaseImportProgressCallback | undefined; + onProgress?.({ phase: 'table_data_start', processedRows: 0, totalRows: 12 }); + onProgress?.({ + phase: 'table_data_progress', + tableId: 'tblTarget', + tableName: sourceTableName, + processedRows: 5, + batchProcessedRows: 5, + currentBatch: 1, + totalRows: 12, + }); + onProgress?.({ phase: 'table_data_done', processedRows: 12, totalRows: 12 }); + return 12; + }); + vi.spyOn(internals, 'duplicateAttachments').mockResolvedValue(undefined); + vi.spyOn(internals, 'duplicateLinkJunction').mockResolvedValue(undefined); + + const result = await service.duplicateBaseV2( + { + fromBaseId: 'bseSource', + spaceId, + name: duplicateBaseName, + withRecords: true, + }, + true, + BaseDuplicateMode.Normal, + (event: string | IBaseImportProgress) => progressEvents.push(event) + ); + + expect(internals.duplicateTableData).toHaveBeenCalledWith( + targetBaseId, + tableIdMap, + fieldIdMap, + viewIdMap, + {}, + expect.any(Function) + ); + expect(progressEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ phase: 'table_data_start', processedRows: 0, totalRows: 12 }), + expect.objectContaining({ + phase: 'table_data_progress', + processedRows: 5, + batchProcessedRows: 5, + totalRows: 12, + }), + expect.objectContaining({ phase: 'table_data_done', processedRows: 12, totalRows: 12 }), + expect.objectContaining({ phase: 'attachments_copying', processedRows: 12, totalRows: 12 }), + expect.objectContaining({ phase: 'duplicate_done', processedRows: 12, totalRows: 12 }), + ]) + ); + expect(result.recordsLength).toBe(12); + }); + + it('should aggregate table copy batches into global duplicate progress', async () => { + const dataPrisma = { + $queryRawUnsafe: vi.fn((query: string) => { + if (query.includes('source_a') && query.includes('count')) { + return Promise.resolve([{ count: 3 }]); + } + if (query.includes('source_b') && query.includes('count')) { + return Promise.resolve([{ count: 2 }]); + } + return Promise.resolve([]); + }), + $executeRawUnsafe: vi.fn().mockResolvedValue(0), + }; + const tableDuplicateService = { + duplicateTableData: vi.fn( + async ( + sourceDbTableName: string, + _targetDbTableName: string, + _viewIdMap: Record, + _fieldIdMap: Record, + _crossBaseLinkInfo: unknown[], + _dataPrisma: unknown, + options?: { + onProgress?: (progress: { + batchProcessedRows: number; + currentBatch: number; + processedRows: number; + totalRows: number; + }) => void; + } + ) => { + const batchProcessedRows = sourceDbTableName === 'source_a' ? 3 : 2; + options?.onProgress?.({ + batchProcessedRows, + currentBatch: 1, + processedRows: batchProcessedRows, + totalRows: batchProcessedRows, + }); + return batchProcessedRows; + } + ), + }; + const metaPrisma = { + tableMeta: { + findMany: vi.fn().mockResolvedValue([ + { id: 'tblA', dbTableName: 'source_a', name: 'A' }, + { id: 'tblB', dbTableName: 'source_b', name: 'B' }, + { id: 'tblTargetA', dbTableName: 'target_a', name: 'A Copy' }, + { id: 'tblTargetB', dbTableName: 'target_b', name: 'B Copy' }, + ]), + }, + }; + const knex = vi.fn((tableName: string) => ({ + count: vi.fn().mockReturnValue({ + toQuery: () => `select count(*) from ${tableName}`, + }), + })); + const progressEvents: unknown[] = []; + const service = new BaseDuplicateService( + { + txClient: vi.fn().mockReturnValue(metaPrisma), + } as unknown as IServiceArgs[0], + tableDuplicateService as unknown as IServiceArgs[1], + {} as IServiceArgs[2], + {} as IServiceArgs[3], + { + getForeignKeysInfo: vi.fn().mockReturnValue('select foreign keys'), + } as unknown as IServiceArgs[4], + knex as unknown as IServiceArgs[5], + {} as IServiceArgs[6], + {} as IServiceArgs[7], + {} as IServiceArgs[8], + { + dataPrismaForBase: vi.fn().mockResolvedValue(dataPrisma), + } as unknown as IServiceArgs[9], + {} as IServiceArgs[10], + {} as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + + const recordsLength = await internals.duplicateTableData( + 'bseTarget', + { tblA: 'tblTargetA', tblB: 'tblTargetB' }, + {}, + {}, + {}, + (event: string | IBaseImportProgress) => progressEvents.push(event) + ); + + expect(recordsLength).toBe(5); + expect(progressEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ phase: 'table_data_start', processedRows: 0, totalRows: 5 }), + expect.objectContaining({ + phase: 'table_data_progress', + tableId: 'tblTargetA', + tableName: 'A', + processedRows: 3, + batchProcessedRows: 3, + totalRows: 5, + }), + expect.objectContaining({ + phase: 'table_data_progress', + tableId: 'tblTargetB', + tableName: 'B', + processedRows: 5, + batchProcessedRows: 2, + totalRows: 5, + }), + expect.objectContaining({ phase: 'table_data_done', processedRows: 5, totalRows: 5 }), + ]) + ); + }); + + it('should read source records from the full source db table name', async () => { + const query = { + select: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + orderBy: vi.fn().mockReturnThis(), + limit: vi + .fn() + .mockResolvedValueOnce([{ __id: 'recSource', __auto_number: 1, fldText: 'A' }]) + .mockResolvedValueOnce([]), + }; + const dataKnex = vi.fn().mockReturnValue(query); + const service = new BaseDuplicateService( + {} as IServiceArgs[0], + {} as IServiceArgs[1], + {} as IServiceArgs[2], + {} as IServiceArgs[3], + {} as IServiceArgs[4], + {} as IServiceArgs[5], + {} as IServiceArgs[6], + {} as IServiceArgs[7], + {} as IServiceArgs[8], + { + dataKnexForBase: vi.fn().mockResolvedValue(dataKnex), + } as unknown as IServiceArgs[9], + {} as IServiceArgs[10], + {} as IServiceArgs[11] + ); + const internals = service as unknown as IDuplicateServiceInternals; + const source = internals.createDuplicateBaseSource( + 'bseSource', + { + name: duplicateBaseName, + tables: [ + { + id: 'tblSource', + name: sourceTableName, + dbTableName: 'tblShortName', + fields: [ + { + id: 'fldText', + name: 'Text', + dbFieldName: 'fldText', + type: FieldType.SingleLineText, + }, + ], + views: [], + }, + ], + }, + {}, + { tblSource: 'bseSource.tblShortName' } + ); + + const records = []; + for await (const record of source.records('tblSource')) { + records.push(record); + } + + expect(dataKnex).toHaveBeenCalledWith('bseSource.tblShortName'); + expect(records).toEqual([ + { + recordId: 'recSource', + fields: { fldText: 'A' }, + autoNumber: 1, + }, + ]); + }); +}); diff --git a/apps/nestjs-backend/src/features/base/base-duplicate.service.ts b/apps/nestjs-backend/src/features/base/base-duplicate.service.ts index a3dc155fea..232051e222 100644 --- a/apps/nestjs-backend/src/features/base/base-duplicate.service.ts +++ b/apps/nestjs-backend/src/features/base/base-duplicate.service.ts @@ -1,33 +1,85 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { Injectable, Logger } from '@nestjs/common'; import type { ILinkFieldOptions } from '@teable/core'; -import { FieldType } from '@teable/core'; +import { FieldType, HttpErrorCode } from '@teable/core'; import { PrismaService, ProvisionState } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { BaseDuplicateMode, CreateRecordAction, type ICreateBaseFromTemplateRo, + type ICreateBaseVo, + type ICrossSpaceBaseAffectedField, type IDuplicateBaseRo, } from '@teable/openapi'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { + DuplicateBaseCommand, + v2CoreTokens, + type DotTeaFieldInput, + type DuplicateBaseResult, + type DuplicateBaseSource, + type ICommandBus, + type NormalizedDotTeaField, + type NormalizedDotTeaStructure, +} from '@teable/v2-core'; +import { normalizeField } from '@teable/v2-dottea'; import { Knex } from 'knex'; -import { groupBy } from 'lodash'; +import type { Kysely } from 'kysely'; +import { groupBy, omit } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; import { EventEmitterService } from '../../event-emitter/event-emitter.service'; import { Events } from '../../event-emitter/events'; +import { DataDbClientManager } from '../../global/data-db-client-manager.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import type { IClsStore } from '../../types/cls'; import { createFieldInstanceByRaw } from '../field/model/factory'; import { PersistedComputedBackfillService } from '../record/computed/services/persisted-computed-backfill.service'; import { TableDuplicateService } from '../table/table-duplicate.service'; +import { V2ContainerService } from '../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../v2/v2-execution-context.factory'; import { BaseExportService } from './base-export.service'; +import type { BaseImportProgressCallback } from './base-import.service'; import { BaseImportService } from './base-import.service'; +import { + collectCrossSpaceAffectedFieldIds, + extractForeignTableId, +} from './cross-space-detection.util'; import { mergeLinkFieldTableMaps } from './utils'; +import type { ILinkFieldTableInfo, ILinkFieldTableMap } from './utils'; type DuplicatedBase = Awaited>['base']; +const v2DuplicateReadBatchSize = 500; +const v2DuplicateCopyBatchSize = 500; +type DuplicateStructureConfig = Awaited>; +type DuplicateV2FieldConfig = Omit< + DuplicateStructureConfig['tables'][number]['fields'][number], + keyof NormalizedDotTeaField +> & + NormalizedDotTeaField; +type DuplicateV2TableConfig = Omit & { + fields: DuplicateV2FieldConfig[]; +}; +type DuplicateV2StructureConfig = Omit & + Omit & { + tables: DuplicateV2TableConfig[]; + }; +type DuplicateStructureConfigResult = { + structure: DuplicateStructureConfig; + sourceDbTableNameByTableId: Record; +}; + +type IDataPrismaExecutor = { + $executeRawUnsafe(query: string, ...values: unknown[]): Promise; + $queryRawUnsafe(query: string, ...values: unknown[]): Promise; +}; + +type IDataPrismaScopedClient = IDataPrismaExecutor & { + txClient?: () => IDataPrismaExecutor; +}; @Injectable() export class BaseDuplicateService { @@ -35,7 +87,6 @@ export class BaseDuplicateService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, private readonly tableDuplicateService: TableDuplicateService, private readonly baseExportService: BaseExportService, private readonly baseImportService: BaseImportService, @@ -43,9 +94,16 @@ export class BaseDuplicateService { @InjectModel(DATA_KNEX) private readonly knex: Knex, private readonly persistedComputedBackfillService: PersistedComputedBackfillService, private readonly cls: ClsService, - private readonly eventEmitterService: EventEmitterService + private readonly eventEmitterService: EventEmitterService, + private readonly dataDbClientManager: DataDbClientManager, + private readonly v2ContainerService: V2ContainerService, + private readonly v2ContextFactory: V2ExecutionContextFactory ) {} + private getDataPrismaExecutor(prisma: IDataPrismaScopedClient): IDataPrismaExecutor { + return prisma.txClient?.() ?? prisma; + } + async duplicateBase( duplicateBaseRo: IDuplicateBaseRo, allowCrossBase: boolean = true, @@ -75,14 +133,7 @@ export class BaseDuplicateService { const { base: _base, tableIdMap, fieldIdMap, viewIdMap, ...rest } = duplicated; const crossBaseLinkFieldTableMap = allowCrossBase - ? ({} as Record< - string, - { - dbFieldName: string; - selfKeyName: string; - isMultipleCellValue: boolean; - }[] - >) + ? await this.getCrossBaseLinkFieldTableMap(tableIdMap, spaceId) : await this.getCrossBaseLinkFieldTableMap(tableIdMap); const disconnectedLinkFieldTableMap = await this.getDisconnectedLinkFieldTableMap( @@ -106,6 +157,7 @@ export class BaseDuplicateService { let recordsLength = 0; if (withRecords) { + await this.assertSameDataDatabaseForRecordCopy(fromBaseId, base.id); await prisma.base.update({ where: { id: base.id }, data: { @@ -115,13 +167,15 @@ export class BaseDuplicateService { }); recordsLength = await this.duplicateTableData( + base.id, tableIdMap, fieldIdMap, viewIdMap, mergedLinkFieldTableMap ); - await this.duplicateAttachments(tableIdMap, fieldIdMap); + await this.duplicateAttachments(base.id, tableIdMap, fieldIdMap); await this.duplicateLinkJunction( + base.id, tableIdMap, fieldIdMap, allowCrossBase, @@ -159,6 +213,182 @@ export class BaseDuplicateService { } } + async duplicateBaseV2( + duplicateBaseRo: IDuplicateBaseRo, + allowCrossBase: boolean = true, + duplicateMode: BaseDuplicateMode = BaseDuplicateMode.Normal, + onProgress?: BaseImportProgressCallback + ) { + const { fromBaseId, spaceId, withRecords, name, baseId, nodes } = duplicateBaseRo; + const userId = this.cls.get('user.id'); + const prisma = this.prismaService.txClient(); + const skipParentNodes = duplicateMode === BaseDuplicateMode.CopyShareBase; + let base: ICreateBaseVo | undefined; + + try { + onProgress?.({ phase: 'structure_creating' }); + const { structure, sourceDbTableNameByTableId } = await this.buildDuplicateStructureConfig( + fromBaseId, + name, + allowCrossBase, + nodes, + duplicateMode + ); + + const sourceTableIdMap = Object.fromEntries( + structure.tables.flatMap((table) => (table.id ? ([[table.id, table.id]] as const) : [])) + ); + // Cross-space links can't survive duplication (per-space data-DB sharding), + // so their cell values are always downgraded to text — even when + // allowCrossBase=true keeps same-space cross-base links intact. Mirrors + // the v1 branch in duplicateBase above. + const crossBaseLinkFieldTableMap: ILinkFieldTableMap = allowCrossBase + ? await this.getCrossBaseLinkFieldTableMap(sourceTableIdMap, spaceId) + : await this.getCrossBaseLinkFieldTableMap(sourceTableIdMap); + const disconnectedLinkFieldTableMap = await this.getDisconnectedLinkFieldTableMap( + sourceTableIdMap, + fromBaseId, + nodes, + skipParentNodes + ); + const mergedLinkFieldTableMap = mergeLinkFieldTableMaps( + crossBaseLinkFieldTableMap, + disconnectedLinkFieldTableMap + ); + const disconnectedLinkFieldIds = await this.getDisconnectedLinkFieldIds( + sourceTableIdMap, + fromBaseId, + nodes, + skipParentNodes + ); + const container = await this.v2ContainerService.getContainerForSpace(spaceId); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const db = container.resolve>(v2PostgresDbTokens.db); + const context = await this.v2ContextFactory.createContext(container); + base = await this.baseImportService.createBaseV2( + db, + spaceId, + structure.name, + structure.icon || undefined, + baseId, + duplicateMode !== BaseDuplicateMode.CopyShareBase + ); + if (withRecords) { + await this.assertSameDataDatabaseForRecordCopy(fromBaseId, base.id); + await prisma.base.update({ + where: { id: base.id }, + data: { + provisionState: ProvisionState.pending, + lastModifiedBy: userId, + }, + }); + } + + const normalizedStructure = this.normalizeDuplicateStructureForV2(structure); + const source = this.createDuplicateBaseSource( + fromBaseId, + normalizedStructure, + mergedLinkFieldTableMap, + sourceDbTableNameByTableId + ); + const commandResult = DuplicateBaseCommand.createFromSource({ + baseId: base.id, + source, + withRecords: false, + }); + if (commandResult.isErr()) { + throw new Error(commandResult.error.message); + } + const result = await commandBus.execute( + context, + commandResult.value + ); + if (result.isErr()) { + throw new Error(result.error.message); + } + + let tableIdMap: Record = {}; + let fieldIdMap: Record = {}; + let viewIdMap: Record = {}; + let recordsLength = 0; + for await (const event of result.value) { + if (event.id === 'error') { + throw new Error(event.message); + } + + if (event.id === 'progress') { + onProgress?.(event); + continue; + } + + tableIdMap = event.tableIdMap; + fieldIdMap = event.fieldIdMap; + viewIdMap = event.viewIdMap; + recordsLength = event.recordsLength; + } + + onProgress?.({ phase: 'structure_created', detail: base.id }); + await this.baseImportService.restoreBaseExtrasV2( + db, + base.id, + structure, + { tableIdMap, fieldIdMap, viewIdMap }, + onProgress + ); + if (withRecords) { + recordsLength = await this.duplicateTableData( + base.id, + tableIdMap, + fieldIdMap, + viewIdMap, + mergedLinkFieldTableMap, + onProgress + ); + onProgress?.({ + phase: 'attachments_copying', + processedRows: recordsLength, + totalRows: recordsLength, + }); + await this.duplicateAttachments(base.id, tableIdMap, fieldIdMap); + await this.duplicateLinkJunction( + base.id, + tableIdMap, + fieldIdMap, + allowCrossBase, + disconnectedLinkFieldIds + ); + await this.persistedComputedBackfillService.recomputeForTables(Object.values(tableIdMap)); + await prisma.base.update({ + where: { id: base.id }, + data: { + provisionState: ProvisionState.ready, + lastModifiedBy: userId, + }, + }); + } + onProgress?.({ + phase: 'duplicate_done', + processedRows: recordsLength, + totalRows: recordsLength, + }); + + return { base, tableIdMap, fieldIdMap, viewIdMap, recordsLength, structure }; + } catch (error) { + if (base?.id) { + await prisma.base + .update({ + where: { id: base.id }, + data: { + provisionState: ProvisionState.error, + lastModifiedBy: userId, + }, + }) + .catch(() => undefined); + } + throw error; + } + } + private async getDisconnectedLinkFieldIds( tableIdMap: Record, fromBaseId: string, @@ -190,6 +420,259 @@ export class BaseDuplicateService { .map((f) => f.id); } + private async assertSameDataDatabaseForRecordCopy(sourceBaseId: string, targetBaseId: string) { + const [source, target] = await Promise.all([ + this.dataDbClientManager.getDataDatabaseForBase(sourceBaseId, { useTransaction: true }), + this.dataDbClientManager.getDataDatabaseForBase(targetBaseId, { useTransaction: true }), + ]); + + if (source.cacheKey === target.cacheKey) { + return; + } + + throw new CustomHttpException( + 'Duplicating records across different space data databases is not supported yet', + HttpErrorCode.VALIDATION_ERROR + ); + } + + private async buildDuplicateStructureConfig( + fromBaseId: string, + baseName?: string, + allowCrossBase?: boolean, + nodes?: string[], + duplicateMode: BaseDuplicateMode = BaseDuplicateMode.Normal + ): Promise { + const prisma = this.prismaService.txClient(); + const baseRaw = await prisma.base.findUniqueOrThrow({ + where: { + id: fromBaseId, + deletedTime: null, + }, + }); + baseRaw.name = baseName || `${baseRaw.name} (Copy)`; + + const skipParentNodes = duplicateMode === BaseDuplicateMode.CopyShareBase; + const { + finalIncludeNodes, + includedTableIds, + includedFolderIds, + includedDashboardIds, + includedWorkflowIds, + includedAppIds, + excludedTableIds, + } = await this.collectNodesAndResourceIds(fromBaseId, nodes, skipParentNodes); + const rootNodeIds = skipParentNodes ? [...(nodes || [])] : undefined; + + const tableRaws = await prisma.tableMeta.findMany({ + where: { + baseId: fromBaseId, + deletedTime: null, + ...(includedTableIds !== undefined ? { id: { in: includedTableIds } } : {}), + }, + orderBy: { + order: 'asc', + }, + }); + const tableIds = tableRaws.map(({ id }) => id); + const fieldRaws = await prisma.field.findMany({ + where: { + tableId: { in: tableIds }, + deletedTime: null, + }, + }); + const viewRaws = await prisma.view.findMany({ + where: { + tableId: { in: tableIds }, + deletedTime: null, + }, + orderBy: { + order: 'asc', + }, + }); + + const structure = await this.baseExportService.generateBaseStructConfig({ + baseRaw, + tableRaws, + fieldRaws, + viewRaws, + allowCrossBase, + includeNodes: finalIncludeNodes, + includedFolderIds, + includedDashboardIds, + includedWorkflowIds, + includedAppIds, + excludedTableIds, + rootNodeIds, + }); + + return { + structure, + sourceDbTableNameByTableId: Object.fromEntries( + tableRaws.flatMap(({ id, dbTableName }) => + dbTableName ? ([[id, dbTableName]] as const) : [] + ) + ), + }; + } + + private createDuplicateBaseSource( + sourceBaseId: string, + structure: DuplicateV2StructureConfig, + disconnectedLinkFieldTableMap: ILinkFieldTableMap, + sourceDbTableNameByTableId: Record = {} + ): DuplicateBaseSource { + const tableById = new Map(structure.tables.map((table) => [table.id, table])); + const readRows = (sourceDbTableName: string, crossBaseLinkInfo: ILinkFieldTableInfo[]) => + this.createSourceTableRecordRows(sourceBaseId, sourceDbTableName, crossBaseLinkInfo); + const toRecordInput = (table: DuplicateV2TableConfig, row: Record) => + this.toDuplicateBaseRecordInput(table, row); + + return { + structure, + async *records(tableId: string) { + const table = tableById.get(tableId); + const sourceDbTableName = sourceDbTableNameByTableId[tableId] ?? table?.dbTableName; + if (!table || !sourceDbTableName) { + return; + } + + for await (const row of readRows( + sourceDbTableName, + disconnectedLinkFieldTableMap[tableId] || [] + )) { + yield toRecordInput(table, row); + } + }, + }; + } + + private normalizeDuplicateStructureForV2( + structure: DuplicateStructureConfig + ): DuplicateV2StructureConfig { + const availableTableIds = new Set( + structure.tables.flatMap((table) => (table.id ? [table.id] : [])) + ); + const fieldIdsByTableId = new Map( + structure.tables.flatMap((table) => + table.id + ? [ + [ + table.id, + new Set(table.fields.flatMap((field) => (field.id ? [field.id] : []))), + ] as const, + ] + : [] + ) + ); + + return { + ...structure, + tables: structure.tables.map((table) => { + const tableFieldTypesById = new Map( + table.fields.filter((field) => field.id).map((field) => [field.id!, field.type] as const) + ); + + return { + ...table, + fields: table.fields.map((field) => { + const normalized = normalizeField(field as DotTeaFieldInput, tableFieldTypesById, { + availableTableIds, + fieldIdsByTableId, + }); + const normalizedField = { ...field, ...normalized }; + + if ( + normalized.type === FieldType.SingleLineText && + (field.isLookup || + field.isConditionalLookup || + field.type === FieldType.Rollup || + field.type === FieldType.ConditionalRollup) + ) { + return omit(normalizedField, ['isLookup', 'isConditionalLookup', 'lookupOptions']); + } + + return normalizedField; + }), + }; + }), + }; + } + + private shouldSkipDuplicateRecordColumn(columnName: string) { + return ( + columnName === '__id' || + columnName.startsWith('__row_') || + this.isRestoreSystemColumn(columnName) + ); + } + + private shouldSkipDuplicateRecordField(field: DuplicateV2FieldConfig) { + if (field.isLookup) { + return true; + } + + switch (field.type) { + case FieldType.Button: + case FieldType.Formula: + case FieldType.Rollup: + case FieldType.ConditionalRollup: + case 'lookup': + case 'conditionalLookup': + return true; + default: + return false; + } + } + + private toDuplicateRecordFieldValue(field: DuplicateV2FieldConfig, value: unknown) { + return this.isJsonDbField(field.dbFieldType) ? this.parseJsonCellValue(value) : value; + } + + private toDuplicateBaseRecordInput(table: DuplicateV2TableConfig, row: Record) { + const fields: Record = {}; + const fieldsByDbFieldName = new Map(table.fields.map((field) => [field.dbFieldName, field])); + for (const [columnName, value] of Object.entries(row)) { + if (this.shouldSkipDuplicateRecordColumn(columnName)) { + continue; + } + + const field = fieldsByDbFieldName.get(columnName); + if (!field?.id || this.shouldSkipDuplicateRecordField(field)) { + continue; + } + + const fieldId = field.id; + fields[fieldId] = this.toDuplicateRecordFieldValue(field, value); + } + + const orders = Object.fromEntries( + Object.entries(row).flatMap(([columnName, value]) => { + if (!columnName.startsWith('__row_')) { + return []; + } + const order = Number(value); + return Number.isFinite(order) ? [[columnName.slice('__row_'.length), order]] : []; + }) + ); + + return { + recordId: typeof row.__id === 'string' ? row.__id : undefined, + fields, + ...(Object.keys(orders).length ? { orders } : {}), + ...(row.__version ? { version: Number(row.__version) } : {}), + ...(row.__auto_number ? { autoNumber: Number(row.__auto_number) } : {}), + ...(row.__created_time ? { createdTime: this.toRestoreString(row.__created_time) } : {}), + ...(row.__created_by ? { createdBy: this.toRestoreString(row.__created_by) } : {}), + ...(row.__last_modified_time + ? { lastModifiedTime: this.toRestoreString(row.__last_modified_time) } + : {}), + ...(row.__last_modified_by + ? { lastModifiedBy: this.toRestoreString(row.__last_modified_by) } + : {}), + }; + } + private async duplicateStructure( fromBaseId: string, spaceId: string, @@ -268,6 +751,7 @@ export class BaseDuplicateService { includedAppIds, excludedTableIds, rootNodeIds, + destSpaceId: spaceId, }); this.logger.log(`base-duplicate-service: Start to getting base structure config successfully`); @@ -283,7 +767,9 @@ export class BaseDuplicateService { structure, baseId, undefined, - duplicateMode + duplicateMode, + undefined, + { useTransaction: true } ); return { base: newBase, tableIdMap, fieldIdMap, viewIdMap, ...rest }; @@ -434,11 +920,8 @@ export class BaseDuplicateService { fromBaseId: string, nodes?: string[], skipParentNodes: boolean = false - ) { - const tableId2DbFieldNameMap: Record< - string, - { dbFieldName: string; selfKeyName: string; isMultipleCellValue: boolean }[] - > = {}; + ): Promise { + const tableId2DbFieldNameMap: ILinkFieldTableMap = {}; const { excludedTableIds } = await this.collectNodesAndResourceIds( fromBaseId, nodes, @@ -462,56 +945,93 @@ export class BaseDuplicateService { .map((f) => ({ ...createFieldInstanceByRaw(f), tableId: f.tableId })) .filter((f) => excludedTableIds.includes((f.options as ILinkFieldOptions)?.foreignTableId)); - // relative fields - // const disconnectedLinkRelativeFields = allFieldRaws - // .map((f) => ({ ...createFieldInstanceByRaw(f), tableId: f.tableId })) - // .filter( - // ({ type, isLookup }) => - // isLookup || type === FieldType.Rollup || type === FieldType.ConditionalRollup - // ) - // .filter(({ lookupOptions }) => { - // if (!lookupOptions || !isLinkLookupOptions(lookupOptions)) { - // return false; - // } - // return disconnectedLinkFields.map(({ id }) => id).includes(lookupOptions.linkFieldId); - // }); - - const groupedDisconnectedLinkFields = groupBy([...disconnectedLinkFields], 'tableId'); - - Object.entries(groupedDisconnectedLinkFields).map(([tableId, fields]) => { - tableId2DbFieldNameMap[tableId] = fields.map( - ({ dbFieldName, options, isMultipleCellValue }) => { - return { - dbFieldName, - selfKeyName: (options as ILinkFieldOptions).selfKeyName, - isMultipleCellValue: !!isMultipleCellValue, - }; - } - ); - - tableId2DbFieldNameMap[tableIdMap[tableId]] = fields.map( - ({ dbFieldName, options, isMultipleCellValue }) => { - return { - dbFieldName, - selfKeyName: (options as ILinkFieldOptions).selfKeyName, - isMultipleCellValue: !!isMultipleCellValue, - }; - } - ); + const toLinkInfo = ({ + dbFieldName, + options, + isMultipleCellValue, + }: (typeof disconnectedLinkFields)[number]): ILinkFieldTableInfo => ({ + dbFieldName, + selfKeyName: (options as ILinkFieldOptions).selfKeyName, + isMultipleCellValue: !!isMultipleCellValue, + }); - return { - tableId2DbFieldNameMap, - }; + Object.entries(groupBy(disconnectedLinkFields, 'tableId')).forEach(([tableId, fields]) => { + const info = fields.map(toLinkInfo); + tableId2DbFieldNameMap[tableId] = info; + tableId2DbFieldNameMap[tableIdMap[tableId]] = info; }); return tableId2DbFieldNameMap; } - private async getCrossBaseLinkFieldTableMap(tableIdMap: Record) { - const tableId2DbFieldNameMap: Record< - string, - { dbFieldName: string; selfKeyName: string; isMultipleCellValue: boolean }[] - > = {}; + async previewCrossSpaceAffectedFields( + fromBaseId: string, + destSpaceId: string + ): Promise { + const prisma = this.prismaService.txClient(); + const tables = await prisma.tableMeta.findMany({ + where: { baseId: fromBaseId, deletedTime: null }, + select: { id: true, name: true }, + }); + if (!tables.length) return []; + const tableNameMap = new Map(tables.map((t) => [t.id, t.name])); + const tableIds = tables.map((t) => t.id); + + const allFields = await prisma.field.findMany({ + where: { tableId: { in: tableIds }, deletedTime: null }, + select: { + id: true, + name: true, + type: true, + tableId: true, + isLookup: true, + isConditionalLookup: true, + options: true, + lookupOptions: true, + }, + }); + + const inBaseTableIds = new Set(tableIds); + const foreignTableIds = Array.from( + new Set( + allFields + .map((f) => extractForeignTableId(f)) + .filter((ft): ft is string => !!ft && !inBaseTableIds.has(ft)) + ) + ); + if (!foreignTableIds.length) return []; + + const foreignTables = await prisma.tableMeta.findMany({ + where: { id: { in: foreignTableIds }, deletedTime: null }, + select: { id: true, base: { select: { spaceId: true } } }, + }); + const foreignSpaceMap = new Map(foreignTables.map((t) => [t.id, t.base.spaceId])); + + const affected = collectCrossSpaceAffectedFieldIds({ + fields: allFields, + isForeignInternal: (ft) => inBaseTableIds.has(ft), + isForeignCrossSpace: (ft) => { + const s = foreignSpaceMap.get(ft); + return Boolean(s && s !== destSpaceId); + }, + }); + + return allFields + .filter((f) => affected.has(f.id)) + .map((f) => ({ + fieldId: f.id, + fieldName: f.name, + type: f.type, + tableId: f.tableId, + tableName: tableNameMap.get(f.tableId) ?? '', + })); + } + + private async getCrossBaseLinkFieldTableMap( + tableIdMap: Record, + destSpaceId?: string + ): Promise { + const tableId2DbFieldNameMap: ILinkFieldTableMap = {}; const prisma = this.prismaService.txClient(); const allFieldRaws = await prisma.field.findMany({ where: { @@ -520,71 +1040,87 @@ export class BaseDuplicateService { }, }); - const crossBaseLinkFields = allFieldRaws + const linkFields = allFieldRaws .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) .map((f) => ({ ...createFieldInstanceByRaw(f), tableId: f.tableId })) .filter((f) => (f.options as ILinkFieldOptions).baseId); - const groupedCrossBaseLinkFields = groupBy(crossBaseLinkFields, 'tableId'); - - Object.entries(groupedCrossBaseLinkFields).map(([tableId, fields]) => { - tableId2DbFieldNameMap[tableId] = fields.map( - ({ dbFieldName, options, isMultipleCellValue }) => { - return { - dbFieldName, - selfKeyName: (options as ILinkFieldOptions).selfKeyName, - isMultipleCellValue: !!isMultipleCellValue, - }; - } + let crossBaseLinkFields = linkFields; + if (destSpaceId) { + const foreignBaseIds = Array.from( + new Set( + linkFields + .map((f) => (f.options as ILinkFieldOptions).baseId) + .filter((x): x is string => Boolean(x)) + ) ); - tableId2DbFieldNameMap[tableIdMap[tableId]] = fields.map( - ({ dbFieldName, options, isMultipleCellValue }) => { - return { - dbFieldName, - selfKeyName: (options as ILinkFieldOptions).selfKeyName, - isMultipleCellValue: !!isMultipleCellValue, - }; - } + const bases = foreignBaseIds.length + ? await prisma.base.findMany({ + where: { id: { in: foreignBaseIds }, deletedTime: null }, + select: { id: true, spaceId: true }, + }) + : []; + const crossSpaceBaseIds = new Set( + bases.filter((b) => b.spaceId !== destSpaceId).map((b) => b.id) ); + crossBaseLinkFields = linkFields.filter((f) => { + const baseId = (f.options as ILinkFieldOptions).baseId; + return baseId && crossSpaceBaseIds.has(baseId); + }); + } + + const toLinkInfo = ({ + dbFieldName, + options, + isMultipleCellValue, + }: (typeof crossBaseLinkFields)[number]): ILinkFieldTableInfo => ({ + dbFieldName, + selfKeyName: (options as ILinkFieldOptions).selfKeyName, + isMultipleCellValue: !!isMultipleCellValue, + }); + + Object.entries(groupBy(crossBaseLinkFields, 'tableId')).forEach(([tableId, fields]) => { + const info = fields.map(toLinkInfo); + tableId2DbFieldNameMap[tableId] = info; + tableId2DbFieldNameMap[tableIdMap[tableId]] = info; }); return tableId2DbFieldNameMap; } private async duplicateTableData( + targetBaseId: string, tableIdMap: Record, fieldIdMap: Record, viewIdMap: Record, - crossBaseLinkFieldTableMap: Record< - string, - { dbFieldName: string; selfKeyName: string; isMultipleCellValue: boolean }[] - > + crossBaseLinkFieldTableMap: ILinkFieldTableMap, + onProgress?: BaseImportProgressCallback ): Promise { - const prisma = this.dataPrismaService.txClient(); + const prisma = this.getDataPrismaExecutor( + (await this.dataDbClientManager.dataPrismaForBase(targetBaseId, { + useTransaction: true, + })) as IDataPrismaScopedClient + ); const metaPrisma = this.prismaService.txClient(); const tableId2DbTableNameMap: Record = {}; + const tableId2NameMap: Record = {}; const allTableId = Object.keys(tableIdMap).concat(Object.values(tableIdMap)); - const sourceTableRaws = await metaPrisma.tableMeta.findMany({ + const tableRaws = await metaPrisma.tableMeta.findMany({ where: { id: { in: allTableId }, deletedTime: null }, select: { id: true, dbTableName: true, + name: true, }, }); - const targetTableRaws = await metaPrisma.tableMeta.findMany({ - where: { id: { in: allTableId }, deletedTime: null }, - select: { - id: true, - dbTableName: true, - }, - }); - sourceTableRaws.forEach((tableRaw) => { + tableRaws.forEach((tableRaw) => { tableId2DbTableNameMap[tableRaw.id] = tableRaw.dbTableName; + tableId2NameMap[tableRaw.id] = tableRaw.name; }); const oldTableId = Object.keys(tableIdMap); - const dbTableNames = targetTableRaws.map((tableRaw) => tableRaw.dbTableName); + const dbTableNames = tableRaws.map((tableRaw) => tableRaw.dbTableName); // Query total records count from all source tables before duplicating let totalRecordsCount = 0; @@ -592,9 +1128,18 @@ export class BaseDuplicateService { const sourceDbTableName = tableId2DbTableNameMap[tableId]; const countQuery = this.knex(sourceDbTableName).count('*', { as: 'count' }).toQuery(); const countResult = await prisma.$queryRawUnsafe<[{ count: bigint | number }]>(countQuery); - totalRecordsCount += Number(countResult[0]?.count || 0); + const tableRecordCount = Number(countResult[0]?.count || 0); + totalRecordsCount += tableRecordCount; } + onProgress?.({ + phase: 'table_data_start', + processedRows: 0, + batchProcessedRows: 0, + currentBatch: 0, + totalRows: totalRecordsCount, + }); + const allForeignKeyInfos = [] as { constraint_name: string; column_name: string; @@ -633,25 +1178,20 @@ export class BaseDuplicateService { await prisma.$executeRawUnsafe(dropForeignKeyQuery); } + const progressState = { processedRows: 0, totalRows: totalRecordsCount }; for (const tableId of oldTableId) { - const newTableId = tableIdMap[tableId]; - const oldDbTableName = tableId2DbTableNameMap[tableId]; - const newDbTableName = tableId2DbTableNameMap[newTableId]; - try { - await this.tableDuplicateService.duplicateTableData( - oldDbTableName, - newDbTableName, - viewIdMap, - fieldIdMap, - crossBaseLinkFieldTableMap[tableId] || [] - ); - } catch (error) { - this.logger.error( - `exc duplicate table data error: ${(error as Error)?.message}`, - (error as Error)?.stack - ); - throw error; - } + await this.duplicateSingleTableData({ + tableId, + tableIdMap, + fieldIdMap, + viewIdMap, + crossBaseLinkFieldTableMap, + tableId2DbTableNameMap, + tableId2NameMap, + prisma, + progressState, + onProgress, + }); } for (const { @@ -674,32 +1214,265 @@ export class BaseDuplicateService { await prisma.$executeRawUnsafe(addForeignKeyQuerySql); } - return totalRecordsCount; + onProgress?.({ + phase: 'table_data_done', + processedRows: progressState.processedRows, + totalRows: totalRecordsCount, + }); + + return onProgress ? progressState.processedRows : totalRecordsCount; + } + + private createDuplicateTableProgressOptions( + onProgress: BaseImportProgressCallback | undefined, + progressState: { processedRows: number; totalRows: number }, + tableId: string, + tableName: string + ) { + if (!onProgress) { + return undefined; + } + + return { + batchSize: v2DuplicateCopyBatchSize, + onProgress: (progress: { batchProcessedRows: number; currentBatch: number }) => { + progressState.processedRows += progress.batchProcessedRows; + onProgress({ + phase: 'table_data_progress', + tableId, + tableName, + processedRows: progressState.processedRows, + batchProcessedRows: progress.batchProcessedRows, + currentBatch: progress.currentBatch, + totalRows: progressState.totalRows, + }); + }, + }; + } + + private async duplicateSingleTableData(params: { + tableId: string; + tableIdMap: Record; + fieldIdMap: Record; + viewIdMap: Record; + crossBaseLinkFieldTableMap: ILinkFieldTableMap; + tableId2DbTableNameMap: Record; + tableId2NameMap: Record; + prisma: IDataPrismaExecutor; + progressState: { processedRows: number; totalRows: number }; + onProgress?: BaseImportProgressCallback; + }) { + const { + tableId, + tableIdMap, + fieldIdMap, + viewIdMap, + crossBaseLinkFieldTableMap, + tableId2DbTableNameMap, + tableId2NameMap, + prisma, + progressState, + onProgress, + } = params; + const targetTableId = tableIdMap[tableId]; + const sourceDbTableName = tableId2DbTableNameMap[tableId]; + const targetDbTableName = tableId2DbTableNameMap[targetTableId]; + const tableName = tableId2NameMap[tableId] ?? sourceDbTableName; + const processedRowsBeforeTable = progressState.processedRows; + + try { + const tableRows = await this.tableDuplicateService.duplicateTableData( + sourceDbTableName, + targetDbTableName, + viewIdMap, + fieldIdMap, + crossBaseLinkFieldTableMap[tableId] || [], + prisma, + this.createDuplicateTableProgressOptions( + onProgress, + progressState, + targetTableId, + tableName + ) + ); + if (onProgress && progressState.processedRows === processedRowsBeforeTable && tableRows > 0) { + progressState.processedRows += tableRows; + onProgress({ + phase: 'table_data_progress', + tableId: targetTableId, + tableName, + processedRows: progressState.processedRows, + batchProcessedRows: tableRows, + currentBatch: 1, + totalRows: progressState.totalRows, + }); + } + onProgress?.({ + phase: 'table_data_done', + tableId: targetTableId, + tableName, + processedRows: progressState.processedRows, + totalRows: progressState.totalRows, + }); + } catch (error) { + this.logger.error( + `exc duplicate table data error: ${(error as Error)?.message}`, + (error as Error)?.stack + ); + throw error; + } + } + + private isRestoreSystemColumn(columnName: string) { + return [ + '__version', + '__auto_number', + '__created_time', + '__created_by', + '__last_modified_time', + '__last_modified_by', + ].includes(columnName); + } + + private parseJsonCellValue(value: unknown) { + if (typeof value !== 'string') { + return value; + } + + try { + return JSON.parse(value); + } catch { + return value; + } + } + + private isJsonDbField(dbFieldType?: string | null) { + return typeof dbFieldType === 'string' && dbFieldType.toLowerCase() === 'json'; + } + + private toRestoreString(value: unknown) { + if (value instanceof Date) { + return value.toISOString(); + } + return String(value); + } + + private async *createSourceTableRecordRows( + sourceBaseId: string, + sourceDbTableName: string, + crossBaseLinkInfo: ILinkFieldTableInfo[] + ): AsyncGenerator> { + const dataKnex = await this.dataDbClientManager.dataKnexForBase(sourceBaseId, { + useTransaction: true, + }); + let lastAutoNumber = 0; + + while (true) { + const rows = await dataKnex>(sourceDbTableName) + .select('*') + .where('__auto_number', '>', lastAutoNumber) + .orderBy('__auto_number', 'asc') + .limit(v2DuplicateReadBatchSize); + if (!rows.length) { + return; + } + + for (const row of rows) { + yield this.normalizeCrossBaseLinkColumns(row, crossBaseLinkInfo); + } + + lastAutoNumber = Number(rows[rows.length - 1]?.__auto_number ?? lastAutoNumber); + } + } + + private normalizeCrossBaseLinkColumns( + row: Record, + crossBaseLinkInfo: ILinkFieldTableInfo[] + ) { + if (!crossBaseLinkInfo.length) { + return row; + } + + const nextRow = { ...row }; + for (const { dbFieldName, isMultipleCellValue } of crossBaseLinkInfo) { + if (!(dbFieldName in nextRow)) { + continue; + } + + nextRow[dbFieldName] = this.toCrossBaseLinkTitle(nextRow[dbFieldName], isMultipleCellValue); + } + + return nextRow; + } + + private toCrossBaseLinkTitle(value: unknown, isMultipleCellValue: boolean): unknown { + if (value == null) { + return value; + } + + if (typeof value === 'string') { + try { + return this.toCrossBaseLinkTitle(JSON.parse(value), isMultipleCellValue); + } catch { + return value; + } + } + + if (isMultipleCellValue) { + return Array.isArray(value) + ? value + .map((item) => + item && typeof item === 'object' && 'title' in item + ? String((item as { title?: unknown }).title ?? '') + : '' + ) + .filter(Boolean) + .join(', ') + : value; + } + + return value && typeof value === 'object' && 'title' in value + ? String((value as { title?: unknown }).title ?? '') + : value; } private async duplicateAttachments( + targetBaseId: string, tableIdMap: Record, fieldIdMap: Record ) { + const dataPrisma = this.getDataPrismaExecutor( + (await this.dataDbClientManager.dataPrismaForBase(targetBaseId, { + useTransaction: true, + })) as IDataPrismaScopedClient + ); for (const [sourceTableId, targetTableId] of Object.entries(tableIdMap)) { await this.tableDuplicateService.duplicateAttachments( sourceTableId, targetTableId, - fieldIdMap + fieldIdMap, + dataPrisma ); } } private async duplicateLinkJunction( + targetBaseId: string, tableIdMap: Record, fieldIdMap: Record, allowCrossBase: boolean = true, disconnectedLinkFieldIds?: string[] ) { + const dataPrisma = this.getDataPrismaExecutor( + (await this.dataDbClientManager.dataPrismaForBase(targetBaseId, { + useTransaction: true, + })) as IDataPrismaScopedClient + ); await this.tableDuplicateService.duplicateLinkJunction( tableIdMap, fieldIdMap, allowCrossBase, + dataPrisma, disconnectedLinkFieldIds ); } diff --git a/apps/nestjs-backend/src/features/base/base-export-v2.service.ts b/apps/nestjs-backend/src/features/base/base-export-v2.service.ts new file mode 100644 index 0000000000..f53a782646 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-export-v2.service.ts @@ -0,0 +1,1002 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { PassThrough, Readable } from 'stream'; +import { Injectable, Logger } from '@nestjs/common'; +import * as Sentry from '@sentry/nestjs'; +import type { ILocalization } from '@teable/core'; +import { getRandomString } from '@teable/core'; +import { UploadType } from '@teable/openapi'; +import type { ExportBaseProgressCallback, IExportBaseVo } from '@teable/openapi'; +import { v2DataDbTokens, v2MetaDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { + FieldKeyType, + ListTableRecordsQuery, + normalizeDotTeaExportFieldsForSelfContainedBase, + v2CoreTokens, + type IExecutionContext, + type IQueryBus, + type ListTableRecordsResult, +} from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; +import archiver from 'archiver'; +import { stringify } from 'csv-stringify/sync'; +import type { Kysely } from 'kysely'; +import { sql } from 'kysely'; +import { ClsService } from 'nestjs-cls'; +import { IStorageConfig, StorageConfig } from '../../configs/storage'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; +import type { IClsStore } from '../../types/cls'; +import type { I18nPath } from '../../types/i18n.generated'; +import { resolveBuildVersion } from '../../utils/build-version'; +import { second } from '../../utils/second'; +import StorageAdapter from '../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../attachments/plugins/storage'; +import { NotificationService } from '../notification/notification.service'; +import { V2ContainerService } from '../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../v2/v2-execution-context.factory'; +import { EXCLUDE_SYSTEM_FIELDS } from './constant'; + +type ExportDb = Kysely; + +type BaseRow = { + id: string; + name: string; + icon: string | null; + v2_enabled: boolean | null; +}; + +type TableRow = { + id: string; + name: string; + description: string | null; + icon: string | null; + db_table_name: string; + order: number; +}; + +type FieldRow = { + id: string; + name: string; + description: string | null; + options: string | null; + meta: string | null; + ai_config: string | null; + type: string; + cell_value_type: string; + is_multiple_cell_value: boolean | null; + db_field_type: string; + db_field_name: string; + not_null: boolean | null; + unique: boolean | null; + is_primary: boolean | null; + is_lookup: boolean | null; + is_conditional_lookup: boolean | null; + has_error: boolean | null; + lookup_options: string | null; + table_id: string; + order: number; + created_time: Date | string; +}; + +type ViewRow = { + id: string; + name: string; + description: string | null; + table_id: string; + type: string; + sort: string | null; + filter: string | null; + group: string | null; + options: string | null; + column_meta: string | null; + enable_share: boolean | null; + share_meta: string | null; + share_id: string | null; + is_locked: boolean | null; + order: number; +}; + +type AttachmentFileRow = { + token: string; + name: string | null; + path: string; + thumbnail_path: string | null; +}; + +type AttachmentMetadataRow = { + id: string; + token: string; + hash: string; + size: number | bigint; + mimetype: string; + path: string; + width: number | null; + height: number | null; + deleted_time: Date | string | null; + created_time: Date | string; + created_by: string; + last_modified_by: string | null; + thumbnail_path: string | null; +}; + +const csvChunkSize = 500; +const fileSuffix = 'tea'; + +const parseJson = (value: string | null | undefined): unknown | undefined => { + if (!value) return undefined; + return JSON.parse(value); +}; + +const parseJsonOrNull = (value: string | null | undefined): unknown | null => { + if (!value) return null; + return JSON.parse(value); +}; + +const toIsoString = (value: Date | string): string => + value instanceof Date ? value.toISOString() : new Date(value).toISOString(); + +const splitDbTableName = (dbTableName: string): [string, string] => { + const parts = dbTableName.split('.'); + if (parts.length === 1) return ['public', parts[0]!]; + return [parts[0]!, parts.slice(1).join('.')]; +}; + +const identifier = (name: string) => sql.id(...name.split('.')); + +const serializeCsvValue = (value: unknown): unknown => { + if (value == null) return ''; + if (value instanceof Date) return value.toISOString(); + if (typeof value === 'bigint') return value.toString(); + if (typeof value === 'object') return JSON.stringify(value); + return value; +}; + +@Injectable() +export class BaseExportV2Service { + private readonly logger = new Logger(BaseExportV2Service.name); + + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly v2ContextFactory: V2ExecutionContextFactory, + private readonly cls: ClsService, + private readonly notificationService: NotificationService, + private readonly eventEmitterService: EventEmitterService, + @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, + @StorageConfig() private readonly storageConfig: IStorageConfig + ) {} + + async exportBaseZip( + baseId: string, + includeData = true, + onProgress?: ExportBaseProgressCallback + ): Promise { + onProgress?.('preparing'); + const container = await this.v2ContainerService.getContainerForBase(baseId); + const metaDb = container.resolve(v2MetaDbTokens.db); + const dataDb = container.resolve(v2DataDbTokens.db); + const base = await this.getBase(metaDb, baseId); + const baseName = base.name; + const passThrough = new PassThrough(); + const archive = archiver('zip', { + zlib: { level: 9 }, + }); + + archive.on('warning', (err) => { + if (err.code !== 'ENOENT') { + passThrough.emit('error', err); + } + }); + archive.on('error', (err) => { + passThrough.emit('error', err); + }); + archive.pipe(passThrough); + + const token = getRandomString(12); + const bucket = StorageAdapter.getBucket(UploadType.ExportBase); + const pathDir = StorageAdapter.getDir(UploadType.ExportBase); + const exportFileName = `${baseName}.${fileSuffix}`; + const uploadPromise = this.storageAdapter.uploadFileStream( + bucket, + `${pathDir}/${token}.${fileSuffix}`, + passThrough, + { + 'Content-Type': 'application/octet-stream', + 'Content-Disposition': `attachment; filename*=UTF-8''${encodeURIComponent(exportFileName)}`, + } + ); + + try { + onProgress?.('exporting_archive'); + await this.processExportBaseZip( + metaDb, + dataDb, + container, + base, + includeData, + archive, + onProgress + ); + await archive.finalize(); + onProgress?.('uploading_archive'); + const uploadResult = await uploadPromise; + onProgress?.('generating_download_url'); + const previewUrl = await this.storageAdapter.getPreviewUrl( + bucket, + uploadResult.path, + second(this.storageConfig.tokenExpireIn), + { + 'Content-Disposition': `attachment; filename*=UTF-8''${encodeURIComponent(exportFileName)}`, + } + ); + + await this.notifyExportResult( + baseId, + { + i18nKey: 'common.email.templates.notify.exportBase.success.message', + context: { + baseName, + previewUrl, + name: exportFileName, + }, + }, + { + status: 'success', + previewUrl, + attachment: { + name: exportFileName, + path: uploadResult.path, + }, + } + ); + onProgress?.('done'); + return { previewUrl, baseName, fileName: exportFileName }; + } catch (error) { + this.captureExportError(error, { baseId, baseName, includeData }); + const message = + error instanceof Error + ? error.message + : typeof error === 'string' + ? error + : 'Unknown error'; + await this.notifyExportResult( + baseId, + { + i18nKey: 'common.email.templates.notify.exportBase.failed.message', + context: { + baseName, + errorMessage: message, + }, + }, + { + status: 'failed', + errorMessage: message, + } + ); + if (onProgress) { + throw error; + } + } + } + + private getQueryBus(container: DependencyContainer): IQueryBus { + return container.resolve(v2CoreTokens.queryBus); + } + + private async processExportBaseZip( + metaDb: ExportDb, + dataDb: ExportDb, + container: DependencyContainer, + base: BaseRow, + includeData: boolean, + archive: archiver.Archiver, + onProgress?: ExportBaseProgressCallback + ) { + const [tables, fields, views] = await Promise.all([ + this.getTables(metaDb, base.id), + this.getFields(metaDb, base.id), + this.getViews(metaDb, base.id), + ]); + onProgress?.('exporting_structure'); + const structure = await this.extendBaseStructConfig( + await this.generateBaseStructConfig(metaDb, base, tables, fields, views), + base.id + ); + archive.append(Readable.from(JSON.stringify(structure, null, 2)), { name: 'structure.json' }); + + if (includeData) { + onProgress?.('exporting_attachments'); + await this.appendAttachments(metaDb, 'attachments', tables, archive); + onProgress?.('exporting_attachment_metadata'); + await this.appendAttachmentsDataCsv(metaDb, 'attachments', tables, archive); + onProgress?.('exporting_table_data'); + await this.appendTableDataCsvs( + dataDb, + container, + 'tables', + tables, + fields, + archive, + onProgress + ); + } + + onProgress?.('exporting_extra_files'); + await this.appendExtraArchiveFiles(base.id, archive); + } + + protected async extendBaseStructConfig(structure: unknown, baseId: string): Promise { + void baseId; + return structure; + } + + protected async appendExtraArchiveFiles(baseId: string, archive: archiver.Archiver) { + void baseId; + void archive; + } + + private async getBase(db: ExportDb, baseId: string): Promise { + const result = await sql` + select "id", "name", "icon", "v2_enabled" + from "base" + where "id" = ${baseId} + and "deleted_time" is null + limit 1 + `.execute(db); + const base = result.rows[0]; + if (!base) { + throw new Error('Base not found'); + } + return base; + } + + private async getTables(db: ExportDb, baseId: string): Promise { + const result = await sql` + select "id", "name", "description", "icon", "db_table_name", "order" + from "table_meta" + where "base_id" = ${baseId} + and "deleted_time" is null + order by "order" asc + `.execute(db); + return [...result.rows]; + } + + private async getFields(db: ExportDb, baseId: string): Promise { + const result = await sql` + select f.* + from "field" f + inner join "table_meta" t on t."id" = f."table_id" + where t."base_id" = ${baseId} + and t."deleted_time" is null + and f."deleted_time" is null + order by f."order" asc + `.execute(db); + return [...result.rows]; + } + + private async getViews(db: ExportDb, baseId: string): Promise { + const result = await sql` + select v.* + from "view" v + inner join "table_meta" t on t."id" = v."table_id" + where t."base_id" = ${baseId} + and t."deleted_time" is null + and v."deleted_time" is null + order by v."order" asc + `.execute(db); + return [...result.rows]; + } + + private async generateBaseStructConfig( + db: ExportDb, + base: BaseRow, + tables: TableRow[], + fields: FieldRow[], + views: ViewRow[] + ) { + return { + id: base.id, + name: base.name, + icon: base.icon, + version: resolveBuildVersion(), + tables: tables.map((table) => ({ + id: table.id, + name: table.name, + order: table.order, + description: table.description ?? undefined, + icon: table.icon ?? undefined, + dbTableName: table.db_table_name.split('.').pop(), + fields: this.generateFieldConfig(fields.filter((field) => field.table_id === table.id)), + views: this.generateViewConfig(views.filter((view) => view.table_id === table.id)), + })), + folders: await this.generateFolderConfig(db, base.id), + nodes: await this.generateNodeConfig(db, base.id), + plugins: await this.generatePluginConfig(db, base.id, tables), + }; + } + + private generateFieldConfig(fields: FieldRow[]) { + return normalizeDotTeaExportFieldsForSelfContainedBase( + fields.map((field) => ({ + id: field.id, + name: field.name, + description: field.description ?? undefined, + type: field.type, + options: parseJson(field.options), + dbFieldName: field.db_field_name, + notNull: field.not_null ?? undefined, + unique: field.unique ?? undefined, + isPrimary: field.is_primary ?? undefined, + hasError: field.has_error ?? undefined, + order: field.order, + lookupOptions: parseJson(field.lookup_options), + isLookup: field.is_lookup ?? undefined, + isConditionalLookup: field.is_conditional_lookup ?? undefined, + aiConfig: parseJson(field.ai_config), + meta: parseJson(field.meta), + dbFieldType: field.db_field_type, + cellValueType: field.cell_value_type, + isMultipleCellValue: field.is_multiple_cell_value ?? undefined, + createdTime: toIsoString(field.created_time), + })) + ); + } + + private generateViewConfig(views: ViewRow[]) { + return views.map((view, index) => ({ + id: view.id, + name: view.name, + description: view.description ?? undefined, + type: view.type, + sort: parseJson(view.sort), + filter: parseJson(view.filter), + group: parseJson(view.group), + options: parseJson(view.options), + columnMeta: parseJsonOrNull(view.column_meta), + enableShare: view.enable_share ?? undefined, + shareMeta: parseJson(view.share_meta), + shareId: view.share_id ?? undefined, + isLocked: view.is_locked ?? undefined, + order: index, + })); + } + + private async generateFolderConfig(db: ExportDb, baseId: string) { + const result = await sql<{ id: string; name: string }>` + select "id", "name" + from "base_node_folder" + where "base_id" = ${baseId} + order by "created_time" asc + `.execute(db); + return result.rows.map((folder) => ({ + id: folder.id, + name: folder.name, + })); + } + + private async generateNodeConfig(db: ExportDb, baseId: string) { + const result = await sql<{ + id: string; + parent_id: string | null; + resource_id: string; + resource_type: string; + order: number; + }>` + select "id", "parent_id", "resource_id", "resource_type", "order" + from "base_node" + where "base_id" = ${baseId} + order by "created_time" asc + `.execute(db); + return result.rows.map((node) => ({ + id: node.id, + parentId: node.parent_id, + resourceId: node.resource_id, + resourceType: node.resource_type, + order: node.order, + })); + } + + private async generatePluginConfig(db: ExportDb, baseId: string, tables: TableRow[]) { + return { + dashboard: await this.generateDashboardConfig(db, baseId), + panel: await this.generatePluginPanelConfig(db, tables), + view: await this.generatePluginViewConfig(db, tables), + }; + } + + private async generateDashboardConfig(db: ExportDb, baseId: string) { + const dashboards = await sql<{ id: string; name: string; layout: string | null }>` + select "id", "name", "layout" + from "dashboard" + where "base_id" = ${baseId} + order by "created_time" asc + `.execute(db); + const installs = await this.getPluginInstalls( + db, + dashboards.rows.map((dashboard) => dashboard.id) + ); + + return dashboards.rows.map((dashboard) => ({ + id: dashboard.id, + name: dashboard.name, + layout: dashboard.layout ? JSON.parse(dashboard.layout) : null, + pluginInstall: installs + .filter((install) => install.position_id === dashboard.id) + .map(this.mapPluginInstall), + })); + } + + private async generatePluginPanelConfig(db: ExportDb, tables: TableRow[]) { + if (!tables.length) return []; + const tableIds = tables.map((table) => table.id); + const panels = await sql<{ id: string; name: string; layout: string | null; table_id: string }>` + select "id", "name", "layout", "table_id" + from "plugin_panel" + where "table_id" in (${sql.join(tableIds)}) + order by "created_time" asc + `.execute(db); + const installs = await this.getPluginInstalls( + db, + panels.rows.map((panel) => panel.id) + ); + + return panels.rows.map((panel) => ({ + id: panel.id, + name: panel.name, + layout: panel.layout ? JSON.parse(panel.layout) : null, + tableId: panel.table_id, + pluginInstall: installs + .filter((install) => install.position_id === panel.id) + .map(this.mapPluginInstall), + })); + } + + private async generatePluginViewConfig(db: ExportDb, tables: TableRow[]) { + if (!tables.length) return []; + const tableIds = tables.map((table) => table.id); + const views = await sql` + select v.* + from "view" v + where v."table_id" in (${sql.join(tableIds)}) + and v."type" = 'plugin' + and v."deleted_time" is null + order by v."created_time" asc + `.execute(db); + const installs = await this.getPluginInstalls( + db, + views.rows.map((view) => view.id) + ); + + return views.rows.map((view) => ({ + id: view.id, + name: view.name, + description: view.description ?? undefined, + type: view.type, + isLocked: view.is_locked ?? undefined, + tableId: view.table_id, + order: view.order, + columnMeta: parseJsonOrNull(view.column_meta), + options: parseJsonOrNull(view.options), + filter: parseJsonOrNull(view.filter), + group: parseJsonOrNull(view.group), + shareMeta: parseJsonOrNull(view.share_meta), + pluginInstall: this.mapPluginInstall( + installs.find((install) => install.position_id === view.id)! + ), + })); + } + + private async getPluginInstalls(db: ExportDb, positionIds: string[]) { + if (!positionIds.length) return []; + const result = await sql<{ + id: string; + plugin_id: string; + position_id: string; + position: string; + name: string; + storage: string | null; + }>` + select "id", "plugin_id", "position_id", "position", "name", "storage" + from "plugin_install" + where "position_id" in (${sql.join(positionIds)}) + `.execute(db); + return [...result.rows]; + } + + private mapPluginInstall(install: { + id: string; + plugin_id: string; + position_id: string; + position: string; + name: string; + storage: string | null; + }) { + return { + id: install.id, + pluginId: install.plugin_id, + positionId: install.position_id, + position: install.position, + name: install.name, + storage: install.storage ? JSON.parse(install.storage) : null, + }; + } + + private async appendTableDataCsvs( + db: ExportDb, + container: DependencyContainer, + filePath: string, + tables: TableRow[], + fields: FieldRow[], + archive: archiver.Archiver, + onProgress?: ExportBaseProgressCallback + ) { + const queryBus = this.getQueryBus(container); + const context = await this.v2ContextFactory.createContext(container); + + for (const [index, table] of tables.entries()) { + onProgress?.('table_data_started', table.name, { + type: 'progress', + phase: 'table_data_started', + tableId: table.id, + tableName: table.name, + tableIndex: index + 1, + totalTables: tables.length, + }); + const tableFields = fields.filter((field) => field.table_id === table.id); + await this.appendTableDataCsv( + db, + queryBus, + context, + archive, + filePath, + table, + tableFields, + onProgress + ); + onProgress?.('table_data_done', table.name, { + type: 'progress', + phase: 'table_data_done', + tableId: table.id, + tableName: table.name, + tableIndex: index + 1, + totalTables: tables.length, + }); + } + } + + private async appendTableDataCsv( + db: ExportDb, + queryBus: IQueryBus, + context: IExecutionContext, + archive: archiver.Archiver, + filePath: string, + table: TableRow, + fields: FieldRow[], + onProgress?: ExportBaseProgressCallback + ) { + const columns = await this.getPhysicalColumnNames(db, table.db_table_name); + const buttonDbFieldNames = new Set( + fields.filter((field) => field.type === 'button').map((field) => field.db_field_name) + ); + const fieldDbNames = fields + .map((field) => field.db_field_name) + .filter((name) => !buttonDbFieldNames.has(name)); + const headers = [ + ...columns.filter( + (name) => !EXCLUDE_SYSTEM_FIELDS.includes(name) && !buttonDbFieldNames.has(name) + ), + ...fieldDbNames.filter((name) => !columns.includes(name)), + ]; + const physicalHeaders = headers.filter((header) => columns.includes(header)); + + archive.append( + Readable.from( + this.createTableDataCsvStream( + db, + queryBus, + context, + table, + fields, + headers, + physicalHeaders, + onProgress + ) + ), + { name: `${filePath}/${table.id}.csv` } + ); + } + + private async *createTableDataCsvStream( + db: ExportDb, + queryBus: IQueryBus, + context: IExecutionContext, + table: TableRow, + fields: FieldRow[], + headers: string[], + physicalHeaders: string[], + onProgress?: ExportBaseProgressCallback + ): AsyncGenerator { + yield `${headers.join(',')}\n`; + let offset = 0; + let processedRows = 0; + let hasMore = true; + + while (hasMore) { + const rawRows = await this.getRawRows(db, table.db_table_name, physicalHeaders, offset); + if (rawRows.length === 0) { + hasMore = false; + break; + } + const recordFields = await this.getRecordFieldsByDbName( + queryBus, + context, + table.id, + fields, + rawRows.flatMap((row) => (typeof row.__id === 'string' ? [row.__id] : [])) + ); + + const rows = rawRows.map((rawRow) => { + const recordId = String(rawRow.__id ?? ''); + const fieldValues = recordFields.get(recordId) ?? {}; + return Object.fromEntries( + headers.map((header) => [ + header, + serializeCsvValue(rawRow[header] !== undefined ? rawRow[header] : fieldValues[header]), + ]) + ); + }); + + yield stringify(rows, { columns: headers }); + offset += csvChunkSize; + processedRows += rawRows.length; + onProgress?.('table_data_progress', table.name, { + type: 'progress', + phase: 'table_data_progress', + tableId: table.id, + tableName: table.name, + processedRows, + batchProcessedRows: rawRows.length, + currentBatch: Math.ceil(offset / csvChunkSize), + }); + } + } + + private async getPhysicalColumnNames(db: ExportDb, dbTableName: string): Promise { + const [schemaName, tableName] = splitDbTableName(dbTableName); + const result = await sql<{ column_name: string }>` + select "column_name" + from "information_schema"."columns" + where "table_schema" = ${schemaName} + and "table_name" = ${tableName} + order by "ordinal_position" asc + `.execute(db); + return result.rows.map((row) => row.column_name); + } + + private async getRawRows( + db: ExportDb, + dbTableName: string, + headers: string[], + offset: number + ): Promise>> { + const selectedColumns = headers.filter((header) => header !== '' && !header.includes('\u0000')); + if (!selectedColumns.length) return []; + const result = await sql>` + select ${sql.join(selectedColumns.map((column) => sql.id(column)))} + from ${identifier(dbTableName)} + order by "__auto_number" asc + limit ${csvChunkSize} + offset ${offset} + `.execute(db); + return [...result.rows]; + } + + private async getRecordFieldsByDbName( + queryBus: IQueryBus, + context: IExecutionContext, + tableId: string, + fields: FieldRow[], + recordIds: string[] + ): Promise>> { + if (recordIds.length === 0) return new Map(); + + const commandResult = ListTableRecordsQuery.create({ + tableId, + fieldKeyType: FieldKeyType.Id, + selectedRecordIds: recordIds, + limit: recordIds.length, + ignoreViewQuery: true, + }); + if (commandResult.isErr()) { + throw new Error(commandResult.error.message); + } + + const result = await queryBus.execute( + context, + commandResult.value + ); + if (result.isErr()) { + throw new Error(result.error.message); + } + + const dbFieldNameByFieldId = new Map(fields.map((field) => [field.id, field.db_field_name])); + return new Map( + result.value.records.map((record) => [ + record.id, + Object.fromEntries( + Object.entries(record.fields).flatMap(([fieldId, value]) => { + const dbFieldName = dbFieldNameByFieldId.get(fieldId); + return dbFieldName ? [[dbFieldName, value]] : []; + }) + ), + ]) + ); + } + + private async appendAttachments( + db: ExportDb, + filePath: string, + tables: TableRow[], + archive: archiver.Archiver + ) { + if (!tables.length) return; + const tableIds = tables.map((table) => table.id); + const result = await sql` + select at."token", at."name", a."path", a."thumbnail_path" + from "attachments_table" at + inner join "attachments" a on a."token" = at."token" + where at."table_id" in (${sql.join(tableIds)}) + and a."deleted_time" is null + `.execute(db); + const bucket = StorageAdapter.getBucket(UploadType.Table); + + for (const attachment of result.rows) { + const suffix = attachment.name?.split('.').pop(); + const archivePath = `${filePath}/${attachment.token}${suffix ? `.${suffix}` : ''}`; + await this.appendFileToArchive(archive, bucket, attachment.path, archivePath); + } + + const prefix = `${filePath}/thumbnail__`; + for (const attachment of result.rows.filter((row) => row.thumbnail_path)) { + const suffix = attachment.name?.split('.').pop() || 'jpg'; + const thumbnails = JSON.parse(attachment.thumbnail_path ?? '{}') as Record; + for (const thumbnailPath of Object.values(thumbnails).filter(Boolean)) { + const fileName = thumbnailPath.split('/').pop(); + if (fileName) { + await this.appendFileToArchive( + archive, + bucket, + thumbnailPath, + `${prefix}${fileName}.${suffix}` + ); + } + } + } + } + + private async appendAttachmentsDataCsv( + db: ExportDb, + filePath: string, + tables: TableRow[], + archive: archiver.Archiver + ) { + if (!tables.length) return; + const tableIds = tables.map((table) => table.id); + const result = await sql` + select distinct + a."id", + a."token", + a."hash", + a."size", + a."mimetype", + a."path", + a."width", + a."height", + a."deleted_time", + a."created_time", + a."created_by", + a."last_modified_by", + a."thumbnail_path" + from "attachments" a + inner join "attachments_table" at on at."token" = a."token" + where at."table_id" in (${sql.join(tableIds)}) + and a."deleted_time" is null + `.execute(db); + if (!result.rows.length) return; + + const csvStream = new PassThrough(); + const attachments = result.rows.map((row) => ({ + id: row.id, + token: row.token, + hash: row.hash, + size: row.size, + mimetype: row.mimetype, + path: row.path, + width: row.width, + height: row.height, + deletedTime: row.deleted_time, + createdTime: row.created_time, + createdBy: row.created_by, + lastModifiedBy: row.last_modified_by, + thumbnailPath: row.thumbnail_path, + })); + const headers = Object.keys(attachments[0]!); + csvStream.write(`${headers.join(',')}\n`); + archive.append(csvStream, { name: `${filePath}/attachments.csv` }); + csvStream.write( + stringify( + attachments.map((row) => { + const values = row as Record; + return Object.fromEntries( + headers.map((header) => [header, serializeCsvValue(values[header])]) + ); + }), + { columns: headers } + ) + ); + csvStream.end(); + } + + private async appendFileToArchive( + archive: archiver.Archiver, + bucket: string, + s3Path: string, + archivePath: string + ): Promise { + try { + const stream = await this.storageAdapter.downloadFile(bucket, s3Path); + archive.append(stream, { name: archivePath }); + return true; + } catch (error) { + this.logger.error( + `Failed to export file ${s3Path} to ${archivePath}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + return false; + } + } + + private captureExportError( + error: unknown, + context: { baseId: string; includeData: boolean; baseName?: string } + ) { + const err = error instanceof Error ? error : new Error(String(error)); + const userId = this.cls.get('user.id'); + + Sentry.withScope((scope) => { + scope.setTag('feature', 'base-export-v2'); + scope.setContext('base-export-v2', { + ...context, + userId, + }); + scope.setLevel?.('error'); + Sentry.captureException(err); + }); + + this.logger.error(`export v2 base zip failed: ${err.message}`, err.stack ?? undefined); + } + + private async notifyExportResult( + baseId: string, + message: string | ILocalization, + result?: { + status: 'success' | 'failed'; + previewUrl?: string; + attachment?: { name: string; path: string }; + errorMessage?: string; + } + ) { + const userId = this.cls.get('user.id'); + await this.eventEmitterService.emit(Events.BASE_EXPORT_COMPLETE, { + status: result?.status, + previewUrl: result?.previewUrl, + attachment: result?.attachment, + errorMessage: result?.errorMessage, + }); + await this.notificationService.sendExportBaseResultNotify({ + baseId, + toUserId: userId, + message, + }); + } +} diff --git a/apps/nestjs-backend/src/features/base/base-export.service.ts b/apps/nestjs-backend/src/features/base/base-export.service.ts index d612305081..850368fdfc 100644 --- a/apps/nestjs-backend/src/features/base/base-export.service.ts +++ b/apps/nestjs-backend/src/features/base/base-export.service.ts @@ -10,11 +10,16 @@ import type { IConditionalLookupOptions, } from '@teable/core'; import { FieldType, getRandomString, ViewType, isLinkLookupOptions } from '@teable/core'; +import { DataPrismaService } from '@teable/db-data-prisma'; import type { Field, View, TableMeta, Base } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PluginPosition, UploadType } from '@teable/openapi'; -import type { BaseNodeResourceType, IBaseJson } from '@teable/openapi'; +import type { + BaseNodeResourceType, + ExportBaseProgressCallback, + IBaseJson, + IExportBaseVo, +} from '@teable/openapi'; import archiver from 'archiver'; import { stringify } from 'csv-stringify/sync'; import { Knex } from 'knex'; @@ -27,6 +32,7 @@ import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; import { EventEmitterService } from '../../event-emitter/event-emitter.service'; import { Events } from '../../event-emitter/events'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import type { IClsStore } from '../../types/cls'; import type { I18nPath } from '../../types/i18n.generated'; @@ -68,7 +74,6 @@ export class BaseExportService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, private readonly cls: ClsService, private readonly notificationService: NotificationService, private readonly eventEmitterService: EventEmitterService, @@ -76,7 +81,8 @@ export class BaseExportService { @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, @StorageConfig() private readonly storageConfig: IStorageConfig, - @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly databaseRouter: DatabaseRouter ) {} private captureExportError( @@ -159,8 +165,13 @@ export class BaseExportService { } } - async exportBaseZip(baseId: string, includeData = true) { + async exportBaseZip( + baseId: string, + includeData = true, + onProgress?: ExportBaseProgressCallback + ): Promise { let baseName: string | undefined; + onProgress?.('preparing'); try { ({ name: baseName } = await this.prismaService.base.findFirstOrThrow({ where: { @@ -223,10 +234,11 @@ export class BaseExportService { ); try { + onProgress?.('exporting_archive'); await this.prismaService.$tx( async (prisma) => { await prisma.$executeRawUnsafe('SET TRANSACTION READ ONLY'); - await this.pipeArchive(archive, baseId, includeData); + await this.pipeArchive(archive, baseId, includeData, onProgress); }, { isolationLevel: Prisma.TransactionIsolationLevel.RepeatableRead, @@ -234,8 +246,10 @@ export class BaseExportService { } ); archive.finalize(); + onProgress?.('uploading_archive'); const uploadResult = await uploadPromise; const { path } = uploadResult; + onProgress?.('generating_download_url'); const previewUrl = await this.storageAdapter.getPreviewUrl( StorageAdapter.getBucket(UploadType.ExportBase), path, @@ -253,7 +267,16 @@ export class BaseExportService { name: exportFileName, }, }; - this.notifyExportResult(baseId, message, previewUrl); + this.notifyExportResult(baseId, message, { + status: 'success', + previewUrl, + attachment: { + name: exportFileName, + path, + }, + }); + onProgress?.('done'); + return { previewUrl, baseName, fileName: exportFileName }; } catch (e) { this.captureExportError(e, { stage: 'processExport', @@ -269,16 +292,32 @@ export class BaseExportService { errorMessage: e.message, }, }; - this.notifyExportResult(baseId, message); + this.notifyExportResult(baseId, message, { + status: 'failed', + errorMessage: e.message, + }); + } + if (onProgress) { + throw e; } } } - async pipeArchive(archive: archiver.Archiver, baseId: string, includeData: boolean) { - await this.processExportBaseZip(baseId, includeData, archive); + async pipeArchive( + archive: archiver.Archiver, + baseId: string, + includeData: boolean, + onProgress?: ExportBaseProgressCallback + ) { + await this.processExportBaseZip(baseId, includeData, archive, onProgress); } - async processExportBaseZip(baseId: string, includeData: boolean, archive: archiver.Archiver) { + async processExportBaseZip( + baseId: string, + includeData: boolean, + archive: archiver.Archiver, + onProgress?: ExportBaseProgressCallback + ) { const prisma = this.prismaService.txClient(); // 1. get all raw info const baseRaw = await prisma.base.findUniqueOrThrow({ @@ -318,6 +357,7 @@ export class BaseExportService { }); // 2. generate base structure json + onProgress?.('exporting_structure'); const structure = await this.generateBaseStructConfig({ baseRaw, tableRaws, @@ -332,6 +372,7 @@ export class BaseExportService { // 4 export data if (includeData) { + onProgress?.('exporting_attachments'); this.logger.log(`export base ${baseRaw.id}/${baseRaw.name}: Start exporting attachments`); // 4.0 export attachments await this.appendAttachments('attachments', tableRaws, archive); @@ -339,6 +380,7 @@ export class BaseExportService { `export base ${baseRaw.id}/${baseRaw.name}: End exporting attachments data csv` ); + onProgress?.('exporting_attachment_metadata'); // 4.1 export attachments data .csv this.logger.log( `export base ${baseRaw.id}/${baseRaw.name}: Start exporting attachments data csv` @@ -348,6 +390,7 @@ export class BaseExportService { `export base ${baseRaw.id}/${baseRaw.name}: End exporting attachments data csv` ); + onProgress?.('exporting_table_data'); this.logger.log(`export base ${baseRaw.id}/${baseRaw.name}: Start exporting table data csv`); // 4.2 export table data csv @@ -357,7 +400,15 @@ export class BaseExportService { crossBaseRelativeFieldIds.has(id) ); - for (const tableRaw of tableRaws) { + for (const [index, tableRaw] of tableRaws.entries()) { + onProgress?.('table_data_started', tableRaw.name, { + type: 'progress', + phase: 'table_data_started', + tableId: tableRaw.id, + tableName: tableRaw.name, + tableIndex: index + 1, + totalTables: tableRaws.length, + }); const crossBaseFieldRaws = crossBaseRelativeFieldsRaws.filter( ({ tableId }) => tableId === tableRaw.id ); @@ -374,23 +425,33 @@ export class BaseExportService { 'tables', tableRaw, crossBaseFieldRaws, - excludeDbFieldNames + excludeDbFieldNames, + onProgress ); + onProgress?.('table_data_done', tableRaw.name, { + type: 'progress', + phase: 'table_data_done', + tableId: tableRaw.id, + tableName: tableRaw.name, + tableIndex: index + 1, + totalTables: tableRaws.length, + }); } - const linkFieldInstances = fieldRaws + const linkFieldRaws = fieldRaws .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) - .filter(({ id }) => !crossBaseRelativeFieldIds.has(id)) - .map((f) => createFieldInstanceByRaw(f)); + .filter(({ id }) => !crossBaseRelativeFieldIds.has(id)); // 5. export junction csv for link fields const junctionTableName = [] as string[]; - for (const linkField of linkFieldInstances) { + for (const linkFieldRaw of linkFieldRaws) { + const linkField = createFieldInstanceByRaw(linkFieldRaw); const { options } = linkField; const { fkHostTableName, selfKeyName, foreignKeyName } = options as ILinkFieldOptions; if (fkHostTableName.includes('junction_') && !junctionTableName.includes(fkHostTableName)) { await this.appendJunctionCsv( 'tables', + linkFieldRaw.tableId, fkHostTableName, selfKeyName, foreignKeyName, @@ -419,6 +480,10 @@ export class BaseExportService { includedWorkflowIds, // Root node IDs - nodes that should have their parentId set to null rootNodeIds, + // When set, fields whose foreign base lives in a different space are also + // degraded to text even when allowCrossBase=true (same-space cross-base + // remains a real link). + destSpaceId, }: { baseRaw: Base; tableRaws: TableMeta[]; @@ -432,8 +497,13 @@ export class BaseExportService { includedWorkflowIds?: string[]; excludedTableIds?: string[]; rootNodeIds?: string[]; + destSpaceId?: string; }) { const { name: baseName, icon: baseIcon, id: baseId } = baseRaw; + const crossSpaceForeignBaseIds = await this.computeCrossSpaceForeignBaseIds( + fieldRaws, + destSpaceId + ); const tables = [] as IBaseJson['tables']; for (const table of tableRaws) { const { name, description, order, id, icon, dbTableName } = table; @@ -450,7 +520,8 @@ export class BaseExportService { tableObject.fields = this.generateFieldConfig( currentTableFields, allowCrossBase, - excludedTableIds + excludedTableIds, + crossSpaceForeignBaseIds ); tableObject.views = this.generateViewConfig(viewRaws.filter(({ tableId }) => tableId === id)); tables.push(tableObject); @@ -562,11 +633,14 @@ export class BaseExportService { filePath: string, tableRaw: TableMeta, crossBaseRelativeFields: Field[], - excludeDbFieldNames: string[] + excludeDbFieldNames: string[], + onProgress?: ExportBaseProgressCallback ) { const { dbTableName, id } = tableRaw; const csvStream = new PassThrough(); - const prisma = this.dataPrismaService.txClient(); + const prisma = await this.databaseRouter.dataPrismaExecutorForTable(id, { + useTransaction: true, + }); const columnInfoQuery = this.dbProvider.columnInfo(dbTableName); const columnInfo = await prisma.$queryRawUnsafe<{ name: string }[]>(columnInfoQuery); @@ -586,6 +660,7 @@ export class BaseExportService { csvStream.write(`${headerRow}\n`); let offset = 0; + let processedRows = 0; let hasMoreData = true; archive.append(csvStream, { name: `${filePath}/${id}.csv` }); @@ -610,6 +685,7 @@ export class BaseExportService { // 2. write csv content while (hasMoreData) { const csvChunk = await this.getCsvChunk( + prisma, dbTableName, offset, crossBaseRelativeFields, @@ -624,6 +700,16 @@ export class BaseExportService { }); csvStream.write(csvString); offset += BaseExportService.CSV_CHUNK; + processedRows += csvChunk.length; + onProgress?.('table_data_progress', tableRaw.name, { + type: 'progress', + phase: 'table_data_progress', + tableId: tableRaw.id, + tableName: tableRaw.name, + processedRows, + batchProcessedRows: csvChunk.length, + currentBatch: Math.ceil(offset / BaseExportService.CSV_CHUNK), + }); } csvStream.end(); } @@ -706,13 +792,16 @@ export class BaseExportService { private async appendJunctionCsv( filePath: string, + tableId: string, fkHostTableName: string, selfKeyName: string, foreignKeyName: string, archive: archiver.Archiver ) { const csvStream = new PassThrough(); - const prisma = this.dataPrismaService.txClient(); + const prisma = await this.databaseRouter.dataPrismaExecutorForTable(tableId, { + useTransaction: true, + }); const columnInfoQuery = this.dbProvider.columnInfo(fkHostTableName); const columnInfo = await prisma.$queryRawUnsafe<{ name: string }[]>(columnInfoQuery); @@ -750,6 +839,7 @@ export class BaseExportService { // 2. write csv content while (hasMoreData) { const csvChunk = await this.getJunctionChunk( + prisma, fkHostTableName, offset, [selfKeyName, foreignKeyName], @@ -769,12 +859,13 @@ export class BaseExportService { } private async getCsvChunk( + prisma: { $queryRawUnsafe(query: string, ...values: unknown[]): Promise }, dbTableName: string, offset: number, crossBaseRelativeFields: Field[], excludeFieldNames: string[] ) { - const rawRecords = await this.getChunkRecords(dbTableName, offset); + const rawRecords = await this.getChunkRecords(prisma, dbTableName, offset); // 1. clear unless fields const records = rawRecords.map((record) => omit(record, excludeFieldNames)); // 2. convert to csv value @@ -784,12 +875,12 @@ export class BaseExportService { } private async getJunctionChunk( + prisma: { $queryRawUnsafe(query: string, ...values: unknown[]): Promise }, fkHostTableName: string, offset: number, convertFields: [string, string], excludeFieldNames: string[] ) { - const prisma = this.dataPrismaService.txClient(); const recordsQuery = await this.knex(fkHostTableName) .select('*') .limit(BaseExportService.CSV_CHUNK) @@ -814,8 +905,11 @@ export class BaseExportService { }); } - private async getChunkRecords(dbTableName: string, offset: number) { - const prisma = this.dataPrismaService.txClient(); + private async getChunkRecords( + prisma: { $queryRawUnsafe(query: string, ...values: unknown[]): Promise }, + dbTableName: string, + offset: number + ) { const recordsQuery = await this.knex(dbTableName) .select('*') .limit(BaseExportService.CSV_CHUNK) @@ -869,7 +963,8 @@ export class BaseExportService { private generateFieldConfig( fieldRaws: Field[], allowCrossBase = false, - excludedTableIds?: string[] + excludedTableIds?: string[], + crossSpaceForeignBaseIds: Set = new Set() ) { const fields = fieldRaws.map((fieldRaw) => createFieldInstanceByRaw(fieldRaw)); const createdTimeMap = fieldRaws.reduce( @@ -880,7 +975,11 @@ export class BaseExportService { {} as Record ); - const crossBaseRelativeFields = this.getCrossBaseFields(fieldRaws, allowCrossBase); + const crossBaseRelativeFields = this.getCrossBaseFields( + fieldRaws, + allowCrossBase, + crossSpaceForeignBaseIds + ); const disconnectedFields = this.getDisconnectedFields( fieldRaws, @@ -1039,7 +1138,41 @@ export class BaseExportService { ] as IBaseJson['tables'][number]['fields']; } - private getCrossBaseFields(fieldRaws: Field[], allowCrossBase = false) { + private async computeCrossSpaceForeignBaseIds( + fieldRaws: Field[], + destSpaceId?: string + ): Promise> { + if (!destSpaceId) return new Set(); + const foreignBaseIds = new Set(); + for (const f of fieldRaws) { + try { + const opts = f.options ? JSON.parse(f.options as string) : null; + const baseId = opts?.baseId; + if (typeof baseId === 'string') foreignBaseIds.add(baseId); + } catch { + // ignore + } + try { + const lopts = f.lookupOptions ? JSON.parse(f.lookupOptions as string) : null; + const baseId = lopts?.baseId; + if (typeof baseId === 'string') foreignBaseIds.add(baseId); + } catch { + // ignore + } + } + if (foreignBaseIds.size === 0) return new Set(); + const bases = await this.prismaService.txClient().base.findMany({ + where: { id: { in: Array.from(foreignBaseIds) }, deletedTime: null }, + select: { id: true, spaceId: true }, + }); + return new Set(bases.filter((b) => b.spaceId !== destSpaceId).map((b) => b.id)); + } + + private getCrossBaseFields( + fieldRaws: Field[], + allowCrossBase = false, + crossSpaceForeignBaseIds: Set = new Set() + ) { const fields = fieldRaws.map((fieldRaw) => createFieldInstanceByRaw(fieldRaw)); const createdTimeMap = fieldRaws.reduce( (acc, field) => { @@ -1048,26 +1181,47 @@ export class BaseExportService { }, {} as Record ); + + // When allowCrossBase=false, every cross-base field is degraded (legacy behavior). + // When allowCrossBase=true, only fields whose foreign base lives in a different + // space than the destination (crossSpaceForeignBaseIds) are degraded — same-space + // cross-base relationships are preserved. + const shouldDegradeByBaseId = (baseId: string | undefined): boolean => { + if (!baseId) return false; + if (!allowCrossBase) return true; + return crossSpaceForeignBaseIds.has(baseId); + }; + + const linkDegradeIds = new Set( + fields + .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) + .filter(({ options }) => + shouldDegradeByBaseId((options as ILinkFieldOptions | undefined)?.baseId) + ) + .map(({ id }) => id) + ); + + const omitDegradedKeys = [ + 'options', + 'lookupOptions', + 'isLookup', + 'isConditionalLookup', + 'isMultipleCellValue', + ] as const; + const crossBaseLinkFields = fields .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) .filter(({ options }) => Boolean((options as ILinkFieldOptions)?.baseId)) .map((field, index) => { + const degrade = linkDegradeIds.has(field.id); const res = { ...pick(field, BaseExportService.EXPORT_FIELD_COLUMNS), - type: allowCrossBase ? field.type : FieldType.SingleLineText, + type: degrade ? FieldType.SingleLineText : field.type, createdTime: createdTimeMap[field.id], order: fieldRaws[index].order, }; - return allowCrossBase - ? res - : omit(res, [ - 'options', - 'lookupOptions', - 'isLookup', - 'isConditionalLookup', - 'isMultipleCellValue', - ]); + return degrade ? omit(res, omitDegradedKeys) : res; }); // fields which rely on the cross base link fields (link-based lookup/rollup) @@ -1092,24 +1246,25 @@ export class BaseExportService { return crossBaseLinkFields.map(({ id }) => id).includes(lookupOptions.linkFieldId); }) .map((field, index) => { + let degrade: boolean; + if (field.type === FieldType.Link && (field.options as ILinkFieldOptions)?.baseId) { + degrade = shouldDegradeByBaseId((field.options as ILinkFieldOptions).baseId); + } else if (field.lookupOptions && isLinkLookupOptions(field.lookupOptions)) { + degrade = linkDegradeIds.has(field.lookupOptions.linkFieldId); + } else { + degrade = !allowCrossBase; + } + const res = { ...pick(field, BaseExportService.EXPORT_FIELD_COLUMNS), - type: allowCrossBase ? field.type : FieldType.SingleLineText, + type: degrade ? FieldType.SingleLineText : field.type, createdTime: createdTimeMap[field.id], order: fieldRaws[index].order, - dbFieldType: allowCrossBase ? field.dbFieldType : 'TEXT', - cellValueType: allowCrossBase ? field.cellValueType : 'string', + dbFieldType: degrade ? 'TEXT' : field.dbFieldType, + cellValueType: degrade ? 'string' : field.cellValueType, }; - return allowCrossBase - ? res - : omit(res, [ - 'options', - 'lookupOptions', - 'isLookup', - 'isConditionalLookup', - 'isMultipleCellValue', - ]); + return degrade ? omit(res, omitDegradedKeys) : res; }); const alreadyHandledIds = new Set([ @@ -1141,24 +1296,22 @@ export class BaseExportService { return false; }) .map((field, index) => { + const conditionalBaseId = + field.isLookup && field.isConditionalLookup + ? (field.lookupOptions as IConditionalLookupOptions | undefined)?.baseId + : (field.options as IConditionalRollupFieldOptions | undefined)?.baseId; + const degrade = shouldDegradeByBaseId(conditionalBaseId); + const res = { ...pick(field, BaseExportService.EXPORT_FIELD_COLUMNS), - type: allowCrossBase ? field.type : FieldType.SingleLineText, + type: degrade ? FieldType.SingleLineText : field.type, createdTime: createdTimeMap[field.id], order: fieldRaws[index].order, - dbFieldType: allowCrossBase ? field.dbFieldType : 'TEXT', - cellValueType: allowCrossBase ? field.cellValueType : 'string', + dbFieldType: degrade ? 'TEXT' : field.dbFieldType, + cellValueType: degrade ? 'string' : field.cellValueType, }; - return allowCrossBase - ? res - : omit(res, [ - 'options', - 'lookupOptions', - 'isLookup', - 'isConditionalLookup', - 'isMultipleCellValue', - ]); + return degrade ? omit(res, omitDegradedKeys) : res; }); return [ @@ -1483,11 +1636,19 @@ export class BaseExportService { private async notifyExportResult( baseId: string, message: string | ILocalization, - previewUrl?: string + result?: { + status: 'success' | 'failed'; + previewUrl?: string; + attachment?: { name: string; path: string }; + errorMessage?: string; + } ) { const userId = this.cls.get('user.id'); await this.eventEmitterService.emit(Events.BASE_EXPORT_COMPLETE, { - previewUrl, + status: result?.status, + previewUrl: result?.previewUrl, + attachment: result?.attachment, + errorMessage: result?.errorMessage, }); await this.notificationService.sendExportBaseResultNotify({ baseId: baseId, diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts new file mode 100644 index 0000000000..8d41623670 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts @@ -0,0 +1,92 @@ +import Knex from 'knex'; +import { vi } from 'vitest'; + +import { BaseImportCsvQueueProcessor } from './base-import-csv.processor'; + +describe('BaseImportCsvQueueProcessor', () => { + it('writes imported record history into the routed data DB internal schema', async () => { + const executedSql: string[] = []; + const dataKnex = Knex({ client: 'pg' }); + const dataPrisma = { + $queryRawUnsafe: vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ name: '__id' }, { name: 'fldText' }]), + $executeRawUnsafe: vi.fn(async (sql: string) => { + executedSql.push(sql); + return 1; + }), + }; + const prismaService = { + tableMeta: { + findUniqueOrThrow: vi.fn().mockResolvedValue({ dbTableName: 'bse_data.tbl_imported' }), + }, + txClient: vi.fn().mockReturnValue({ + attachmentsTable: { + createMany: vi.fn().mockResolvedValue(undefined), + }, + }), + }; + const dataDbClientManager = { + dataPrismaForBase: vi.fn().mockResolvedValue(dataPrisma), + dataKnexForBase: vi.fn().mockResolvedValue(dataKnex), + getDataDatabaseForBase: vi.fn().mockResolvedValue({ + url: 'postgresql://user:pass@example.test:5432/data?schema=teable_internal', + }), + }; + const processor = Object.create(BaseImportCsvQueueProcessor.prototype) as { + handleChunk: ( + results: Record[], + config: { + baseId: string; + tableId: string; + userId: string; + fieldIdMap: Record; + viewIdMap: Record; + fkMap: Record; + attachmentsFields: { dbFieldName: string; id: string }[]; + notNullFieldMap: Map; + fieldDbNameMap: Map; + }, + excludeDbFieldNames: string[] + ) => Promise; + prismaService: typeof prismaService; + dbProvider: { + getForeignKeysInfo: ReturnType; + columnInfo: ReturnType; + }; + dataDbClientManager: typeof dataDbClientManager; + }; + + processor.prismaService = prismaService; + processor.dbProvider = { + getForeignKeysInfo: vi.fn().mockReturnValue('SELECT * FROM foreign_keys'), + columnInfo: vi.fn().mockReturnValue('SELECT * FROM columns'), + }; + processor.dataDbClientManager = dataDbClientManager; + + await processor.handleChunk( + [{ __id: 'recImported', fldText: 'Imported value' }], + { + baseId: 'bseImport', + tableId: 'tblImport', + userId: 'usrImport', + fieldIdMap: {}, + viewIdMap: {}, + fkMap: {}, + attachmentsFields: [], + notNullFieldMap: new Map(), + fieldDbNameMap: new Map([['fldText', 'fldMappedText']]), + }, + [] + ); + + expect(executedSql.some((sql) => sql.includes('"bse_data"."tbl_imported"'))).toBe(true); + expect(executedSql.some((sql) => sql.includes('"teable_internal"."record_history"'))).toBe( + true + ); + expect(executedSql.some((sql) => sql.includes('insert into "record_history"'))).toBe(false); + + await dataKnex.destroy(); + }); +}); diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts index 9ac8a7e7a0..c1c4a01424 100644 --- a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts @@ -2,22 +2,24 @@ import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq'; import { Injectable, Logger } from '@nestjs/common'; import type { IAttachmentCellValue, ILinkFieldOptions } from '@teable/core'; -import { DbFieldType, FieldType, generateAttachmentId } from '@teable/core'; +import { + DbFieldType, + FieldType, + generateAttachmentId, + generateRecordHistoryId, +} from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { IBaseJson, ImportBaseRo } from '@teable/openapi'; import { CreateRecordAction, UploadType } from '@teable/openapi'; import { Queue, Job } from 'bullmq'; import * as csvParser from 'csv-parser'; -import { Knex } from 'knex'; -import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; import * as unzipper from 'unzipper'; import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; import { Events } from '../../../event-emitter/events'; -import { DATA_KNEX } from '../../../global/knex/knex.module'; +import { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import type { IClsStore } from '../../../types/cls'; import StorageAdapter from '../../attachments/plugins/adapter'; import { InjectStorageAdapter } from '../../attachments/plugins/storage'; @@ -25,6 +27,17 @@ import { PersistedComputedBackfillService } from '../../record/computed/services import { BatchProcessor } from '../BatchProcessor.class'; import { EXCLUDE_SYSTEM_FIELDS } from '../constant'; import { BaseImportJunctionCsvQueueProcessor } from './base-import-junction.processor'; + +type IDataPrismaExecutor = { + $queryRawUnsafe(query: string, ...values: unknown[]): Promise; + $executeRawUnsafe(query: string, ...values: unknown[]): Promise; +}; + +type IDataPrismaScopedClient = IDataPrismaExecutor & { + $tx?: (fn: (prisma: IDataPrismaExecutor) => Promise) => Promise; + $transaction?: (fn: (prisma: IDataPrismaExecutor) => Promise) => Promise; +}; + interface IBaseImportCsvJob { path: string; userId: string; @@ -55,15 +68,14 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, private readonly baseImportJunctionCsvQueueProcessor: BaseImportJunctionCsvQueueProcessor, private readonly persistedComputedBackfillService: PersistedComputedBackfillService, - @InjectModel(DATA_KNEX) private readonly knex: Knex, @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, @InjectQueue(BASE_IMPORT_CSV_QUEUE) public readonly queue: Queue, @InjectDbProvider() private readonly dbProvider: IDbProvider, private readonly cls: ClsService, - private readonly eventEmitterService: EventEmitterService + private readonly eventEmitterService: EventEmitterService, + private readonly dataDbClientManager: DataDbClientManager ) { super(); } @@ -89,7 +101,7 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { } private async handleBaseImportCsv(job: Job): Promise { - const { path, userId, tableIdMap, fieldIdMap, viewIdMap, structure, fkMap } = job.data; + const { path, userId, baseId, tableIdMap, fieldIdMap, viewIdMap, structure, fkMap } = job.data; const csvStream = await this.storageAdapter.downloadFile( StorageAdapter.getBucket(UploadType.Import), path @@ -163,12 +175,16 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { }); } }); + const fieldDbNameMap = new Map( + table?.fields?.map(({ dbFieldName, id }) => [dbFieldName, fieldIdMap[id] ?? id]) ?? [] + ); const batchProcessor = new BatchProcessor>(async (chunk) => { totalRecordsCount += chunk.length; await this.handleChunk( chunk, { + baseId, tableId: tableIdMap[tableId], userId, fieldIdMap, @@ -176,6 +192,7 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { fkMap, attachmentsFields, notNullFieldMap, + fieldDbNameMap, }, excludeDbFieldNames ); @@ -248,9 +265,29 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { ); } + private async dataTransaction( + dataPrisma: IDataPrismaScopedClient, + fn: (prisma: IDataPrismaExecutor) => Promise + ) { + if (dataPrisma.$tx) { + return await dataPrisma.$tx(fn); + } + + if (dataPrisma.$transaction) { + return await dataPrisma.$transaction(fn); + } + + return await fn(dataPrisma); + } + + private getDataDbInternalSchema(dataDbUrl: string) { + return new URL(dataDbUrl).searchParams.get('schema') || 'public'; + } + private async handleChunk( results: Record[], config: { + baseId: string; tableId: string; userId: string; fieldIdMap: Record; @@ -258,10 +295,20 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { fkMap: Record; attachmentsFields: { dbFieldName: string; id: string }[]; notNullFieldMap: Map; + fieldDbNameMap: Map; }, excludeDbFieldNames: string[] ) { - const { tableId, userId, fieldIdMap, attachmentsFields, fkMap, notNullFieldMap } = config; + const { + baseId, + tableId, + userId, + fieldIdMap, + attachmentsFields, + fkMap, + notNullFieldMap, + fieldDbNameMap, + } = config; const { dbTableName } = await this.prismaService.tableMeta.findUniqueOrThrow({ where: { id: tableId }, select: { @@ -285,8 +332,24 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { recordId: string; fieldId: string; }[]; - - await this.dataPrismaService.$tx(async (prisma) => { + const recordHistoryList: { + id: string; + table_id: string; + record_id: string; + field_id: string; + before: string; + after: string; + created_by: string; + }[] = []; + + const dataPrisma = (await this.dataDbClientManager.dataPrismaForBase( + baseId + )) as IDataPrismaScopedClient; + const dataKnex = await this.dataDbClientManager.dataKnexForBase(baseId); + const dataDb = await this.dataDbClientManager.getDataDatabaseForBase(baseId); + const dataDbInternalSchema = this.getDataDbInternalSchema(dataDb.url); + + await this.dataTransaction(dataPrisma, async (prisma) => { // delete foreign keys if(exist) then duplicate table data const foreignKeysInfoSql = this.dbProvider.getForeignKeysInfo(dbTableName); const foreignKeysInfo = await prisma.$queryRawUnsafe< @@ -305,7 +368,7 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { allForeignKeyInfos.push(...newForeignKeyInfos); for (const { constraint_name, column_name, dbTableName } of allForeignKeyInfos) { - const dropForeignKeyQuery = this.knex.schema + const dropForeignKeyQuery = dataKnex.schema .alterTable(dbTableName, (table) => { table.dropForeign(column_name, constraint_name); }) @@ -373,6 +436,24 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { }); }); } + + if (key.startsWith('__') && !key.startsWith('__fk_')) { + return; + } + + const sourceFieldId = key.startsWith('__fk_') ? key.slice(5) : key; + const fieldId = fieldIdMap[sourceFieldId] ?? fieldDbNameMap.get(key) ?? sourceFieldId; + if (fieldId && value !== '' && value != null) { + recordHistoryList.push({ + id: generateRecordHistoryId(), + table_id: tableId, + record_id: res['__id'] as string, + field_id: fieldId, + before: JSON.stringify({ data: null }), + after: JSON.stringify({ data: value }), + created_by: userId, + }); + } }); // default value set @@ -389,7 +470,7 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { .filter((name) => name.startsWith('__row_')); for (const name of lackingColumns) { - const sql = this.knex.schema + const sql = dataKnex.schema .alterTable(dbTableName, (table) => { table.double(name); }) @@ -398,8 +479,17 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { } } - const sql = this.knex.table(dbTableName).insert(recordsToInsert).toQuery(); + const sql = dataKnex.table(dbTableName).insert(recordsToInsert).toQuery(); await prisma.$executeRawUnsafe(sql); + + if (recordHistoryList.length) { + const historySql = dataKnex + .withSchema(dataDbInternalSchema) + .insert(recordHistoryList) + .into('record_history') + .toQuery(); + await prisma.$executeRawUnsafe(historySql); + } }); // restore foreign keys with NOT VALID @@ -412,7 +502,7 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { referenced_column_name: referencedColumnName, } of allForeignKeyInfos) { const [schema, tableName] = dbTableName.split('.'); - const addForeignKeyQuery = this.knex + const addForeignKeyQuery = dataKnex .raw( 'ALTER TABLE ??.?? ADD CONSTRAINT ?? FOREIGN KEY (??) REFERENCES ??.??(??) NOT VALID', [ @@ -426,7 +516,7 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { ] ) .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(addForeignKeyQuery); + await dataPrisma.$executeRawUnsafe(addForeignKeyQuery); } await this.updateAttachmentTable(userId, attachmentsTableData); @@ -479,6 +569,7 @@ export class BaseImportCsvQueueProcessor extends WorkerHost { await this.baseImportJunctionCsvQueueProcessor.queue.add( 'import_base_junction_csv', { + baseId: job.data.baseId, tableIdMap, fieldIdMap, path, diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-junction.processor.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-junction.processor.ts index a4ea016ad3..a0e6baafd8 100644 --- a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-junction.processor.ts +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-junction.processor.ts @@ -8,26 +8,34 @@ import { import type { ILinkFieldOptions } from '@teable/core'; import { FieldType } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { IBaseJson } from '@teable/openapi'; import { UploadType } from '@teable/openapi'; import type { Job } from 'bullmq'; import { Queue } from 'bullmq'; import * as csvParser from 'csv-parser'; -import { Knex } from 'knex'; -import { InjectModel } from 'nest-knexjs'; import * as unzipper from 'unzipper'; import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; -import { DATA_KNEX } from '../../../global/knex/knex.module'; +import { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import StorageAdapter from '../../attachments/plugins/adapter'; import { InjectStorageAdapter } from '../../attachments/plugins/storage'; import { createFieldInstanceByRaw } from '../../field/model/factory'; import { PersistedComputedBackfillService } from '../../record/computed/services/persisted-computed-backfill.service'; import { BatchProcessor } from '../BatchProcessor.class'; +type IDataPrismaExecutor = { + $queryRawUnsafe(query: string, ...values: unknown[]): Promise; + $executeRawUnsafe(query: string, ...values: unknown[]): Promise; +}; + +type IDataPrismaScopedClient = IDataPrismaExecutor & { + $tx?: (fn: (prisma: IDataPrismaExecutor) => Promise) => Promise; + $transaction?: (fn: (prisma: IDataPrismaExecutor) => Promise) => Promise; +}; + interface IBaseImportJunctionCsvJob { path: string; + baseId: string; tableIdMap: Record; fieldIdMap: Record; structure: IBaseJson; @@ -43,13 +51,12 @@ export class BaseImportJunctionCsvQueueProcessor extends WorkerHost { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, private readonly persistedComputedBackfillService: PersistedComputedBackfillService, - @InjectModel(DATA_KNEX) private readonly knex: Knex, @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, @InjectQueue(BASE_IMPORT_JUNCTION_CSV_QUEUE) public readonly queue: Queue, - @InjectDbProvider() private readonly dbProvider: IDbProvider + @InjectDbProvider() private readonly dbProvider: IDbProvider, + private readonly dataDbClientManager: DataDbClientManager ) { super(); } @@ -63,10 +70,10 @@ export class BaseImportJunctionCsvQueueProcessor extends WorkerHost { this.processedJobs.add(jobId); - const { path, tableIdMap, fieldIdMap, structure } = job.data; + const { path, baseId, tableIdMap, fieldIdMap, structure } = job.data; try { - await this.importJunctionChunk(path, fieldIdMap, structure); + await this.importJunctionChunk(path, baseId, fieldIdMap, structure); await this.persistedComputedBackfillService.recomputeForTables(Object.values(tableIdMap)); } catch (error) { this.logger.error( @@ -78,6 +85,7 @@ export class BaseImportJunctionCsvQueueProcessor extends WorkerHost { private async importJunctionChunk( path: string, + baseId: string, fieldIdMap: Record, structure: IBaseJson ) { @@ -173,7 +181,7 @@ export class BaseImportJunctionCsvQueueProcessor extends WorkerHost { } = junctionInfo; const batchProcessor = new BatchProcessor>((chunk) => - this.handleJunctionChunk(chunk, targetFkHostTableName) + this.handleJunctionChunk(baseId, chunk, targetFkHostTableName) ); entry @@ -216,7 +224,23 @@ export class BaseImportJunctionCsvQueueProcessor extends WorkerHost { }); } + private async dataTransaction( + dataPrisma: IDataPrismaScopedClient, + fn: (prisma: IDataPrismaExecutor) => Promise + ) { + if (dataPrisma.$tx) { + return await dataPrisma.$tx(fn); + } + + if (dataPrisma.$transaction) { + return await dataPrisma.$transaction(fn); + } + + return await fn(dataPrisma); + } + private async handleJunctionChunk( + baseId: string, results: Record[], targetFkHostTableName: string ) { @@ -229,7 +253,12 @@ export class BaseImportJunctionCsvQueueProcessor extends WorkerHost { dbTableName: string; }[]; - await this.dataPrismaService.$tx(async (prisma) => { + const dataPrisma = (await this.dataDbClientManager.dataPrismaForBase( + baseId + )) as IDataPrismaScopedClient; + const dataKnex = await this.dataDbClientManager.dataKnexForBase(baseId); + + await this.dataTransaction(dataPrisma, async (prisma) => { // delete foreign keys if(exist) then duplicate table data const foreignKeysInfoSql = this.dbProvider.getForeignKeysInfo(targetFkHostTableName); const foreignKeysInfo = await prisma.$queryRawUnsafe< @@ -248,7 +277,7 @@ export class BaseImportJunctionCsvQueueProcessor extends WorkerHost { allForeignKeyInfos.push(...newForeignKeyInfos); for (const { constraint_name, column_name, dbTableName } of allForeignKeyInfos) { - const dropForeignKeyQuery = this.knex.schema + const dropForeignKeyQuery = dataKnex.schema .alterTable(dbTableName, (table) => { table.dropForeign(column_name, constraint_name); }) @@ -257,7 +286,7 @@ export class BaseImportJunctionCsvQueueProcessor extends WorkerHost { await prisma.$executeRawUnsafe(dropForeignKeyQuery); } - const sql = this.knex.table(targetFkHostTableName).insert(results).toQuery(); + const sql = dataKnex.table(targetFkHostTableName).insert(results).toQuery(); try { await prisma.$executeRawUnsafe(sql); } catch (error) { @@ -289,7 +318,7 @@ export class BaseImportJunctionCsvQueueProcessor extends WorkerHost { referenced_column_name: referencedColumnName, } of allForeignKeyInfos) { const [schema, tableName] = dbTableName.split('.'); - const addForeignKeyQuery = this.knex + const addForeignKeyQuery = dataKnex .raw( 'ALTER TABLE ??.?? ADD CONSTRAINT ?? FOREIGN KEY (??) REFERENCES ??.??(??) NOT VALID', [ diff --git a/apps/nestjs-backend/src/features/base/base-import.service.spec.ts b/apps/nestjs-backend/src/features/base/base-import.service.spec.ts index f7c73c91d4..45deeffa89 100644 --- a/apps/nestjs-backend/src/features/base/base-import.service.spec.ts +++ b/apps/nestjs-backend/src/features/base/base-import.service.spec.ts @@ -1,7 +1,11 @@ +import type { Readable } from 'stream'; import { DbFieldType, FieldType } from '@teable/core'; +import type { IBaseJson, ImportBaseRo } from '@teable/openapi'; import type { RestoreRecordInput } from '@teable/v2-core'; +import archiver from 'archiver'; +import { vi } from 'vitest'; -import { BaseImportService } from './base-import.service'; +import { BaseImportService, formatBaseImportError } from './base-import.service'; interface IRestoreRecordInputBuilder { toRestoreRecordInput( @@ -26,6 +30,15 @@ interface IRestoreRecordInputBuilder { ): RestoreRecordInput; } +interface IProcessStructureService { + processStructure( + zipStream: Readable, + importBaseRo: Pick, + onProgress?: (...args: unknown[]) => void + ): Promise; + createBaseStructure: ReturnType; +} + const dbTableName = 'bse_test.tbl_test'; const jsonColumnName = 'json_col'; const textColumnName = 'text_col'; @@ -34,7 +47,147 @@ const textCellValue = 'plain text'; const createService = () => Object.create(BaseImportService.prototype) as IRestoreRecordInputBuilder; +const createZipStream = (structure: IBaseJson) => { + const archive = archiver('zip', { zlib: { level: 0 } }); + archive.append(JSON.stringify(structure), { name: 'structure.json' }); + void archive.finalize(); + return archive; +}; + describe('BaseImportService', () => { + describe('formatBaseImportError', () => { + it('falls back when an Error has an empty message', () => { + expect(formatBaseImportError(new Error(''), 'Unknown import error')).toBe( + 'Unknown import error' + ); + }); + + it('uses domain error code and details when message is empty', () => { + expect( + formatBaseImportError( + { + code: 'dottea.parse_failed', + message: '', + details: { file: 'structure.json' }, + tags: ['unexpected'], + }, + 'Failed to import dottea structure' + ) + ).toBe('Failed to import dottea structure: dottea.parse_failed - {"file":"structure.json"}'); + }); + + it('uses network error code with a specific fallback when message is empty', () => { + expect( + formatBaseImportError({ code: 'ECONNREFUSED', message: '' }, 'Failed to connect data DB') + ).toBe('Failed to connect data DB: ECONNREFUSED'); + }); + }); + + describe('processStructure', () => { + it('passes transaction-aware data DB routing into structure creation', async () => { + const service = Object.create(BaseImportService.prototype) as IProcessStructureService; + const structure = { + id: 'bseSource', + name: 'Source base', + tables: [], + plugins: {}, + folders: [], + nodes: [], + } as unknown as IBaseJson; + const expectedResult = { + base: { id: 'bseImported' }, + tableIdMap: {}, + fieldIdMap: {}, + viewIdMap: {}, + fkMap: {}, + structure, + }; + + service.createBaseStructure = vi.fn().mockResolvedValue(expectedResult); + + await expect( + service.processStructure(createZipStream(structure), { spaceId: 'spcImport' }) + ).resolves.toBe(expectedResult); + + expect(service.createBaseStructure).toHaveBeenCalledWith( + 'spcImport', + structure, + undefined, + undefined, + undefined, + undefined, + { useTransaction: true } + ); + }); + + it('creates imported base schemas through the space routed data client', async () => { + const createdBase = { + id: 'bseImported', + name: 'Imported base', + spaceId: 'spcImport', + order: 1, + }; + const baseCreate = vi.fn().mockResolvedValue(createdBase); + const baseUpdate = vi.fn().mockResolvedValue({ + ...createdBase, + name: 'Imported base', + }); + const routedExecute = vi.fn().mockResolvedValue(0); + const fallbackExecute = vi.fn().mockResolvedValue(0); + const service = Object.create(BaseImportService.prototype) as { + getMaxOrder: ReturnType; + createBase: ( + spaceId: string, + name: string, + icon: string | undefined, + routingOptions: { useTransaction: true } + ) => Promise; + cls: unknown; + prismaService: unknown; + dbProvider: unknown; + dataDbClientManager: unknown; + dataPrismaService: unknown; + }; + + service.getMaxOrder = vi.fn().mockResolvedValue(0); + service.cls = { get: vi.fn().mockReturnValue('usrImport') }; + service.prismaService = { + txClient: vi.fn().mockReturnValue({ + base: { + create: baseCreate, + update: baseUpdate, + }, + }), + }; + service.dbProvider = { + createSchema: vi.fn().mockReturnValue(['CREATE SCHEMA "bseImported"']), + }; + service.dataDbClientManager = { + dataPrismaForSpace: vi.fn().mockResolvedValue({ + txClient: vi.fn().mockReturnValue({ + $executeRawUnsafe: routedExecute, + }), + }), + }; + service.dataPrismaService = { + $executeRawUnsafe: fallbackExecute, + }; + + await expect( + service.createBase('spcImport', 'Imported base', 'icon', { useTransaction: true }) + ).resolves.toMatchObject({ + id: 'bseImported', + name: 'Imported base', + }); + + expect(service.dataDbClientManager.dataPrismaForSpace).toHaveBeenCalledWith('spcImport', { + useTransaction: true, + }); + expect(routedExecute).toHaveBeenCalledWith('CREATE SCHEMA "bseImported"'); + expect(fallbackExecute).not.toHaveBeenCalled(); + }); + }); + describe('toRestoreRecordInput', () => { it('serializes JSON extra column values for v2 dottea row restore', () => { const service = createService(); diff --git a/apps/nestjs-backend/src/features/base/base-import.service.ts b/apps/nestjs-backend/src/features/base/base-import.service.ts index 6fb3ffa51b..b195cb1a98 100644 --- a/apps/nestjs-backend/src/features/base/base-import.service.ts +++ b/apps/nestjs-backend/src/features/base/base-import.service.ts @@ -19,7 +19,6 @@ import { ViewType, } from '@teable/core'; import { PrismaService, ProvisionState } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { ICreateBaseVo, IBaseJson, @@ -56,6 +55,7 @@ import { type RestoreRecordsStreamResult, type UpdateManyStreamBatchInput, } from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; import * as csvParser from 'csv-parser'; import { Knex } from 'knex'; @@ -68,6 +68,8 @@ import * as unzipper from 'unzipper'; import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { DataDbClientManager } from '../../global/data-db-client-manager.service'; +import type { IDataDbRoutingOptions } from '../../global/data-db-client-manager.service'; import type { IClsStore } from '../../types/cls'; import StorageAdapter from '../attachments/plugins/adapter'; import { InjectStorageAdapter } from '../attachments/plugins/storage'; @@ -98,16 +100,72 @@ export type BaseImportProgressCallback = ( detail?: string ) => void; +type IDataPrismaExecutor = { + $executeRawUnsafe(query: string, ...values: unknown[]): Promise; +}; + +type IDataPrismaScopedClient = IDataPrismaExecutor & { + txClient?: () => IDataPrismaExecutor; +}; + const tableDataImportBatchSize = 100; const linkFieldImportBatchSize = 25; +const stringifyErrorDetails = (details: unknown): string | undefined => { + if (details === undefined || details === null) { + return undefined; + } + if (typeof details === 'string') { + return details.trim() || undefined; + } + try { + return JSON.stringify(details); + } catch { + return String(details); + } +}; + +const formatBaseImportObjectError = (error: object, fallback: string): string => { + const candidate = error as { + code?: unknown; + details?: unknown; + message?: unknown; + name?: unknown; + }; + const message = typeof candidate.message === 'string' ? candidate.message.trim() : ''; + const code = typeof candidate.code === 'string' ? candidate.code.trim() : ''; + const details = stringifyErrorDetails(candidate.details); + + if (message) { + return code ? `${message} (${code})` : message; + } + if (code) { + return details ? `${fallback}: ${code} - ${details}` : `${fallback}: ${code}`; + } + if (error instanceof Error && typeof candidate.name === 'string' && candidate.name !== 'Error') { + return `${fallback}: ${candidate.name}`; + } + return fallback; +}; + +export const formatBaseImportError = (error: unknown, fallback = 'Import failed'): string => { + if (typeof error === 'string') { + return error.trim() || fallback; + } + + if (error && typeof error === 'object') { + return formatBaseImportObjectError(error, fallback); + } + + return fallback; +}; + @Injectable() export class BaseImportService { private logger = new Logger(BaseImportService.name); constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, private readonly cls: ClsService, private readonly tableService: TableService, private readonly fieldDuplicateService: FieldDuplicateService, @@ -119,7 +177,8 @@ export class BaseImportService { @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, private readonly eventEmitter: EventEmitter2, private readonly v2ContainerService: V2ContainerService, - private readonly v2ContextFactory: V2ExecutionContextFactory + private readonly v2ContextFactory: V2ExecutionContextFactory, + private readonly dataDbClientManager: DataDbClientManager ) {} private async getMaxOrder(spaceId: string) { @@ -130,7 +189,12 @@ export class BaseImportService { return spaceAggregate._max.order || 0; } - private async createBase(spaceId: string, name: string, icon?: string) { + private async createBase( + spaceId: string, + name: string, + icon?: string, + routingOptions?: IDataDbRoutingOptions + ) { const userId = this.cls.get('user.id'); const order = (await this.getMaxOrder(spaceId)) + 1; @@ -156,10 +220,15 @@ export class BaseImportService { try { const sqlList = this.dbProvider.createSchema(base.id); if (sqlList) { + const scopedDataPrisma = (await this.dataDbClientManager.dataPrismaForSpace( + spaceId, + routingOptions + )) as IDataPrismaScopedClient; + const dataPrisma = scopedDataPrisma.txClient?.() ?? scopedDataPrisma; for (const sql of sqlList) { // Keep schema creation visible to the subsequent data-plane DDL/insert steps even when // import structure creation is wrapped in an outer shared meta transaction. - await this.dataPrismaService.$executeRawUnsafe(sql); + await dataPrisma.$executeRawUnsafe(sql); } } @@ -178,13 +247,56 @@ export class BaseImportService { } } - private async createBaseV2( + async createBaseV2( db: Kysely, spaceId: string, name: string, - icon?: string + icon?: string, + baseId?: string, + updateExistingBase: boolean = true ): Promise { const userId = this.cls.get('user.id'); + if (baseId) { + const existingResult = await sql<{ + id: string; + name: string; + icon: string | null; + space_id: string; + }>` + select "id", "name", "icon", "space_id" + from "base" + where "id" = ${baseId} + and "deleted_time" is null + limit 1 + `.execute(db); + const existing = existingResult.rows[0]; + if (!existing) { + throw new Error(`Base not found: ${baseId}`); + } + if (updateExistingBase) { + await sql` + update "base" + set + "name" = ${name || 'Untitled Base'}, + "icon" = ${icon ?? null}, + "last_modified_by" = ${userId}, + "last_modified_time" = ${new Date()} + where "id" = ${baseId} + `.execute(db); + return { + id: existing.id, + name: name || 'Untitled Base', + spaceId: existing.space_id, + }; + } + + return { + id: existing.id, + name: existing.name, + spaceId: existing.space_id, + }; + } + const base = { id: generateBaseId(), name: name || 'Untitled Base', @@ -223,7 +335,11 @@ export class BaseImportService { `.execute(trx); }); - return base; + return { + id: base.id, + name: base.name, + spaceId: base.spaceId, + }; } async importBase(importBaseRo: ImportBaseRo, onProgress?: BaseImportProgressCallback) { @@ -303,7 +419,14 @@ export class BaseImportService { ); const structure = await this.readDotTeaStructure(structureStream); onProgress?.('creating_base', structure.name); - const container = await this.v2ContainerService.getContainer(); + let container: DependencyContainer; + try { + container = await this.v2ContainerService.getContainerForSpace(spaceId); + } catch (error) { + throw new Error( + formatBaseImportError(error, `Failed to connect space data database for ${spaceId}`) + ); + } const commandBus = container.resolve(v2CoreTokens.commandBus); const queryBus = container.resolve(v2CoreTokens.queryBus); const tableRecordRepository = container.resolve( @@ -311,7 +434,7 @@ export class BaseImportService { ); const unitOfWork = container.resolve(v2CoreTokens.unitOfWork); const db = container.resolve>(v2PostgresDbTokens.db); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const base = await this.createBaseV2(db, spaceId, structure.name, structure.icon || undefined); const dotTeaStream = await this.storageAdapter.downloadFile( @@ -326,7 +449,7 @@ export class BaseImportService { }); if (commandResult.isErr()) { - throw new Error(commandResult.error.message); + throw new Error(formatBaseImportError(commandResult.error, 'Invalid dottea import command')); } const result = await commandBus.execute< @@ -335,7 +458,7 @@ export class BaseImportService { >(context, commandResult.value); if (result.isErr()) { - throw new Error(result.error.message); + throw new Error(formatBaseImportError(result.error, 'Failed to import dottea structure')); } const { tableIdMap, fieldIdMap, viewIdMap } = result.value; @@ -382,7 +505,7 @@ export class BaseImportService { }; } - private async restoreBaseExtrasV2( + async restoreBaseExtrasV2( db: Kysely, baseId: string, structure: IBaseJson, @@ -1174,7 +1297,9 @@ export class BaseImportService { enqueueDeferredComputedUpdates: true, }); if (commandResult.isErr()) { - throw new Error(commandResult.error.message); + throw new Error( + formatBaseImportError(commandResult.error, 'Invalid table data import command') + ); } const result = await commandBus.execute< @@ -1182,7 +1307,7 @@ export class BaseImportService { RestoreRecordsStreamResult >(context, commandResult.value); if (result.isErr()) { - throw new Error(result.error.message); + throw new Error(formatBaseImportError(result.error, 'Failed to import table data')); } for await (const event of result.value) { @@ -1199,7 +1324,7 @@ export class BaseImportService { } if (event.id === 'error') { - throw new Error(event.message); + throw new Error(formatBaseImportError(event.message, 'Failed to import table data')); } onProgress?.({ @@ -1465,7 +1590,9 @@ export class BaseImportService { for await (const batchResult of this.createTableLinkFieldUpdateBatchStream(entry, config)) { if (batchResult.isErr()) { - throw new Error(batchResult.error.message); + throw new Error( + formatBaseImportError(batchResult.error, 'Invalid link field import batch') + ); } const result = await unitOfWork.withTransaction(context, async (transactionContext) => @@ -1476,7 +1603,7 @@ export class BaseImportService { }) ); if (result.isErr()) { - throw new Error(result.error.message); + throw new Error(formatBaseImportError(result.error, 'Failed to import link fields')); } onLinkBatchUpdated(result.value.totalUpdated); @@ -1501,11 +1628,11 @@ export class BaseImportService { } private toRestoreRecordInput( - row: Record, + row: Record, config: Awaited>, viewIdMap: Record ): RestoreRecordInput { - const recordId = row.__id || generateRecordId(); + const recordId = typeof row.__id === 'string' && row.__id ? row.__id : generateRecordId(); const fields: Record = {}; const extraColumnValues: Record = {}; const orders: Record = {}; @@ -1561,10 +1688,14 @@ export class BaseImportService { ...(Object.keys(orders).length ? { orders } : {}), ...(row.__version ? { version: Number(row.__version) } : {}), ...(row.__auto_number ? { autoNumber: Number(row.__auto_number) } : {}), - ...(row.__created_time ? { createdTime: row.__created_time } : {}), - ...(row.__created_by ? { createdBy: row.__created_by } : {}), - ...(row.__last_modified_time ? { lastModifiedTime: row.__last_modified_time } : {}), - ...(row.__last_modified_by ? { lastModifiedBy: row.__last_modified_by } : {}), + ...(row.__created_time ? { createdTime: this.toRestoreString(row.__created_time) } : {}), + ...(row.__created_by ? { createdBy: this.toRestoreString(row.__created_by) } : {}), + ...(row.__last_modified_time + ? { lastModifiedTime: this.toRestoreString(row.__last_modified_time) } + : {}), + ...(row.__last_modified_by + ? { lastModifiedBy: this.toRestoreString(row.__last_modified_by) } + : {}), ...(Object.keys(extraColumnValues).length ? { extraColumnValues } : {}), }; } @@ -1699,9 +1830,19 @@ export class BaseImportService { } private normalizeDotTeaCsvValue( - value: string, + value: unknown, field?: { dbFieldType?: string; isMultipleCellValue?: boolean; notNull?: boolean } ): unknown { + if (typeof value !== 'string') { + if (value != null || !field?.notNull) { + return value; + } + return this.getNotNullDefault( + field.dbFieldType || DbFieldType.Text, + Boolean(field.isMultipleCellValue) + ); + } + if (value !== '') { switch (this.normalizeDbFieldType(field?.dbFieldType)) { case DbFieldType.Integer: { @@ -1760,6 +1901,10 @@ export class BaseImportService { return JSON.stringify(value); } + private toRestoreString(value: unknown) { + return value instanceof Date ? value.toISOString() : String(value); + } + private getNotNullDefault(dbFieldType: string, isMultipleCellValue: boolean): unknown { switch (this.normalizeDbFieldType(dbFieldType)) { case DbFieldType.Integer: @@ -1843,7 +1988,8 @@ export class BaseImportService { undefined, undefined, undefined, - onProgress + onProgress, + { useTransaction: true } ); resolve(result); } catch (error) { @@ -1919,7 +2065,8 @@ export class BaseImportService { baseId?: string, skipCreateBaseNodes?: boolean, duplicateMode: BaseDuplicateMode = BaseDuplicateMode.Normal, - onProgress?: BaseImportProgressCallback + onProgress?: BaseImportProgressCallback, + routingOptions?: IDataDbRoutingOptions ) { const { name, icon, tables, plugins, folders } = structure; @@ -1937,7 +2084,7 @@ export class BaseImportService { spaceId: true, }, }) - : await this.createBase(spaceId, name, icon || undefined); + : await this.createBase(spaceId, name, icon || undefined, routingOptions); this.logger.log(`base-duplicate-service: Duplicate base successfully`); // update base icon and name (skip when copying into an existing base) @@ -1970,7 +2117,8 @@ export class BaseImportService { ({ tableIdMap, fieldIdMap, viewIdMap, fkMap } = await this.createTables( newBase.id, effectiveTables as IBaseJson['tables'], - onProgress + onProgress, + routingOptions )); } finally { this.cls.set('skipFieldComputation', false); @@ -2036,7 +2184,8 @@ export class BaseImportService { private async createTables( baseId: string, tables: IBaseJson['tables'], - onProgress?: BaseImportProgressCallback + onProgress?: BaseImportProgressCallback, + routingOptions?: IDataDbRoutingOptions ) { const tableIdMap: Record = {}; // Build a name lookup: oldTableId → tableName @@ -2060,7 +2209,8 @@ export class BaseImportService { tables, tableIdMap, tableNameMap, - onProgress + onProgress, + routingOptions ); this.logger.log(`base-duplicate-service: Duplicate table fields successfully`); @@ -2076,7 +2226,8 @@ export class BaseImportService { tables: IBaseJson['tables'], tableIdMap: Record, tableNameMap?: Record, - onProgress?: BaseImportProgressCallback + onProgress?: BaseImportProgressCallback, + routingOptions?: IDataDbRoutingOptions ) { const fieldMap: Record = {}; const fkMap: Record = {}; @@ -2149,13 +2300,17 @@ export class BaseImportService { }; emitFieldProgress('creating_common_fields', commonFields); - await this.fieldDuplicateService.createCommonFields(commonFields, fieldMap); + await this.fieldDuplicateService.createCommonFields(commonFields, fieldMap, routingOptions); emitFieldProgress('creating_button_fields', buttonFields); - await this.fieldDuplicateService.createButtonFields(buttonFields, fieldMap); + await this.fieldDuplicateService.createButtonFields(buttonFields, fieldMap, routingOptions); emitFieldProgress('creating_formula_fields', primaryFormulaFields); - await this.fieldDuplicateService.createTmpPrimaryFormulaFields(primaryFormulaFields, fieldMap); + await this.fieldDuplicateService.createTmpPrimaryFormulaFields( + primaryFormulaFields, + fieldMap, + routingOptions + ); // main fix formula dbField type await this.fieldDuplicateService.repairPrimaryFormulaFields(primaryFormulaFields, fieldMap); @@ -2166,14 +2321,27 @@ export class BaseImportService { emitFieldProgress('creating_primary_dependency_fields', primaryDependencyFields); await this.fieldDuplicateService.bootstrapPrimaryDependencyFields( primaryDependencyFields, - fieldMap + fieldMap, + routingOptions ); emitFieldProgress('creating_link_fields', linkFields); - await this.fieldDuplicateService.createLinkFields(linkFields, tableIdMap, fieldMap, fkMap); + await this.fieldDuplicateService.createLinkFields( + linkFields, + tableIdMap, + fieldMap, + fkMap, + routingOptions + ); emitFieldProgress('creating_lookup_fields', dependencyFields); - await this.fieldDuplicateService.createDependencyFields(dependencyFields, tableIdMap, fieldMap); + await this.fieldDuplicateService.createDependencyFields( + dependencyFields, + tableIdMap, + fieldMap, + 'base', + routingOptions + ); // fix formula expression' field map await this.fieldDuplicateService.repairPrimaryFormulaFields(primaryFormulaFields, fieldMap); diff --git a/apps/nestjs-backend/src/features/base/base-query/base-query.service.ts b/apps/nestjs-backend/src/features/base/base-query/base-query.service.ts index 69f69adbf0..d16e267a4e 100644 --- a/apps/nestjs-backend/src/features/base/base-query/base-query.service.ts +++ b/apps/nestjs-backend/src/features/base/base-query/base-query.service.ts @@ -2,7 +2,6 @@ import { Injectable, Logger } from '@nestjs/common'; import type { IAttachmentCellValue } from '@teable/core'; import { CellFormat, FieldType, HttpErrorCode } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { BaseQueryColumnType, BaseQueryJoinType } from '@teable/openapi'; import type { IBaseQueryJoin, IBaseQuery, IBaseQueryVo, IBaseQueryColumn } from '@teable/openapi'; import { Knex } from 'knex'; @@ -11,6 +10,7 @@ import { ClsService } from 'nestjs-cls'; import { CustomHttpException } from '../../../custom.exception'; import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { DatabaseRouter } from '../../../global/database-router.service'; import { DATA_KNEX } from '../../../global/knex/knex.module'; import type { IClsStore } from '../../../types/cls'; import { FieldService } from '../../field/field.service'; @@ -37,7 +37,7 @@ export class BaseQueryService { private readonly fieldService: FieldService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly cls: ClsService, private readonly recordService: RecordService ) {} @@ -137,9 +137,8 @@ export class BaseQueryService { const { queryBuilder, fieldMap } = await this.parseBaseQuery(baseId, baseQuery, 0); const query = queryBuilder.toQuery(); this.logger.log('baseQuery SQL: ', query); - const rows = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ [key in string]: unknown }[]>(query) + const rows = await this.databaseRouter + .queryDataPrismaForBase<{ [key in string]: unknown }[]>(baseId, query) .catch((e) => { this.logger.error(e); throw new CustomHttpException('Query failed', HttpErrorCode.VALIDATION_ERROR, { diff --git a/apps/nestjs-backend/src/features/base/base.controller.ts b/apps/nestjs-backend/src/features/base/base.controller.ts index f6251fe69b..3f9e600de5 100644 --- a/apps/nestjs-backend/src/features/base/base.controller.ts +++ b/apps/nestjs-backend/src/features/base/base.controller.ts @@ -57,7 +57,9 @@ import type { IDbConnectionVo, IGetBaseAllVo, IGetBasePermissionVo, + IDuplicateBaseCheckVo, IGetBaseVo, + IMoveBaseCheckVo, IGetSharedBaseVo, IImportBaseVo, IListBaseCollaboratorUserVo, @@ -81,8 +83,10 @@ import { V2FeatureGuard } from '../canary/guards/v2-feature.guard'; import { V2IndicatorInterceptor } from '../canary/interceptors/v2-indicator.interceptor'; import { CollaboratorService } from '../collaborator/collaborator.service'; import { InvitationService } from '../invitation/invitation.service'; +import { BaseDuplicateService } from './base-duplicate.service'; +import { BaseExportV2Service } from './base-export-v2.service'; import { BaseExportService } from './base-export.service'; -import { BaseImportService } from './base-import.service'; +import { BaseImportService, formatBaseImportError } from './base-import.service'; import { BaseService } from './base.service'; import { DbConnectionService } from './db-connection.service'; @@ -91,10 +95,12 @@ export class BaseController { constructor( private readonly baseService: BaseService, private readonly baseExportService: BaseExportService, + private readonly baseExportV2Service: BaseExportV2Service, private readonly baseImportService: BaseImportService, private readonly dbConnectionService: DbConnectionService, private readonly collaboratorService: CollaboratorService, private readonly invitationService: InvitationService, + private readonly baseDuplicateService: BaseDuplicateService, private readonly cls: ClsService ) {} @@ -176,7 +182,7 @@ export class BaseController { } catch (error) { sendEvent({ type: 'error', - message: error instanceof Error ? error.message : 'Unknown import error', + message: formatBaseImportError(error, 'Unknown import error'), }); } finally { clearInterval(heartbeat); @@ -185,6 +191,9 @@ export class BaseController { } @Post('duplicate') + @UseV2Feature('duplicateBase') + @UseGuards(V2FeatureGuard) + @UseInterceptors(V2IndicatorInterceptor) @Permissions('base|create') @ResourceMeta('spaceId', 'body') @EmitControllerEvent(Events.BASE_CREATE) @@ -192,9 +201,84 @@ export class BaseController { @Body(new ZodValidationPipe(duplicateBaseRoSchema)) duplicateBaseRo: IDuplicateBaseRo ): Promise { + if (this.cls.get('useV2')) { + return await this.baseService.duplicateBaseV2(duplicateBaseRo); + } return await this.baseService.duplicateBase(duplicateBaseRo); } + @Post('duplicate-stream') + @UseV2Feature('duplicateBase') + @UseGuards(V2FeatureGuard) + @UseInterceptors(V2IndicatorInterceptor) + @Permissions('base|create') + @ResourceMeta('spaceId', 'body') + async duplicateBaseStream( + @Body(new ZodValidationPipe(duplicateBaseRoSchema)) + duplicateBaseRo: IDuplicateBaseRo, + @Res() res: ExpressResponse + ) { + const sseHeartbeatMs = 15_000; + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache, no-transform'); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('X-Accel-Buffering', 'no'); + res.flushHeaders(); + + const isStreamClosed = () => res.writableEnded || res.destroyed; + // eslint-disable-next-line sonarjs/no-identical-functions + const sendEvent = (data: unknown) => { + if (isStreamClosed()) return; + res.write(`data: ${JSON.stringify(data)}\n\n`); + (res as ExpressResponse & { flush?: () => void }).flush?.(); + }; + const heartbeat = setInterval(() => { + if (isStreamClosed()) return; + res.write(': ping\n\n'); + (res as ExpressResponse & { flush?: () => void }).flush?.(); + }, sseHeartbeatMs); + res.on('close', () => clearInterval(heartbeat)); + + try { + sendEvent({ type: 'progress', phase: 'duplicate_started' }); + const result = this.cls.get('useV2') + ? await this.baseService.duplicateBaseV2WithProgress( + duplicateBaseRo, + (phase: string | { phase: string }, detail?: string) => { + sendEvent( + typeof phase === 'string' + ? { type: 'progress', phase, detail } + : { type: 'progress', ...phase } + ); + } + ) + : { base: await this.baseService.duplicateBase(duplicateBaseRo) }; + + sendEvent({ type: 'done', data: result.base }); + } catch (error) { + sendEvent({ + type: 'error', + message: error instanceof Error ? error.message : 'Unknown duplicate base error', + }); + } finally { + clearInterval(heartbeat); + res.end(); + } + } + + @Get(':baseId/duplicate-check') + @Permissions('base|read') + async duplicateBaseCheck( + @Param('baseId') baseId: string, + @Query('destSpaceId') destSpaceId: string + ): Promise { + const affectedFields = await this.baseDuplicateService.previewCrossSpaceAffectedFields( + baseId, + destSpaceId + ); + return { affectedFields }; + } + @Post('create-from-template') @Permissions('base|create') @ResourceMeta('spaceId', 'body') @@ -424,9 +508,72 @@ export class BaseController { async exportBase(@Param('baseId') baseId: string, @Query('includeData') includeData?: string) { const includeDataValue = includeData === undefined ? true : !['false', '0'].includes(includeData.toLowerCase()); + const base = await this.baseService.getBaseById(baseId); + if (base.v2Status?.reason === 'new_base') { + return await this.baseExportV2Service.exportBaseZip(baseId, includeDataValue); + } return await this.baseExportService.exportBaseZip(baseId, includeDataValue); } + @Permissions('base|read') + @Get(':baseId/export-stream') + async exportBaseStream( + @Param('baseId') baseId: string, + @Query('includeData') includeData: string | undefined, + @Res() res: ExpressResponse + ) { + const includeDataValue = + includeData === undefined ? true : !['false', '0'].includes(includeData.toLowerCase()); + const sseHeartbeatMs = 15_000; + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache, no-transform'); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('X-Accel-Buffering', 'no'); + res.flushHeaders(); + + const isStreamClosed = () => res.writableEnded || res.destroyed; + // eslint-disable-next-line sonarjs/no-identical-functions + const sendEvent = (data: unknown) => { + if (isStreamClosed()) return; + res.write(`data: ${JSON.stringify(data)}\n\n`); + (res as ExpressResponse & { flush?: () => void }).flush?.(); + }; + const heartbeat = setInterval(() => { + if (isStreamClosed()) return; + res.write(': ping\n\n'); + (res as ExpressResponse & { flush?: () => void }).flush?.(); + }, sseHeartbeatMs); + res.on('close', () => clearInterval(heartbeat)); + + try { + const base = await this.baseService.getBaseById(baseId); + const exporter = + base.v2Status?.reason === 'new_base' + ? this.baseExportV2Service.exportBaseZip.bind(this.baseExportV2Service) + : this.baseExportService.exportBaseZip.bind(this.baseExportService); + const result = await exporter(baseId, includeDataValue, (phase, detail, event) => { + sendEvent({ + type: 'progress', + ...event, + phase, + detail: event?.detail ?? detail, + }); + }); + if (!result) { + throw new Error('Export base stream ended without result'); + } + sendEvent({ type: 'done', data: result }); + } catch (error) { + sendEvent({ + type: 'error', + message: error instanceof Error ? error.message : 'Unknown export error', + }); + } finally { + clearInterval(heartbeat); + res.end(); + } + } + @Put(':baseId/move') @Permissions('space|update') async moveBase( @@ -436,6 +583,16 @@ export class BaseController { await this.baseService.moveBase(baseId, moveBaseRo); } + @Get(':baseId/move-check') + @Permissions('space|update') + async moveBaseCheck( + @Param('baseId') baseId: string, + @Query('spaceId') spaceId: string + ): Promise { + const affectedFields = await this.baseService.previewMoveBaseCrossSpace(baseId, spaceId); + return { affectedFields }; + } + @Permissions('base|update') @Get(':baseId/erd') async generateBaseErd(@Param('baseId') baseId: string): Promise { diff --git a/apps/nestjs-backend/src/features/base/base.module.ts b/apps/nestjs-backend/src/features/base/base.module.ts index 813d42d4d4..d5542241f2 100644 --- a/apps/nestjs-backend/src/features/base/base.module.ts +++ b/apps/nestjs-backend/src/features/base/base.module.ts @@ -17,7 +17,9 @@ import { TableDuplicateService } from '../table/table-duplicate.service'; import { TableModule } from '../table/table.module'; import { V2Module } from '../v2/v2.module'; import { ViewOpenApiModule } from '../view/open-api/view-open-api.module'; +import { BaseDuplicateV2Service } from './base-duplicate-v2.service'; import { BaseDuplicateService } from './base-duplicate.service'; +import { BaseExportV2Service } from './base-export-v2.service'; import { BaseExportService } from './base-export.service'; import { BaseImportAttachmentsCsvModule } from './base-import-processor/base-import-attachments-csv.module'; import { BaseImportAttachmentsModule } from './base-import-processor/base-import-attachments.module'; @@ -55,9 +57,11 @@ import { DbConnectionService } from './db-connection.service'; DbProvider, BaseService, BaseExportService, + BaseExportV2Service, BaseImportService, DbConnectionService, BaseDuplicateService, + BaseDuplicateV2Service, BaseQueryService, TableDuplicateService, ], @@ -65,7 +69,9 @@ import { DbConnectionService } from './db-connection.service'; BaseService, DbConnectionService, BaseDuplicateService, + BaseDuplicateV2Service, BaseExportService, + BaseExportV2Service, BaseImportService, BaseQueryService, ], diff --git a/apps/nestjs-backend/src/features/base/base.service.spec.ts b/apps/nestjs-backend/src/features/base/base.service.spec.ts index eb168ec6e2..dd6c0a6d12 100644 --- a/apps/nestjs-backend/src/features/base/base.service.spec.ts +++ b/apps/nestjs-backend/src/features/base/base.service.spec.ts @@ -145,4 +145,80 @@ describe('BaseService', () => { expect(result.v2Status).toEqual({ useV2: true, reason: 'space_feature' }); }); }); + + describe('dropBase', () => { + it('runs base-level data DDL through the routed BYODB data client', async () => { + const defaultDataPrisma = { + $executeRawUnsafe: vi.fn(), + }; + const routedTxClient = { + $executeRawUnsafe: vi.fn().mockResolvedValue(1), + }; + const routedDataPrisma = { + txClient: vi.fn().mockReturnValue(routedTxClient), + $executeRawUnsafe: vi.fn(), + }; + const dataDbClientManager = { + dataPrismaForBase: vi.fn().mockResolvedValue(routedDataPrisma), + }; + const dbProvider = { + dropSchema: vi.fn().mockReturnValue('DROP SCHEMA "bse1" CASCADE'), + }; + const tableOpenApiService = { + dropTables: vi.fn(), + }; + const { service } = { + service: new BaseService( + {} as never, + defaultDataPrisma as never, + dataDbClientManager as never, + {} as never, + {} as never, + {} as never, + {} as never, + tableOpenApiService as never, + {} as never, + {} as never, + {} as never, + dbProvider as never, + {} as never + ), + }; + + await service.dropBase('bse1', ['tbl1']); + + expect(dataDbClientManager.dataPrismaForBase).toHaveBeenCalledWith('bse1', { + useTransaction: true, + }); + expect(routedTxClient.$executeRawUnsafe).toHaveBeenCalledWith('DROP SCHEMA "bse1" CASCADE'); + expect(routedDataPrisma.$executeRawUnsafe).not.toHaveBeenCalled(); + expect(defaultDataPrisma.$executeRawUnsafe).not.toHaveBeenCalled(); + expect(tableOpenApiService.dropTables).not.toHaveBeenCalled(); + }); + + it('falls back to table-level routed drops when the provider has no base schema DDL', async () => { + const tableOpenApiService = { + dropTables: vi.fn().mockResolvedValue(undefined), + }; + const service = new BaseService( + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + tableOpenApiService as never, + {} as never, + {} as never, + {} as never, + { dropSchema: vi.fn().mockReturnValue('') } as never, + {} as never + ); + + await service.dropBase('bse1', ['tbl1', 'tbl2']); + + expect(tableOpenApiService.dropTables).toHaveBeenCalledWith(['tbl1', 'tbl2']); + }); + }); }); diff --git a/apps/nestjs-backend/src/features/base/base.service.ts b/apps/nestjs-backend/src/features/base/base.service.ts index 375082eb31..dd67de856d 100644 --- a/apps/nestjs-backend/src/features/base/base.service.ts +++ b/apps/nestjs-backend/src/features/base/base.service.ts @@ -2,18 +2,21 @@ import { Injectable, Logger } from '@nestjs/common'; import { ActionPrefix, actionPrefixMap, + FieldType, generateBaseId, HttpErrorCode, + Relationship, Role, generateTemplateId, + type ILinkFieldOptions, } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService, ProvisionState } from '@teable/db-main-prisma'; import type { IBaseErdVo, ICreateBaseFromTemplateRo, ICreateBaseFromTemplateVo, ICreateBaseRo, + ICrossSpaceAffectedField, IDuplicateBaseRo, IGetBasePermissionVo, IMoveBaseRo, @@ -45,31 +48,82 @@ import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; import { PermissionService } from '../auth/permission.service'; import { CanaryService } from '../canary'; import { CollaboratorService } from '../collaborator/collaborator.service'; +import { FieldOpenApiService } from '../field/open-api/field-open-api.service'; import { GraphService } from '../graph/graph.service'; import { TableOpenApiService } from '../table/open-api/table-open-api.service'; +import { BaseDuplicateV2Service } from './base-duplicate-v2.service'; import { BaseDuplicateService } from './base-duplicate.service'; +import type { BaseImportProgressCallback } from './base-import.service'; +import { + computeCrossSpaceFieldLevels, + extractForeignTableId, + sortByConversionDepth, +} from './cross-space-detection.util'; import { replaceDefaultUrl } from './utils'; +type IDataPrismaExecutor = { + $executeRawUnsafe(query: string, ...values: unknown[]): PromiseLike; +}; + +/** + * Stable key for deduplicating orphan link-storage drops across both sides of + * a symmetric pair. Both sides reference the same underlying junction (M:N) or + * FK column (N:1 / 1:1), so calling cleanForeignKey twice would error on the + * second drop. The key matches the storage `cleanForeignKey` actually targets: + * + * - M:N (and one-way OneMany pointing at a junction) → `table:${junction}` + * - N:1 / 1:1 / two-way OneMany → `column:${host}:${col}` + */ +function computeCrossSpaceCleanupKey(opts: ILinkFieldOptions): string { + const { fkHostTableName, relationship, selfKeyName, foreignKeyName, isOneWay } = opts; + if ( + relationship === Relationship.ManyMany || + (relationship === Relationship.OneMany && isOneWay) + ) { + return `table:${fkHostTableName}`; + } + if (relationship === Relationship.ManyOne) { + return `column:${fkHostTableName}:${foreignKeyName}`; + } + if (relationship === Relationship.OneMany) { + return `column:${fkHostTableName}:${selfKeyName}`; + } + if (relationship === Relationship.OneOne) { + const col = foreignKeyName === '__id' ? selfKeyName : foreignKeyName; + return `column:${fkHostTableName}:${col}`; + } + return `unknown:${fkHostTableName}`; +} + +type IDataPrismaScopedClient = IDataPrismaExecutor & { + txClient?: () => IDataPrismaExecutor; +}; + @Injectable() export class BaseService { private logger = new Logger(BaseService.name); constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, private readonly dataDbClientManager: DataDbClientManager, private readonly cls: ClsService, private readonly collaboratorService: CollaboratorService, private readonly baseDuplicateService: BaseDuplicateService, + private readonly baseDuplicateV2Service: BaseDuplicateV2Service, private readonly permissionService: PermissionService, private readonly tableOpenApiService: TableOpenApiService, private readonly graphService: GraphService, private readonly attachmentsStorageService: AttachmentsStorageService, private readonly canaryService: CanaryService, + private readonly fieldOpenApiService: FieldOpenApiService, @InjectDbProvider() private readonly dbProvider: IDbProvider, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig ) {} + private getDataPrismaExecutor(prisma: IDataPrismaScopedClient): IDataPrismaExecutor { + return prisma.txClient?.() ?? prisma; + } + private async getRoleByBaseId(baseId: string, spaceId: string) { const userId = this.cls.get('user.id'); const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); @@ -164,6 +218,7 @@ export class BaseService { createdBy: true, createdTime: true, lastModifiedTime: true, + v2Enabled: true, }, where: { deletedTime: null, @@ -197,14 +252,16 @@ export class BaseService { const sharedBaseIds = new Set(sharedBaseList.map((s) => s.baseId)); return baseList.map((base) => { + const { v2Enabled, ...baseInfo } = base; const isCreatorInSpace = validCreatorSet.has(`${base.spaceId}:${base.createdBy}`); const displayUserId = isCreatorInSpace ? base.createdBy : spaceOwnerMap.get(base.spaceId); const displayUser = displayUserId ? userMap[displayUserId] : undefined; return { - ...base, + ...baseInfo, role: roleMap[base.id] || roleMap[base.spaceId], isShared: sharedBaseIds.has(base.id), + v2Status: v2Enabled ? ({ useV2: true, reason: 'new_base' } as const) : undefined, lastModifiedTime: base.lastModifiedTime?.toISOString(), createdTime: base.createdTime?.toISOString(), createdUser: displayUser @@ -252,7 +309,9 @@ export class BaseService { try { const sqlList = this.dbProvider.createSchema(base.id); if (sqlList) { - const dataPrisma = await this.dataDbClientManager.dataPrismaForSpace(spaceId); + const dataPrisma = await this.dataDbClientManager.dataPrismaForSpace(spaceId, { + useTransaction: true, + }); for (const sql of sqlList) { await dataPrisma.$executeRawUnsafe(sql); } @@ -408,6 +467,36 @@ export class BaseService { ); } + async duplicateBaseV2(duplicateBaseRo: IDuplicateBaseRo) { + const { fromBaseId } = duplicateBaseRo; + + // Regular permission check, base update permission + await this.checkBaseUpdatePermission(fromBaseId); + + this.logger.log(`base-duplicate-service-v2: Start to duplicating base: ${fromBaseId}`); + + const result = await this.baseDuplicateV2Service.duplicateBase(duplicateBaseRo); + return result.base; + } + + async duplicateBaseV2WithProgress( + duplicateBaseRo: IDuplicateBaseRo, + onProgress?: BaseImportProgressCallback + ) { + const { fromBaseId } = duplicateBaseRo; + + await this.checkBaseUpdatePermission(fromBaseId); + + this.logger.log(`base-duplicate-service-v2: Start to duplicating base stream: ${fromBaseId}`); + + return await this.baseDuplicateV2Service.duplicateBase( + duplicateBaseRo, + true, + BaseDuplicateMode.Normal, + onProgress + ); + } + private async checkBaseUpdatePermission(baseId: string) { // First check if the user has the base read permission await this.permissionService.validPermissions(baseId, ['base|update']); @@ -571,7 +660,9 @@ export class BaseService { await this.dropBase(baseId, tableIds); await this.tableOpenApiService.cleanReferenceFieldIds(tableIds); - await this.tableOpenApiService.cleanTablesRelatedData(baseId, tableIds); + await this.tableOpenApiService.cleanTablesRelatedData(baseId, tableIds, { + useTransaction: true, + }); await this.cleanBaseRelatedData(baseId); }, { @@ -580,25 +671,38 @@ export class BaseService { ); } - private async permanentEmptyBaseRelatedData(baseId: string) { - return await this.prismaService.$tx( - async (prisma) => { - const tables = await prisma.tableMeta.findMany({ - where: { baseId }, - select: { id: true }, - }); - const tableIds = tables.map(({ id }) => id); + private async permanentEmptyBaseRelatedData( + baseId: string, + options: { + transaction?: 'current'; + emitRuntimeEvents?: boolean; + syncButtonField?: boolean; + } = {} + ) { + const remove = async () => { + const prisma = this.prismaService.txClient(); + const tables = await prisma.tableMeta.findMany({ + where: { baseId }, + select: { id: true }, + }); + const tableIds = tables.map(({ id }) => id); - await this.dropBaseTable(tableIds); - await this.tableOpenApiService.cleanReferenceFieldIds(tableIds); - await this.tableOpenApiService.cleanTablesRelatedData(baseId, tableIds); - await this.cleanBaseRelatedDataWithoutBase(baseId); - await this.cleanRelativeNodesData(baseId); - }, - { - timeout: this.thresholdConfig.bigTransactionTimeout, - } - ); + await this.dropBaseTable(tableIds); + await this.tableOpenApiService.cleanReferenceFieldIds(tableIds); + await this.tableOpenApiService.cleanTablesRelatedData(baseId, tableIds, { + useTransaction: true, + }); + await this.cleanBaseRelatedDataWithoutBase(baseId); + await this.cleanRelativeNodesData(baseId); + }; + + if (options.transaction === 'current') { + return await remove(); + } + + return await this.prismaService.$tx(remove, { + timeout: this.thresholdConfig.bigTransactionTimeout, + }); } private async cleanBaseRelatedDataWithoutBase(baseId: string) { @@ -639,7 +743,10 @@ export class BaseService { async dropBase(baseId: string, tableIds: string[]) { const sql = this.dbProvider.dropSchema(baseId); if (sql) { - return await this.dataPrismaService.$executeRawUnsafe(sql); + const scopedDataPrisma = await this.dataDbClientManager.dataPrismaForBase(baseId, { + useTransaction: true, + }); + return await this.getDataPrismaExecutor(scopedDataPrisma).$executeRawUnsafe(sql); } await this.tableOpenApiService.dropTables(tableIds); } @@ -681,13 +788,250 @@ export class BaseService { } async moveBase(baseId: string, moveBaseRo: IMoveBaseRo) { - const { spaceId } = moveBaseRo; + const { spaceId: targetSpaceId } = moveBaseRo; // check if has the permission to create base in the target space - await this.checkBaseCreatePermission(spaceId); - await this.prismaService.base.update({ + await this.checkBaseCreatePermission(targetSpaceId); + + const { affected, levels } = await this.computeMoveBaseCrossSpaceImpact(baseId, targetSpaceId); + // Deepest-first: dependent lookup/rollup fields convert first via the + // regular convertField path so their values are snapshotted by + // cellValue2String before the upstream Link is downgraded. The Link + // fields themselves (level 0) then go through convertCrossSpaceLinkToText, + // which skips the destructive linkToOther cleanup so the symmetric + // partner in the other base survives and can be converted independently + // (preserving its own values). + const conversionOrder = sortByConversionDepth(affected, levels); + + // Snapshot every converted Link's old options so we can drop the now- + // orphaned junction / FK column after the conversion tx commits (cleanup + // is intentionally deferred — running it during convert would break the + // symmetric partner's read path before its own snapshot lands). + const linkOptionsToCleanup: ILinkFieldOptions[] = []; + + try { + await this.prismaService.$tx(async () => { + for (const f of conversionOrder) { + const stillNeedsConversion = await this.prismaService.txClient().field.findFirst({ + where: { id: f.fieldId, tableId: f.tableId, deletedTime: null }, + select: { id: true, type: true, isLookup: true, isConditionalLookup: true }, + }); + if (!stillNeedsConversion) { + // No longer expected with the cross-space convert path (symmetric + // partner is preserved). Log if it happens — it would indicate an + // unexpected upstream change. + this.logger.warn( + `[cross-space] move-base field unexpectedly missing: fieldId=${f.fieldId} tableId=${f.tableId} baseId=${f.baseId} reason=${f.reason}` + ); + continue; + } + if ( + stillNeedsConversion.type === FieldType.SingleLineText && + !stillNeedsConversion.isLookup && + !stillNeedsConversion.isConditionalLookup + ) { + continue; + } + const isRootLink = + stillNeedsConversion.type === FieldType.Link && + !stillNeedsConversion.isLookup && + !stillNeedsConversion.isConditionalLookup; + if (isRootLink) { + const { oldLinkOptions } = await this.fieldOpenApiService.convertCrossSpaceLinkToText( + f.tableId, + f.fieldId + ); + linkOptionsToCleanup.push(oldLinkOptions); + } else { + await this.fieldOpenApiService.convertField(f.tableId, f.fieldId, { + type: FieldType.SingleLineText, + }); + } + } + await this.prismaService.txClient().base.update({ + where: { id: baseId }, + data: { spaceId: targetSpaceId }, + }); + }); + } catch (error) { + this.logger.error( + `[cross-space] move-base failed: baseId=${baseId} targetSpaceId=${targetSpaceId} affected=${affected.length} error=${(error as Error).message}` + ); + throw error; + } + + // Drop orphan junction / FK storage now that every Link in the pair has + // been converted. Best-effort: log warnings on failure rather than abort, + // since the move itself already succeeded and leaving orphan storage is + // recoverable (manual SQL or future sweep). Dedup by storage target so + // both sides of a symmetric pair don't fight over the same DROP. + const cleanupSeen = new Set(); + for (const opts of linkOptionsToCleanup) { + const key = computeCrossSpaceCleanupKey(opts); + if (cleanupSeen.has(key)) continue; + cleanupSeen.add(key); + try { + await this.fieldOpenApiService.cleanOrphanCrossSpaceLinkStorage(opts); + } catch (e) { + this.logger.warn( + `[cross-space] orphan link storage cleanup failed: key=${key} error=${(e as Error).message}` + ); + } + } + } + + async previewMoveBaseCrossSpace( + baseId: string, + targetSpaceId: string + ): Promise { + return (await this.computeMoveBaseCrossSpaceImpact(baseId, targetSpaceId)).affected; + } + + private async computeMoveBaseCrossSpaceImpact( + baseId: string, + targetSpaceId: string + ): Promise<{ affected: ICrossSpaceAffectedField[]; levels: Map }> { + const prisma = this.prismaService.txClient(); + + const movingBase = await prisma.base.findUniqueOrThrow({ where: { id: baseId }, - data: { spaceId }, + select: { id: true, name: true, spaceId: true }, + }); + + const myTables = await prisma.tableMeta.findMany({ + where: { baseId, deletedTime: null }, + select: { id: true, name: true }, }); + if (!myTables.length) return { affected: [], levels: new Map() }; + const myTableIds = myTables.map((t) => t.id); + const myTableNameMap = new Map(myTables.map((t) => [t.id, t.name])); + const myTableSet = new Set(myTableIds); + + const fieldSelect = { + id: true, + name: true, + type: true, + tableId: true, + isLookup: true, + isConditionalLookup: true, + options: true, + lookupOptions: true, + } as const; + + // ---- Outgoing: fields in my tables whose foreignTable lives in a different + // space than the move destination. Closure (direct + lookup/rollup chains) + // is handled by computeCrossSpaceFieldLevels. + const outgoingFields = await prisma.field.findMany({ + where: { tableId: { in: myTableIds }, deletedTime: null }, + select: fieldSelect, + }); + + const outgoingForeignIds = uniq( + outgoingFields + .map((f) => extractForeignTableId(f)) + .filter((ft): ft is string => !!ft && !myTableSet.has(ft)) + ); + const outgoingForeignSpaceMap = outgoingForeignIds.length + ? new Map( + ( + await prisma.tableMeta.findMany({ + where: { id: { in: outgoingForeignIds }, deletedTime: null }, + select: { id: true, base: { select: { spaceId: true } } }, + }) + ).map((t) => [t.id, t.base.spaceId]) + ) + : new Map(); + + const outgoingLevels = computeCrossSpaceFieldLevels({ + fields: outgoingFields, + isForeignInternal: (ft) => myTableSet.has(ft), + isForeignCrossSpace: (ft) => { + const s = outgoingForeignSpaceMap.get(ft); + return Boolean(s && s !== targetSpaceId); + }, + }); + + // ---- Incoming: fields in OTHER tables (outside this base) whose + // foreignTable points at one of my tables, but only when the source-side + // base is not already in the destination space. + const incomingDirect = await prisma.field.findMany({ + where: { + tableId: { notIn: myTableIds }, + deletedTime: null, + OR: [ + { type: FieldType.Link, isLookup: null }, + { isLookup: true, isConditionalLookup: true }, + { type: FieldType.ConditionalRollup }, + ], + }, + select: fieldSelect, + }); + const incomingSourceTableIds = uniq( + incomingDirect.flatMap((f) => { + const ft = extractForeignTableId(f); + return ft && myTableSet.has(ft) ? [f.tableId] : []; + }) + ); + const incomingSourceTables = incomingSourceTableIds.length + ? await prisma.tableMeta.findMany({ + where: { id: { in: incomingSourceTableIds }, deletedTime: null }, + select: { + id: true, + name: true, + base: { select: { id: true, name: true, spaceId: true } }, + }, + }) + : []; + const crossSpaceSourceTables = incomingSourceTables.filter( + (t) => t.base.spaceId !== targetSpaceId + ); + const crossSpaceSourceTableMap = new Map(crossSpaceSourceTables.map((t) => [t.id, t])); + + const incomingFields = crossSpaceSourceTableMap.size + ? await prisma.field.findMany({ + where: { + tableId: { in: Array.from(crossSpaceSourceTableMap.keys()) }, + deletedTime: null, + }, + select: fieldSelect, + }) + : []; + const incomingLevels = computeCrossSpaceFieldLevels({ + fields: incomingFields, + isForeignCrossSpace: (ft) => myTableSet.has(ft), + }); + + const affected: ICrossSpaceAffectedField[] = []; + for (const f of outgoingFields) { + if (!outgoingLevels.has(f.id)) continue; + affected.push({ + fieldId: f.id, + fieldName: f.name, + type: f.type, + tableId: f.tableId, + tableName: myTableNameMap.get(f.tableId) ?? '', + baseId: movingBase.id, + baseName: movingBase.name, + reason: 'direct_link', + }); + } + for (const f of incomingFields) { + if (!incomingLevels.has(f.id)) continue; + const t = crossSpaceSourceTableMap.get(f.tableId); + if (!t) continue; + affected.push({ + fieldId: f.id, + fieldName: f.name, + type: f.type, + tableId: f.tableId, + tableName: t.name, + baseId: t.base.id, + baseName: t.base.name, + reason: 'incoming_link', + }); + } + // FieldIds are globally unique, so outgoing/incoming maps cannot collide. + const levels = new Map([...outgoingLevels, ...incomingLevels]); + return { affected, levels }; } async generateBaseErd(baseId: string): Promise { @@ -863,7 +1207,11 @@ export class BaseService { if (existedBaseId) { // delete some related data - await this.cleanTemplateRelatedData(existedBaseId); + await this.cleanTemplateRelatedData(existedBaseId, { + transaction: 'current', + emitRuntimeEvents: false, + syncButtonField: false, + }); } const { @@ -890,8 +1238,15 @@ export class BaseService { }; } - async cleanTemplateRelatedData(baseId: string) { - await this.permanentEmptyBaseRelatedData(baseId); + async cleanTemplateRelatedData( + baseId: string, + options: { + transaction?: 'current'; + emitRuntimeEvents?: boolean; + syncButtonField?: boolean; + } = {} + ) { + await this.permanentEmptyBaseRelatedData(baseId, options); } /** diff --git a/apps/nestjs-backend/src/features/base/cross-space-detection.util.spec.ts b/apps/nestjs-backend/src/features/base/cross-space-detection.util.spec.ts new file mode 100644 index 0000000000..68eb18080d --- /dev/null +++ b/apps/nestjs-backend/src/features/base/cross-space-detection.util.spec.ts @@ -0,0 +1,437 @@ +import { FieldType } from '@teable/core'; +import { describe, expect, it } from 'vitest'; +import { + collectCrossSpaceAffectedFieldIds, + computeCrossSpaceFieldLevels, + extractForeignTableId, + parseFieldJson, + sortByConversionDepth, + type ICrossSpaceFieldInput, +} from './cross-space-detection.util'; + +const buildField = (overrides: Partial): ICrossSpaceFieldInput => ({ + id: 'fld00000000000000', + type: FieldType.SingleLineText, + isLookup: null, + isConditionalLookup: null, + options: null, + lookupOptions: null, + ...overrides, +}); + +// Helper to model "every foreign table belongs to another space" +const allCrossSpace = (_ft: string) => true; +const allSameSpace = (_ft: string) => false; + +describe('parseFieldJson', () => { + it('parses JSON strings', () => { + expect(parseFieldJson('{"foreignTableId":"tblX"}')).toEqual({ foreignTableId: 'tblX' }); + }); + + it('returns objects as-is', () => { + const o = { foreignTableId: 'tblX' }; + expect(parseFieldJson(o)).toBe(o); + }); + + it('returns undefined for null / empty / malformed', () => { + expect(parseFieldJson(null)).toBeUndefined(); + expect(parseFieldJson(undefined)).toBeUndefined(); + expect(parseFieldJson('')).toBeUndefined(); + expect(parseFieldJson('not json')).toBeUndefined(); + expect(parseFieldJson(42)).toBeUndefined(); + }); +}); + +describe('extractForeignTableId', () => { + it('reads link options.foreignTableId', () => { + expect( + extractForeignTableId( + buildField({ + type: FieldType.Link, + options: { foreignTableId: 'tblForeign' }, + }) + ) + ).toBe('tblForeign'); + }); + + it('skips link fields that are themselves lookups (downstream lookup, not the link)', () => { + expect( + extractForeignTableId( + buildField({ + type: FieldType.Link, + isLookup: true, + options: { foreignTableId: 'tblForeign' }, + }) + ) + ).toBeUndefined(); + }); + + it('reads conditionalLookup lookupOptions.foreignTableId (NOT options)', () => { + expect( + extractForeignTableId( + buildField({ + type: FieldType.Formula, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { foreignTableId: 'tblForeign' }, + options: { foreignTableId: 'wrong' }, + }) + ) + ).toBe('tblForeign'); + }); + + it('reads conditionalRollup options.foreignTableId', () => { + expect( + extractForeignTableId( + buildField({ + type: FieldType.ConditionalRollup, + options: { foreignTableId: 'tblForeign' }, + }) + ) + ).toBe('tblForeign'); + }); + + it('returns undefined for plain rollup (no foreignTableId at top of options)', () => { + // Rollup carries its foreign table indirectly via lookupOptions.linkFieldId + expect( + extractForeignTableId( + buildField({ + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { linkFieldId: 'fldLink', foreignTableId: 'tblForeign' }, + }) + ) + ).toBeUndefined(); + }); + + it('handles JSON-string options', () => { + expect( + extractForeignTableId( + buildField({ + type: FieldType.Link, + options: JSON.stringify({ foreignTableId: 'tblForeign' }), + }) + ) + ).toBe('tblForeign'); + }); +}); + +describe('collectCrossSpaceAffectedFieldIds', () => { + it('returns empty when no fields reference a foreign table', () => { + const fields = [ + buildField({ id: 'fldA', type: FieldType.SingleLineText }), + buildField({ id: 'fldB', type: FieldType.Number }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ fields, isForeignCrossSpace: allCrossSpace }) + ).toEqual(new Set()); + }); + + it('flags a direct cross-space link field', () => { + const fields = [ + buildField({ + id: 'fldLink', + type: FieldType.Link, + options: { foreignTableId: 'tblOtherSpace' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ fields, isForeignCrossSpace: allCrossSpace }) + ).toEqual(new Set(['fldLink'])); + }); + + it('does not flag a same-space link field', () => { + const fields = [ + buildField({ + id: 'fldLink', + type: FieldType.Link, + options: { foreignTableId: 'tblSameSpace' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ fields, isForeignCrossSpace: allSameSpace }) + ).toEqual(new Set()); + }); + + it('flags conditional lookup whose lookupOptions.foreignTableId is cross-space', () => { + const fields = [ + buildField({ + id: 'fldCondLookup', + type: FieldType.Formula, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { foreignTableId: 'tblOther' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ fields, isForeignCrossSpace: allCrossSpace }) + ).toEqual(new Set(['fldCondLookup'])); + }); + + it('flags conditional rollup whose options.foreignTableId is cross-space', () => { + const fields = [ + buildField({ + id: 'fldCondRollup', + type: FieldType.ConditionalRollup, + options: { foreignTableId: 'tblOther' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ fields, isForeignCrossSpace: allCrossSpace }) + ).toEqual(new Set(['fldCondRollup'])); + }); + + it('flags a lookup that chains through a cross-space link (transitive)', () => { + const fields = [ + buildField({ + id: 'fldLink', + type: FieldType.Link, + options: { foreignTableId: 'tblOther' }, + }), + buildField({ + id: 'fldLookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { linkFieldId: 'fldLink', foreignTableId: 'tblOther' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ fields, isForeignCrossSpace: allCrossSpace }) + ).toEqual(new Set(['fldLink', 'fldLookup'])); + }); + + it('flags a rollup chaining through a cross-space link (transitive)', () => { + const fields = [ + buildField({ + id: 'fldLink', + type: FieldType.Link, + options: { foreignTableId: 'tblOther' }, + }), + buildField({ + id: 'fldRollup', + type: FieldType.Rollup, + lookupOptions: { linkFieldId: 'fldLink', foreignTableId: 'tblOther' }, + options: { expression: 'sum({values})' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ fields, isForeignCrossSpace: allCrossSpace }) + ).toEqual(new Set(['fldLink', 'fldRollup'])); + }); + + it('handles a lookup whose linkFieldId targets another already-affected lookup (multi-hop)', () => { + // Direct cross-space link + // Lookup pointing at the link + // Another lookup pointing at the first lookup (rare but observed in the wild) + const fields = [ + buildField({ + id: 'fldLink', + type: FieldType.Link, + options: { foreignTableId: 'tblOther' }, + }), + buildField({ + id: 'fldLookupA', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { linkFieldId: 'fldLink' }, + }), + buildField({ + id: 'fldLookupB', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { linkFieldId: 'fldLookupA' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ fields, isForeignCrossSpace: allCrossSpace }) + ).toEqual(new Set(['fldLink', 'fldLookupA', 'fldLookupB'])); + }); + + it('does not flag lookups chained through a same-space link', () => { + const fields = [ + buildField({ + id: 'fldLink', + type: FieldType.Link, + options: { foreignTableId: 'tblSameSpace' }, + }), + buildField({ + id: 'fldLookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { linkFieldId: 'fldLink' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ fields, isForeignCrossSpace: allSameSpace }) + ).toEqual(new Set()); + }); + + it('skips foreign tables marked internal (e.g. tables also being duplicated)', () => { + const fields = [ + buildField({ + id: 'fldLink', + type: FieldType.Link, + options: { foreignTableId: 'tblInternal' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ + fields, + isForeignInternal: (ft) => ft === 'tblInternal', + isForeignCrossSpace: allCrossSpace, // would flag, but internal filter wins + }) + ).toEqual(new Set()); + }); + + it('does not flag fields when foreign table cannot be resolved (deleted / missing)', () => { + const fields = [ + buildField({ + id: 'fldLink', + type: FieldType.Link, + options: { foreignTableId: 'tblDeleted' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ + fields, + // Caller signals "unknown" by returning false (matches production semantics + // where spaceMap.get() returns undefined for deleted/missing tables) + isForeignCrossSpace: () => false, + }) + ).toEqual(new Set()); + }); + + it('mixes direct, transitive, and unrelated fields correctly', () => { + const fields = [ + // Cross-space link + buildField({ + id: 'fldCrossLink', + type: FieldType.Link, + options: { foreignTableId: 'tblOther' }, + }), + // Same-space link (kept) + buildField({ + id: 'fldSameLink', + type: FieldType.Link, + options: { foreignTableId: 'tblSame' }, + }), + // Lookup of the cross-space link (flagged transitively) + buildField({ + id: 'fldLookupCross', + type: FieldType.Number, + isLookup: true, + lookupOptions: { linkFieldId: 'fldCrossLink' }, + }), + // Lookup of the same-space link (not flagged) + buildField({ + id: 'fldLookupSame', + type: FieldType.Number, + isLookup: true, + lookupOptions: { linkFieldId: 'fldSameLink' }, + }), + // Pure text field (not flagged) + buildField({ id: 'fldText', type: FieldType.SingleLineText }), + // Conditional lookup pointing across (flagged directly) + buildField({ + id: 'fldCondLookup', + type: FieldType.Formula, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { foreignTableId: 'tblOther' }, + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ + fields, + isForeignCrossSpace: (ft) => ft === 'tblOther', + }) + ).toEqual(new Set(['fldCrossLink', 'fldLookupCross', 'fldCondLookup'])); + }); + + it('parses JSON-string options/lookupOptions (raw prisma shape)', () => { + const fields = [ + buildField({ + id: 'fldLink', + type: FieldType.Link, + options: JSON.stringify({ foreignTableId: 'tblOther' }), + }), + buildField({ + id: 'fldLookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: JSON.stringify({ linkFieldId: 'fldLink' }), + }), + ]; + expect( + collectCrossSpaceAffectedFieldIds({ fields, isForeignCrossSpace: allCrossSpace }) + ).toEqual(new Set(['fldLink', 'fldLookup'])); + }); +}); + +describe('computeCrossSpaceFieldLevels', () => { + it('assigns increasing depth across a multi-hop lookup chain regardless of input order', () => { + // Input order intentionally shuffled to prove level depends on dependency + // graph, not array position. Conversion order must run B(2) → A(1) → Link(0). + const fields = [ + buildField({ + id: 'fldLookupB', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { linkFieldId: 'fldLookupA' }, + }), + buildField({ + id: 'fldLink', + type: FieldType.Link, + options: { foreignTableId: 'tblOther' }, + }), + buildField({ + id: 'fldLookupA', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { linkFieldId: 'fldLink' }, + }), + ]; + const levels = computeCrossSpaceFieldLevels({ fields, isForeignCrossSpace: allCrossSpace }); + expect(levels.get('fldLink')).toBe(0); + expect(levels.get('fldLookupA')).toBe(1); + expect(levels.get('fldLookupB')).toBe(2); + }); + + it('does not loop on a malformed lookup cycle (neither side ever gets a level)', () => { + // Cycle: A → B → A. Neither is directly cross-space and neither resolves + // to a base case, so both stay unaffected (and the BFS terminates). + const fields = [ + buildField({ + id: 'fldA', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { linkFieldId: 'fldB' }, + }), + buildField({ + id: 'fldB', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { linkFieldId: 'fldA' }, + }), + ]; + expect(computeCrossSpaceFieldLevels({ fields, isForeignCrossSpace: allCrossSpace })).toEqual( + new Map() + ); + }); +}); + +describe('sortByConversionDepth', () => { + it('orders rows deepest-first so dependent lookups convert before their link', () => { + const rows = [{ fieldId: 'fldLink' }, { fieldId: 'fldLookupB' }, { fieldId: 'fldLookupA' }]; + const levels = new Map([ + ['fldLink', 0], + ['fldLookupA', 1], + ['fldLookupB', 2], + ]); + expect(sortByConversionDepth(rows, levels).map((r) => r.fieldId)).toEqual([ + 'fldLookupB', + 'fldLookupA', + 'fldLink', + ]); + }); +}); diff --git a/apps/nestjs-backend/src/features/base/cross-space-detection.util.ts b/apps/nestjs-backend/src/features/base/cross-space-detection.util.ts new file mode 100644 index 0000000000..0e07316cab --- /dev/null +++ b/apps/nestjs-backend/src/features/base/cross-space-detection.util.ts @@ -0,0 +1,118 @@ +import { FieldType } from '@teable/core'; + +export interface ICrossSpaceFieldInput { + id: string; + type: string; + isLookup?: boolean | null; + isConditionalLookup?: boolean | null; + options?: unknown; + lookupOptions?: unknown; +} + +export function parseFieldJson(raw: unknown): Record | undefined { + if (!raw) return undefined; + if (typeof raw === 'object') return raw as Record; + if (typeof raw !== 'string') return undefined; + try { + return JSON.parse(raw) as Record; + } catch { + return undefined; + } +} + +export function extractForeignTableId(field: ICrossSpaceFieldInput): string | undefined { + const isLinkField = field.type === FieldType.Link && !field.isLookup; + const isCondLookup = Boolean(field.isLookup && field.isConditionalLookup); + const isCondRollup = field.type === FieldType.ConditionalRollup; + if (!isLinkField && !isCondLookup && !isCondRollup) return undefined; + const blob = parseFieldJson(isCondLookup ? field.lookupOptions : field.options); + const value = blob?.foreignTableId; + return typeof value === 'string' && value ? value : undefined; +} + +export interface ICollectCrossSpaceAffectedArgs { + fields: ICrossSpaceFieldInput[]; + /** + * Returns true when foreignTableId resolves to a different space than the + * duplicate destination. Should return false when the foreign table is + * unknown/deleted so unresolvable references are not flagged. + */ + isForeignCrossSpace: (foreignTableId: string) => boolean; + /** Returns true when the foreign table belongs to a table being duplicated. */ + isForeignInternal?: (foreignTableId: string) => boolean; +} + +/** + * Pure transitive-closure detector. Given raw fields and a predicate that + * decides whether a `foreignTableId` is cross-space, returns the IDs of fields + * that must be downgraded: + * - direct cross-space link/conditionalLookup/conditionalRollup + * - link-based lookup/rollup whose lookupOptions.linkFieldId chains (possibly + * through other lookups) to a cross-space link + */ +export function collectCrossSpaceAffectedFieldIds( + args: ICollectCrossSpaceAffectedArgs +): Set { + return new Set(computeCrossSpaceFieldLevels(args).keys()); +} + +/** + * Returns id → BFS depth: 0 = direct cross-space link/condLookup/condRollup, + * 1 = lookup/rollup that depends on a depth-0 link, 2 = lookup that depends on + * a depth-1 lookup, etc. Sort by **descending** depth to obtain a safe + * conversion order: dependent fields must be converted to text BEFORE the link + * they read from is downgraded — otherwise the cascade recompute inside + * `convertField` resolves the lookup against the (now text) source and + * overwrites its stored cellValue with null. + */ +export function computeCrossSpaceFieldLevels( + args: ICollectCrossSpaceAffectedArgs +): Map { + const { fields, isForeignCrossSpace, isForeignInternal } = args; + + const level = new Map(); + for (const f of fields) { + const ft = extractForeignTableId(f); + if (!ft) continue; + if (isForeignInternal?.(ft)) continue; + if (isForeignCrossSpace(ft)) level.set(f.id, 0); + } + if (level.size === 0) return level; + + const crossSpaceLinkIds = new Set( + fields + .filter((f) => f.type === FieldType.Link && !f.isLookup && level.has(f.id)) + .map((f) => f.id) + ); + + let grew = true; + while (grew) { + grew = false; + const snapshot = new Map(level); + for (const f of fields) { + if (level.has(f.id)) continue; + const linkFieldId = parseFieldJson(f.lookupOptions)?.linkFieldId; + if (typeof linkFieldId !== 'string') continue; + let depLevel: number | undefined; + if (crossSpaceLinkIds.has(linkFieldId)) depLevel = 0; + else if (snapshot.has(linkFieldId)) depLevel = snapshot.get(linkFieldId); + if (depLevel !== undefined) { + level.set(f.id, depLevel + 1); + grew = true; + } + } + } + return level; +} + +/** + * Sort affected field rows for sequential convertField: deepest (highest depth) + * first so a dependent lookup's stored value is snapshotted to text via its + * own cellValue2String BEFORE the underlying link is downgraded. + */ +export function sortByConversionDepth( + rows: T[], + levels: Map +): T[] { + return [...rows].sort((a, b) => (levels.get(b.fieldId) ?? 0) - (levels.get(a.fieldId) ?? 0)); +} diff --git a/apps/nestjs-backend/src/features/base/db-connection.service.ts b/apps/nestjs-backend/src/features/base/db-connection.service.ts index 7ec2ac6efd..3dedd78688 100644 --- a/apps/nestjs-backend/src/features/base/db-connection.service.ts +++ b/apps/nestjs-backend/src/features/base/db-connection.service.ts @@ -3,8 +3,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import type { IDsn } from '@teable/core'; import { DriverClient, HttpErrorCode, parseDsn } from '@teable/core'; -import { PrismaService, getDatabaseUrl } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; +import { PrismaService } from '@teable/db-main-prisma'; import type { IDbConnectionVo } from '@teable/openapi'; import { Knex } from 'knex'; import { nanoid } from 'nanoid'; @@ -13,6 +12,7 @@ import { BaseConfig, type IBaseConfig } from '../../configs/base.config'; import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex'; @Injectable() @@ -21,7 +21,7 @@ export class DbConnectionService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly configService: ConfigService, @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel(DATA_KNEX) private readonly knex: Knex, @@ -84,12 +84,16 @@ export class DbConnectionService { ); }); + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForBase(baseId, { + useTransaction: true, + }); + // Revoke permissions from the role for the schema - await this.dataPrismaService.$executeRawUnsafe( + await dataPrisma.$executeRawUnsafe( this.knex.raw('REVOKE USAGE ON SCHEMA ?? FROM ??', [schemaName, readOnlyRole]).toQuery() ); - await this.dataPrismaService.$executeRawUnsafe( + await dataPrisma.$executeRawUnsafe( this.knex .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? REVOKE ALL ON TABLES FROM ??`, [ schemaName, @@ -99,7 +103,7 @@ export class DbConnectionService { ); // Revoke permissions from the role for the tables in schema - await this.dataPrismaService.$executeRawUnsafe( + await dataPrisma.$executeRawUnsafe( this.knex .raw('REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA ?? FROM ??', [ schemaName, @@ -109,7 +113,7 @@ export class DbConnectionService { ); // drop the role - await this.dataPrismaService.$executeRawUnsafe( + await dataPrisma.$executeRawUnsafe( this.knex.raw('DROP ROLE IF EXISTS ??', [readOnlyRole]).toQuery() ); @@ -120,15 +124,17 @@ export class DbConnectionService { }); } - private async roleExits(role: string): Promise { - const roleExists = await this.dataPrismaService.$queryRaw< + private async roleExits(baseId: string, role: string): Promise { + const dataPrisma = await this.databaseRouter.dataPrismaForBase(baseId); + const roleExists = await dataPrisma.$queryRaw< { count: bigint }[] >`SELECT count(*) FROM pg_roles WHERE rolname=${role}`; return Boolean(roleExists[0].count); } - private async getConnectionCount(role: string): Promise { - const roleExists = await this.dataPrismaService.$queryRaw< + private async getConnectionCount(baseId: string, role: string): Promise { + const dataPrisma = await this.databaseRouter.dataPrismaForBase(baseId); + const roleExists = await dataPrisma.$queryRaw< { count: bigint }[] >`SELECT COUNT(*) FROM pg_stat_activity WHERE usename=${role}`; return Number(roleExists[0].count); @@ -159,7 +165,7 @@ export class DbConnectionService { } // Check if the read-only role already exists - if (!(await this.roleExits(readOnlyRole))) { + if (!(await this.roleExits(baseId, readOnlyRole))) { throw new CustomHttpException('Role does not exist', HttpErrorCode.INTERNAL_SERVER_ERROR, { localization: { i18nKey: 'httpErrors.dbConnection.roleNotExist', @@ -170,10 +176,9 @@ export class DbConnectionService { }); } - const currentConnections = await this.getConnectionCount(readOnlyRole); + const currentConnections = await this.getConnectionCount(baseId, readOnlyRole); - const databaseUrl = - this.configService.get('PRISMA_DATA_DATABASE_URL') ?? getDatabaseUrl('data'); + const databaseUrl = await this.databaseRouter.getDataDatabaseUrlForBase(baseId); const { db } = parseDsn(databaseUrl); // Construct the DSN for the read-only role @@ -225,8 +230,7 @@ export class DbConnectionService { const { hostname: dbHostProxy, port: dbPortProxy } = new URL( `https://${publicDatabaseProxy}` ); - const databaseUrl = - this.configService.get('PRISMA_DATA_DATABASE_URL') ?? getDatabaseUrl('data'); + const databaseUrl = await this.databaseRouter.getDataDatabaseUrlForBase(baseId); const { db } = parseDsn(databaseUrl); return this.prismaService.$tx(async (prisma) => { @@ -254,8 +258,12 @@ export class DbConnectionService { data: { schemaPass: password }, }); + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForBase(baseId, { + useTransaction: true, + }); + // Create a read-only role - await this.dataPrismaService.$executeRawUnsafe( + await dataPrisma.$executeRawUnsafe( this.knex .raw( `CREATE ROLE ?? WITH LOGIN PASSWORD ? NOSUPERUSER NOINHERIT NOCREATEDB NOCREATEROLE NOREPLICATION CONNECTION LIMIT ?`, @@ -264,17 +272,17 @@ export class DbConnectionService { .toQuery() ); - await this.dataPrismaService.$executeRawUnsafe( + await dataPrisma.$executeRawUnsafe( this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [schemaName, readOnlyRole]).toQuery() ); - await this.dataPrismaService.$executeRawUnsafe( + await dataPrisma.$executeRawUnsafe( this.knex .raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [schemaName, readOnlyRole]) .toQuery() ); - await this.dataPrismaService.$executeRawUnsafe( + await dataPrisma.$executeRawUnsafe( this.knex .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [ schemaName, diff --git a/apps/nestjs-backend/src/features/base/utils.ts b/apps/nestjs-backend/src/features/base/utils.ts index 320b55e0ba..39318ddc5a 100644 --- a/apps/nestjs-backend/src/features/base/utils.ts +++ b/apps/nestjs-backend/src/features/base/utils.ts @@ -76,13 +76,18 @@ export const replaceDefaultUrl = ( return newDefaultUrl; }; +export interface ILinkFieldTableInfo { + dbFieldName: string; + selfKeyName: string; + isMultipleCellValue: boolean; +} + +export type ILinkFieldTableMap = Record; + export const mergeLinkFieldTableMaps = ( - map1: Record< - string, - { dbFieldName: string; selfKeyName: string; isMultipleCellValue: boolean }[] - >, - map2: Record -) => { + map1: ILinkFieldTableMap, + map2: ILinkFieldTableMap +): ILinkFieldTableMap => { const merged = { ...map1 }; Object.entries(map2).forEach(([tableId, fields]) => { merged[tableId] = [...(merged[tableId] || []), ...fields]; diff --git a/apps/nestjs-backend/src/features/calculation/batch.service.ts b/apps/nestjs-backend/src/features/calculation/batch.service.ts index 487a836760..c8d6a21be5 100644 --- a/apps/nestjs-backend/src/features/calculation/batch.service.ts +++ b/apps/nestjs-backend/src/features/calculation/batch.service.ts @@ -3,7 +3,6 @@ import { Injectable, Logger } from '@nestjs/common'; import { HttpErrorCode, IdPrefix, RecordOpBuilder, FieldType } from '@teable/core'; import type { IOtOperation, IRecord, TableDomain } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { Knex } from 'knex'; import { groupBy, isEmpty, keyBy } from 'lodash'; import { customAlphabet } from 'nanoid'; @@ -15,6 +14,7 @@ import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; import { DATA_KNEX } from '../../global/knex/knex.module'; +import { DatabaseRouter } from '../../global/database-router.service'; import type { IRawOp, IRawOpMap } from '../../share-db/interface'; import { RawOpType } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; @@ -41,7 +41,7 @@ export class BatchService { constructor( private readonly cls: ClsService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @InjectModel(DATA_KNEX) private readonly knex: Knex, @InjectDbProvider() private readonly dbProvider: IDbProvider, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, @@ -108,6 +108,7 @@ export class BatchService { opsPair: [recordId: string, IOtOperation[]][] ) { const raw = await this.fetchRawData( + tableId, dbTableName, opsPair.map(([recordId]) => recordId) ); @@ -134,7 +135,7 @@ export class BatchService { const opsData = this.buildRecordOpsData(opsPair, versionGroup); if (!opsData.length) return; - await this.executeUpdateRecords(dbTableName, fieldMap, opsData); + await this.executeUpdateRecords(tableId, dbTableName, fieldMap, opsData); const opDataList = opsPair.map(([recordId, ops]) => { return { docId: recordId, version: versionGroup[recordId].__version, data: ops }; @@ -225,18 +226,18 @@ export class BatchService { } // @Timing() - private async fetchRawData(dbTableName: string, recordIds: string[]) { + private async fetchRawData(tableId: string, dbTableName: string, recordIds: string[]) { const querySql = this.knex(dbTableName) .whereIn('__id', recordIds) .select('__id', '__version', '__last_modified_time', '__last_modified_by') .toQuery(); - return this.dataPrismaService.txClient().$queryRawUnsafe< + return this.databaseRouter.queryDataPrismaForTable< { __version: number; __id: string; }[] - >(querySql); + >(tableId, querySql, { useTransaction: true }); } private buildRecordOpsData( @@ -282,6 +283,7 @@ export class BatchService { @Timing() private async executeUpdateRecords( + tableId: string, dbTableName: string, fieldMap: { [fieldId: string]: IFieldInstance }, opsData: IOpsData[] @@ -294,7 +296,7 @@ export class BatchService { // group by fieldIds before apply for (const groupKey in opsDataGroup) { - await this.executeUpdateRecordsInner(dbTableName, fieldMap, opsDataGroup[groupKey]); + await this.executeUpdateRecordsInner(tableId, dbTableName, fieldMap, opsDataGroup[groupKey]); } } @@ -302,7 +304,8 @@ export class BatchService { dbTableName: string, idFieldName: string, schemas: { schemaType: SchemaType; dbFieldName: string }[], - data: { id: string; values: { [key: string]: unknown } }[] + data: { id: string; values: { [key: string]: unknown } }[], + routingTableId?: string ) { const tempTableName = `temp_` + customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)(); // 1.create temporary table structure @@ -328,83 +331,98 @@ export class BatchService { const validDbFieldNames = schemas.map((s) => s.dbFieldName).filter((f) => !f.startsWith('__')); - await this.dataPrismaService.$tx(async (tx) => { - // temp table should in one transaction - await tx.$executeRawUnsafe(createTempTableSql); - // 2.initialize temporary table data - await tx.$executeRawUnsafe(insertTempTableSql); - // 3.update data - await handleDBValidationErrors({ - fn: async () => { - await tx.$executeRawUnsafe(updateRecordSql); - }, - handleUniqueError: async () => { - const tables = await this.prismaService.tableMeta.findMany({ - where: { dbTableName }, - select: { id: true, name: true }, - }); - const table = tables[0]; - const fieldRaws = await this.prismaService.field.findMany({ - where: { - tableId: table.id, - dbFieldName: { in: validDbFieldNames }, - unique: true, - deletedTime: null, - }, - select: { id: true, name: true }, - }); - - throw new CustomHttpException( - `Fields ${fieldRaws.map((f) => f.id).join(', ')} unique validation failed`, - HttpErrorCode.VALIDATION_ERROR, - { - localization: { - i18nKey: 'httpErrors.custom.fieldValueDuplicate', - context: { - tableName: table.name, - fieldName: fieldRaws.map((f) => f.name).join(', '), - }, + const resolvedRoutingTableId = + routingTableId ?? + ( + await this.prismaService.txClient().tableMeta.findFirstOrThrow({ + where: { dbTableName, deletedTime: null }, + select: { id: true }, + }) + ).id; + + await this.databaseRouter.dataPrismaTransactionForTable( + resolvedRoutingTableId, + async (tx) => { + // temp table should in one transaction + await tx.$executeRawUnsafe(createTempTableSql); + // 2.initialize temporary table data + await tx.$executeRawUnsafe(insertTempTableSql); + // 3.update data + await handleDBValidationErrors({ + fn: async () => { + await tx.$executeRawUnsafe(updateRecordSql); + }, + handleUniqueError: async () => { + const tables = await this.prismaService.tableMeta.findMany({ + where: { dbTableName }, + select: { id: true, name: true }, + }); + const table = tables[0]; + const fieldRaws = await this.prismaService.field.findMany({ + where: { + tableId: table.id, + dbFieldName: { in: validDbFieldNames }, + unique: true, + deletedTime: null, }, - } - ); - }, - handleNotNullError: async () => { - const tables = await this.prismaService.tableMeta.findMany({ - where: { dbTableName }, - select: { id: true, name: true }, - }); - const table = tables[0]; - const fieldRaws = await this.prismaService.field.findMany({ - where: { - tableId: table.id, - dbFieldName: { in: validDbFieldNames }, - notNull: true, - deletedTime: null, - }, - select: { id: true, name: true }, - }); - - throw new CustomHttpException( - `Fields ${fieldRaws.map((f) => f.id).join(', ')} not null validation failed`, - HttpErrorCode.VALIDATION_ERROR, - { - localization: { - i18nKey: 'httpErrors.custom.fieldValueNotNull', - context: { - tableName: table.name, - fieldName: fieldRaws.map((f) => f.name).join(', '), + select: { id: true, name: true }, + }); + + throw new CustomHttpException( + `Fields ${fieldRaws.map((f) => f.id).join(', ')} unique validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueDuplicate', + context: { + tableName: table.name, + fieldName: fieldRaws.map((f) => f.name).join(', '), + }, }, + } + ); + }, + handleNotNullError: async () => { + const tables = await this.prismaService.tableMeta.findMany({ + where: { dbTableName }, + select: { id: true, name: true }, + }); + const table = tables[0]; + const fieldRaws = await this.prismaService.field.findMany({ + where: { + tableId: table.id, + dbFieldName: { in: validDbFieldNames }, + notNull: true, + deletedTime: null, }, - } - ); - }, - }); - // 4.delete temporary table - await tx.$executeRawUnsafe(dropTempTableSql); - }); + select: { id: true, name: true }, + }); + + throw new CustomHttpException( + `Fields ${fieldRaws.map((f) => f.id).join(', ')} not null validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueNotNull', + context: { + tableName: table.name, + fieldName: fieldRaws.map((f) => f.name).join(', '), + }, + }, + } + ); + }, + }); + // 4.delete temporary table + await tx.$executeRawUnsafe(dropTempTableSql); + }, + undefined, + { useTransaction: true } + ); } private async executeUpdateRecordsInner( + tableId: string, dbTableName: string, fieldMap: { [fieldId: string]: IFieldInstance }, opsData: IOpsData[] @@ -451,7 +469,7 @@ export class BatchService { { dbFieldName: '__version', schemaType: SchemaType.Integer }, ]; - await this.batchUpdateDB(dbTableName, '__id', schemas, data); + await this.batchUpdateDB(dbTableName, '__id', schemas, data, tableId); } @Timing() diff --git a/apps/nestjs-backend/src/features/calculation/field-calculation.service.spec.ts b/apps/nestjs-backend/src/features/calculation/field-calculation.service.spec.ts index 14859b4cdd..072331df6c 100644 --- a/apps/nestjs-backend/src/features/calculation/field-calculation.service.spec.ts +++ b/apps/nestjs-backend/src/features/calculation/field-calculation.service.spec.ts @@ -29,7 +29,7 @@ describe('FieldCalculationService', () => { txClient: () => ({ $queryRawUnsafe: metaQueryRawUnsafe }), } as never, { - txClient: () => ({ $queryRawUnsafe: dataQueryRawUnsafe }), + queryDataPrismaForBase: dataQueryRawUnsafe, } as never, {} as never, {} as never, @@ -38,7 +38,10 @@ describe('FieldCalculationService', () => { ); await expect(service.getRowCount('bseTest.projects')).resolves.toBe(7); - expect(dataQueryRawUnsafe).toHaveBeenCalledTimes(1); + expect(dataQueryRawUnsafe).toHaveBeenCalledWith( + 'bseTest', + 'select count(*) as "count" from "bseTest"."projects"' + ); expect(metaQueryRawUnsafe).not.toHaveBeenCalled(); await knex.destroy(); }); diff --git a/apps/nestjs-backend/src/features/calculation/field-calculation.service.ts b/apps/nestjs-backend/src/features/calculation/field-calculation.service.ts index 65b9b3fcd7..7553d2fb4f 100644 --- a/apps/nestjs-backend/src/features/calculation/field-calculation.service.ts +++ b/apps/nestjs-backend/src/features/calculation/field-calculation.service.ts @@ -1,12 +1,12 @@ import { Injectable } from '@nestjs/common'; import { FieldType, type IRecord } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; import { uniq } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { concatMap, lastValueFrom, map, range, toArray } from 'rxjs'; import { ThresholdConfig, IThresholdConfig } from '../../configs/threshold.config'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import { Timing } from '../../utils/timing'; import type { IFieldInstance, IFieldMap } from '../field/model/factory'; @@ -35,7 +35,7 @@ export interface ITopoOrdersContext { export class FieldCalculationService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly referenceService: ReferenceService, @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder, @InjectModel(DATA_KNEX) private readonly knex: Knex, @@ -114,9 +114,14 @@ export class FieldCalculationService { .limit(chunkSize) .offset(page * chunkSize) .toQuery(); - return this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ [dbFieldName: string]: unknown }[]>(query); + return this.databaseRouter.queryDataPrismaForTable<{ [dbFieldName: string]: unknown }[]>( + tableId, + query + ); + } + + private getBaseIdFromDbTableName(dbTableName: string) { + return dbTableName.split('.')[0]; } async getRecordsBatchByFields( @@ -130,11 +135,11 @@ export class FieldCalculationService { } = {}; const chunkSize = this.thresholdConfig.calcChunkSize; for (const dbTableName in dbTableName2fields) { + const tableId = dbTableName2tableId[dbTableName]; // deduplication is needed - const rowCount = await this.getRowCount(dbTableName); + const rowCount = await this.getRowCount(dbTableName, tableId); const totalPages = Math.ceil(rowCount / chunkSize); const fields = dbTableName2fields[dbTableName]; - const tableId = dbTableName2tableId[dbTableName]; const records = await lastValueFrom( range(0, totalPages).pipe( @@ -152,11 +157,14 @@ export class FieldCalculationService { } @Timing() - async getRowCount(dbTableName: string) { + async getRowCount(dbTableName: string, tableId?: string) { const query = this.knex.count('*', { as: 'count' }).from(dbTableName).toQuery(); - const [{ count }] = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ count: bigint }[]>(query); + const [{ count }] = tableId + ? await this.databaseRouter.queryDataPrismaForTable<{ count: bigint }[]>(tableId, query) + : await this.databaseRouter.queryDataPrismaForBase<{ count: bigint }[]>( + this.getBaseIdFromDbTableName(dbTableName), + query + ); return Number(count); } diff --git a/apps/nestjs-backend/src/features/calculation/link.service.ts b/apps/nestjs-backend/src/features/calculation/link.service.ts index 6505060cb6..5a6715cbba 100644 --- a/apps/nestjs-backend/src/features/calculation/link.service.ts +++ b/apps/nestjs-backend/src/features/calculation/link.service.ts @@ -3,7 +3,6 @@ import { Injectable, Logger } from '@nestjs/common'; import type { ILinkCellValue, ILinkFieldOptions, IRecord, TableDomain } from '@teable/core'; import { FieldType, HttpErrorCode, Relationship } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { Field } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; @@ -12,6 +11,7 @@ import { InjectModel } from 'nest-knexjs'; import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import { Timing } from '../../utils/timing'; import type { IFieldInstance, IFieldMap } from '../field/model/factory'; @@ -60,13 +60,28 @@ export class LinkService { private logger = new Logger(LinkService.name); constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly batchService: BatchService, @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder, @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel(DATA_KNEX) private readonly knex: Knex ) {} + private async executeForTable(tableId: string, query: string) { + return await this.databaseRouter.executeDataPrismaForTable(tableId, query, { + useTransaction: true, + }); + } + + private async queryForTable(tableId: string, query: string, ...values: unknown[]) { + return await this.databaseRouter.queryDataPrismaForTable( + tableId, + query, + { useTransaction: true }, + ...values + ); + } + private validateLinkCell(cell: ILinkCellContext) { if (!Array.isArray(cell.newValue)) { return cell; @@ -614,9 +629,7 @@ export class LinkService { .whereNotNull(foreignKeyName) .toQuery(); - return this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id: string; foreignId: string }[]>(query); + return this.queryForTable<{ id: string; foreignId: string }[]>(options.foreignTableId, query); } async getAllForeignKeys(options: ILinkFieldOptions) { @@ -631,9 +644,7 @@ export class LinkService { .whereNotNull(foreignKeyName) .toQuery(); - return this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id: string; foreignId: string }[]>(query); + return this.queryForTable<{ id: string; foreignId: string }[]>(options.foreignTableId, query); } private async getJoinedForeignKeys(linkRecordIds: string[], options: ILinkFieldOptions) { @@ -653,9 +664,7 @@ export class LinkService { .whereNotNull(foreignKeyName) .toQuery(); - return this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id: string; foreignId: string }[]>(query); + return this.queryForTable<{ id: string; foreignId: string }[]>(options.foreignTableId, query); } /** @@ -1001,9 +1010,10 @@ export class LinkService { const nativeQuery = qb.whereIn('__id', recordIds).toQuery(); this.logger.debug(`Fetch records with query: ${nativeQuery}`); - const recordRaw = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ [dbTableName: string]: unknown }[]>(nativeQuery); + const recordRaw = await this.queryForTable<{ [dbTableName: string]: unknown }[]>( + tableId, + nativeQuery + ); recordRaw.forEach((record) => { const recordId = record.__id as string; @@ -1207,7 +1217,7 @@ export class LinkService { .whereIn(selfKeyName, recordIdsToDeleteAll) .delete() .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(deleteAllQuery); + await this.executeForTable(field.options.foreignTableId, deleteAllQuery); // Re-insert all records in correct order const reinsertData = toDeleteAndReinsert.flatMap(([recordId, newKeys]) => @@ -1227,7 +1237,7 @@ export class LinkService { if (reinsertData.length) { const reinsertQuery = this.knex(fkHostTableName).insert(reinsertData).toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(reinsertQuery); + await this.executeForTable(field.options.foreignTableId, reinsertQuery); } } @@ -1237,7 +1247,7 @@ export class LinkService { .whereIn([selfKeyName, foreignKeyName], toDelete) .delete() .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(query); + await this.executeForTable(field.options.foreignTableId, query); } // Handle regular additions @@ -1259,6 +1269,7 @@ export class LinkService { // Get current max order for this source record if field has order column if (field.getHasOrderColumn()) { currentMaxOrder = await this.getMaxOrderForTarget( + field.options.foreignTableId, fkHostTableName, selfKeyName, sourceRecordId, @@ -1284,7 +1295,7 @@ export class LinkService { } const query = this.knex(fkHostTableName).insert(insertData).toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(query); + await this.executeForTable(field.options.foreignTableId, query); } } @@ -1292,6 +1303,7 @@ export class LinkService { * Get the maximum order value for a specific target record in a link relationship */ private async getMaxOrderForTarget( + routingTableId: string, tableName: string, foreignKeyColumn: string, targetRecordId: string, @@ -1303,9 +1315,10 @@ export class LinkService { .first() .toQuery(); - const maxOrderResult = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ maxOrder: unknown }[]>(maxOrderQuery); + const maxOrderResult = await this.queryForTable<{ maxOrder: unknown }[]>( + routingTableId, + maxOrderQuery + ); const raw = maxOrderResult[0]?.maxOrder as unknown; // Coerce aggregate results safely into number; default to 0 return raw == null ? 0 : Number(raw); @@ -1343,7 +1356,7 @@ export class LinkService { .update(updateFields) .whereIn([selfKeyName, foreignKeyName], toDelete) .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(query); + await this.executeForTable(field.options.foreignTableId, query); } if (toAdd.length) { @@ -1370,6 +1383,7 @@ export class LinkService { // Get current max order for this target record if field has order column if (field.getHasOrderColumn()) { currentMaxOrder = await this.getMaxOrderForTarget( + field.options.foreignTableId, fkHostTableName, foreignKeyName, foreignRecordId, @@ -1393,7 +1407,13 @@ export class LinkService { } } - await this.batchService.batchUpdateDB(fkHostTableName, selfKeyName, dbFields, updateData); + await this.batchService.batchUpdateDB( + fkHostTableName, + selfKeyName, + dbFields, + updateData, + field.options.foreignTableId + ); } } @@ -1422,7 +1442,7 @@ export class LinkService { .forUpdate() .toQuery(); - await this.dataPrismaService.txClient().$queryRawUnsafe(lockQuery); + await this.queryForTable(tableId, lockQuery); } private async saveForeignKeyForOneMany( @@ -1462,7 +1482,7 @@ export class LinkService { .update(clearFields) .where(selfKeyName, recordId) .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(clearQuery); + await this.executeForTable(field.options.foreignTableId, clearQuery); // Re-establish all links with correct order const dbFields = [ @@ -1485,7 +1505,8 @@ export class LinkService { fkHostTableName, foreignKeyName, dbFields, - updateData + updateData, + field.options.foreignTableId ); } else { // Handle regular add/remove operations @@ -1504,7 +1525,7 @@ export class LinkService { .update(updateFields) .whereIn([selfKeyName, foreignKeyName], deleteConditions) .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(query); + await this.executeForTable(field.options.foreignTableId, query); } // Add new links and update order for all current links @@ -1516,6 +1537,7 @@ export class LinkService { if (toAdd.length > 0) { // Get the current maximum order value for this target record const currentMaxOrder = await this.getMaxOrderForTarget( + field.options.foreignTableId, fkHostTableName, selfKeyName, recordId, @@ -1541,7 +1563,8 @@ export class LinkService { fkHostTableName, foreignKeyName, dbFields, - addData + addData, + field.options.foreignTableId ); } } else { @@ -1563,7 +1586,8 @@ export class LinkService { fkHostTableName, foreignKeyName, dbFields, - addData + addData, + field.options.foreignTableId ); } } @@ -1603,7 +1627,7 @@ export class LinkService { .update(updateFields) .whereIn([selfKeyName, foreignKeyName], toDelete) .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(query); + await this.executeForTable(field.options.foreignTableId, query); } if (toAdd.length) { @@ -1627,7 +1651,8 @@ export class LinkService { id: foreignRecordId, values, }; - }) + }), + field.options.foreignTableId ); } } @@ -1912,9 +1937,7 @@ export class LinkService { .whereNotNull(foreignKeyName) .toQuery(); - return this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id: string; foreignId: string }[]>(query); + return this.queryForTable<{ id: string; foreignId: string }[]>(options.foreignTableId, query); } async getRelatedLinkFieldRaws(tableId: string) { diff --git a/apps/nestjs-backend/src/features/calculation/system-field.service.ts b/apps/nestjs-backend/src/features/calculation/system-field.service.ts index ff63f70a0d..33817f1398 100644 --- a/apps/nestjs-backend/src/features/calculation/system-field.service.ts +++ b/apps/nestjs-backend/src/features/calculation/system-field.service.ts @@ -4,10 +4,10 @@ import { Injectable } from '@nestjs/common'; import type { LastModifiedByFieldCore, LastModifiedTimeFieldCore } from '@teable/core'; import { FieldKeyType, TableDomain, FieldType } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import type { IClsStore } from '../../types/cls'; import { Timing } from '../../utils/timing'; @@ -18,11 +18,12 @@ export class SystemFieldService { constructor( private readonly cls: ClsService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @InjectModel(DATA_KNEX) private readonly knex: Knex ) {} private async updateSystemField( + tableId: string, dbTableName: string, recordIds: string[], userId: string, @@ -38,7 +39,7 @@ export class SystemFieldService { .whereIn('__id', recordIds) .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(nativeQuery); + await this.databaseRouter.executeDataPrismaForTable(tableId, nativeQuery); } @Timing() @@ -78,6 +79,7 @@ export class SystemFieldService { const trackedLastModifiedByColumnUpdates: Record = {}; await this.updateSystemField( + table.id, dbTableName, records.map((r) => r.id), user.id, @@ -160,7 +162,7 @@ export class SystemFieldService { }) .whereIn('__id', recordIds) .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(nativeQuery); + await this.databaseRouter.executeDataPrismaForTable(table.id, nativeQuery); } // Persist tracked Last Modified By columns that are not generated from the system column @@ -174,7 +176,7 @@ export class SystemFieldService { }) .whereIn('__id', recordIds) .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(nativeQuery); + await this.databaseRouter.executeDataPrismaForTable(table.id, nativeQuery); } } diff --git a/apps/nestjs-backend/src/features/canary/canary.service.ts b/apps/nestjs-backend/src/features/canary/canary.service.ts index 9ef17a6a1b..50d223b6ba 100644 --- a/apps/nestjs-backend/src/features/canary/canary.service.ts +++ b/apps/nestjs-backend/src/features/canary/canary.service.ts @@ -2,12 +2,12 @@ import { Injectable } from '@nestjs/common'; import type { ICanaryConfig, V2Feature } from '@teable/openapi'; import { SettingKey } from '@teable/openapi'; import { ClsService } from 'nestjs-cls'; -import type { IClsStore, V2Reason } from '../../types/cls'; +import type { IClsStore, IV2Reason } from '../../types/cls'; import { SettingService } from '../setting/setting.service'; export interface IV2Decision { useV2: boolean; - reason: V2Reason; + reason: IV2Reason; } export interface IBaseV2DecisionContext { diff --git a/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts b/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts index 85057bd037..7c7ba4837d 100644 --- a/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts +++ b/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts @@ -88,15 +88,7 @@ export class CollaboratorService { const userIds = collaborators .filter((c) => c.principalType === PrincipalType.User) .map((c) => c.principalId); - if (userIds.length > 0) { - const countMap = await this.countUserOwnedSpaces(userIds); - for (const uid of userIds) { - await this.validateOwnedSpaceLimit( - countMap.get(uid) ?? 0, - uid !== currentUserId ? uid : undefined - ); - } - } + await this.validateOwnedSpaceLimit(spaceId, userIds); } // if has exist base collaborator, then delete it const bases = await this.prismaService.txClient().base.findMany({ @@ -564,14 +556,8 @@ export class CollaboratorService { return collaborators.length === 1 && collaborators[0].principal_id === userId; } - async countUserOwnedSpaces(userId: string): Promise; - async countUserOwnedSpaces(userIds: string[]): Promise>; - async countUserOwnedSpaces( - userIdOrIds: string | string[] - ): Promise> { - const isSingle = typeof userIdOrIds === 'string'; - const userIds = isSingle ? [userIdOrIds] : userIdOrIds; - if (userIds.length === 0) return isSingle ? 0 : new Map(); + protected async getUserOwnedSpaceIds(userIds: string[]): Promise> { + if (userIds.length === 0) return new Map(); const builder = this.knex('collaborator') .join('space', 'collaborator.resource_id', 'space.id') .whereIn('collaborator.principal_id', userIds) @@ -579,44 +565,21 @@ export class CollaboratorService { .where('collaborator.resource_type', CollaboratorType.Space) .where('collaborator.role_name', Role.Owner) .whereNull('space.deleted_time') - .groupBy('collaborator.principal_id') - .select('collaborator.principal_id as user_id') - .count('* as count'); - const result = await this.prismaService + .select('collaborator.principal_id as user_id', 'collaborator.resource_id as space_id'); + const rows = await this.prismaService .txClient() - .$queryRawUnsafe<{ user_id: string; count: number }[]>(builder.toQuery()); - if (isSingle) { - return Number(result[0]?.count ?? 0); - } - const countMap = new Map(); - for (const row of result) { - countMap.set(row.user_id, Number(row.count)); + .$queryRawUnsafe<{ user_id: string; space_id: string }[]>(builder.toQuery()); + const map = new Map(); + for (const row of rows) { + const ids = map.get(row.user_id) ?? []; + ids.push(row.space_id); + map.set(row.user_id, ids); } - return countMap; + return map; } - async validateOwnedSpaceLimit(currentCount: number, userId?: string): Promise { - const maxCount = this.thresholdConfig.maxOwnedSpaceCount; - if (maxCount <= 0 || currentCount < maxCount) return; - - const userName = userId - ? await this.prismaService.user - .findUnique({ where: { id: userId }, select: { name: true, email: true } }) - .then((user) => (user ? `${user.name} (${user.email})` : undefined)) - : undefined; - - throw new CustomHttpException( - `Owned space limit exceeded, max: ${maxCount}${userName ? `, user: ${userName}` : ''}`, - HttpErrorCode.VALIDATION_ERROR, - { - localization: { - i18nKey: userId - ? 'httpErrors.space.ownedSpaceLimitExceededOther' - : 'httpErrors.space.ownedSpaceLimitExceeded', - context: userId ? { max: maxCount, name: userName } : { max: maxCount }, - }, - } - ); + async validateOwnedSpaceLimit(_spaceId: string, _userIds: string[]): Promise { + // no-op in community; EE overrides with free-space limit } async deleteCollaborator({ @@ -736,11 +699,7 @@ export class CollaboratorService { targetColl.roleName !== Role.Owner && principalType === PrincipalType.User ) { - const count = await this.countUserOwnedSpaces(principalId); - await this.validateOwnedSpaceLimit( - count, - principalId !== currentUserId ? principalId : undefined - ); + await this.validateOwnedSpaceLimit(resourceId, [principalId]); } const res = await this.prismaService.txClient().collaborator.updateMany({ diff --git a/apps/nestjs-backend/src/features/database-view/database-view.service.ts b/apps/nestjs-backend/src/features/database-view/database-view.service.ts index 3af92c46e0..963551718d 100644 --- a/apps/nestjs-backend/src/features/database-view/database-view.service.ts +++ b/apps/nestjs-backend/src/features/database-view/database-view.service.ts @@ -1,9 +1,9 @@ import { Injectable, Logger } from '@nestjs/common'; import type { TableDomain } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { DatabaseRouter } from '../../global/database-router.service'; import { ReferenceService } from '../calculation/reference.service'; import { InjectRecordQueryBuilder, IRecordQueryBuilder } from '../record/query-builder'; import type { IDatabaseView } from './database-view.interface'; @@ -18,7 +18,7 @@ export class DatabaseViewService implements IDatabaseView { @InjectRecordQueryBuilder() private readonly recordQueryBuilderService: IRecordQueryBuilder, private readonly prisma: PrismaService, - private readonly dataPrisma: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly referenceService: ReferenceService ) {} @@ -29,7 +29,7 @@ export class DatabaseViewService implements IDatabaseView { const sqls = this.dbProvider.createDatabaseView(table, qb, { materialized: true }); const viewName = this.dbProvider.generateDatabaseViewName(table.id); - await this.dataPrisma.$tx(async (tx) => { + await this.databaseRouter.dataPrismaTransactionForTable(table.id, async (tx) => { for (const sql of sqls) { await tx.$executeRawUnsafe(sql); } @@ -64,7 +64,7 @@ export class DatabaseViewService implements IDatabaseView { }); const sqls = this.dbProvider.recreateDatabaseView(table, qb); - await this.dataPrisma.$tx(async (tx) => { + await this.databaseRouter.dataPrismaTransactionForTable(table.id, async (tx) => { for (const sql of sqls) { await tx.$executeRawUnsafe(sql); } @@ -83,7 +83,7 @@ export class DatabaseViewService implements IDatabaseView { public async refreshView(tableId: string) { const sql = this.dbProvider.refreshDatabaseView(tableId, { concurrently: true }); if (sql) { - await this.dataPrisma.$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } } @@ -93,14 +93,14 @@ export class DatabaseViewService implements IDatabaseView { for (const tableId of tableIds) { const sql = this.dbProvider.refreshDatabaseView(tableId, { concurrently: true }); if (sql) { - await this.dataPrisma.$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } } } private async dropDataView(tableId: string) { const sqls = this.dbProvider.dropDatabaseView(tableId); - await this.dataPrisma.$tx(async (tx) => { + await this.databaseRouter.dataPrismaTransactionForTable(tableId, async (tx) => { for (const sql of sqls) { await tx.$executeRawUnsafe(sql); } diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-converting-link.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-converting-link.service.ts index d831a03918..991bc2d0f9 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-converting-link.service.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-converting-link.service.ts @@ -9,13 +9,13 @@ import { PRIMARY_SUPPORTED_TYPES, HttpErrorCode, } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { isEqual } from 'lodash'; import { CustomHttpException } from '../../../custom.exception'; import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; import { DropColumnOperationType } from '../../../db-provider/drop-database-column-query/drop-database-column-field-visitor.interface'; +import { DatabaseRouter } from '../../../global/database-router.service'; import { FieldCalculationService } from '../../calculation/field-calculation.service'; import type { IOpsMap } from '../../calculation/utils/compose-maps'; import { TableDomainQueryService } from '../../table-domain/table-domain-query.service'; @@ -37,7 +37,7 @@ const isLink = (field: IFieldInstance): field is LinkFieldDto => export class FieldConvertingLinkService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly fieldDeletingService: FieldDeletingService, private readonly fieldCreatingService: FieldCreatingService, private readonly fieldSupplementService: FieldSupplementService, @@ -199,12 +199,22 @@ export class FieldConvertingLinkService { ); // Execute all queries (FK/junction creation, order columns, etc.) for (const query of createColumnQueries) { - await this.dataPrismaService.txClient().$executeRawUnsafe(query); + await this.databaseRouter.executeDataPrismaForTable(tableId, query, { useTransaction: true }); } } - private async linkToOther(tableId: string, oldField: LinkFieldDto) { + private async linkToOther(tableId: string, oldField: LinkFieldDto, skipDestructive?: boolean) { await this.fieldDeletingService.cleanLookupRollupRef(tableId, oldField.id); + + // Cross-space move path: caller wants both sides of a symmetric pair to be + // converted independently, each preserving its own values. Skipping + // cleanForeignKey leaves the junction/FK intact so the partner can still + // read its values when its own conversion runs; skipping the symmetric + // cascade keeps the partner field alive so it can be converted in turn. + // Storage left behind here is orphaned and is dropped by the caller (e.g. + // moveBase) once every paired field has been converted. + if (skipDestructive) return; + await this.fieldSupplementService.cleanForeignKey(oldField.options); if (oldField.options.symmetricFieldId) { @@ -227,14 +237,15 @@ export class FieldConvertingLinkService { async deleteOrCreateSupplementLink( tableId: string, newField: IFieldInstance, - oldField: IFieldInstance + oldField: IFieldInstance, + skipDestructive?: boolean ) { if (isLink(newField) && isLink(oldField) && !isEqual(newField.options, oldField.options)) { return this.linkOptionsChange(tableId, newField, oldField); } if (!isLink(newField) && isLink(oldField)) { - return this.linkToOther(tableId, oldField); + return this.linkToOther(tableId, oldField, skipDestructive); } if (isLink(newField) && !isLink(oldField)) { diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.ts index 20a2e90f9e..bcff52445f 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.ts @@ -22,12 +22,12 @@ import { RecordOpBuilder, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { Knex } from 'knex'; import { difference, intersection, isEmpty, isEqual, keyBy, set, uniq } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { CustomHttpException } from '../../../custom.exception'; import { DATA_KNEX } from '../../../global/knex/knex.module'; +import { DatabaseRouter } from '../../../global/database-router.service'; import { handleDBValidationErrors } from '../../../utils/db-validation-error'; import { majorFieldKeysChanged, @@ -69,7 +69,7 @@ export class FieldConvertingService { private readonly fieldService: FieldService, private readonly batchService: BatchService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly fieldConvertingLinkService: FieldConvertingLinkService, private readonly fieldSupplementService: FieldSupplementService, private readonly fieldCalculationService: FieldCalculationService, @@ -620,11 +620,9 @@ export class FieldConvertingService { .toSQL() .toNative(); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe< - { __id: string; [dbFieldName: string]: string }[] - >(nativeSql.sql, ...nativeSql.bindings); + const result = await this.databaseRouter.queryDataPrismaForTable< + { __id: string; [dbFieldName: string]: string }[] + >(tableId, nativeSql.sql, { useTransaction: true }, ...nativeSql.bindings); for (const row of result) { const oldCellValue = field.convertDBValue2CellValue(row[field.dbFieldName]) as string[]; @@ -674,11 +672,9 @@ export class FieldConvertingService { .toSQL() .toNative(); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe< - { __id: string; [dbFieldName: string]: string }[] - >(nativeSql.sql, ...nativeSql.bindings); + const result = await this.databaseRouter.queryDataPrismaForTable< + { __id: string; [dbFieldName: string]: string }[] + >(tableId, nativeSql.sql, { useTransaction: true }, ...nativeSql.bindings); for (const row of result) { let oldCellValue = field.convertDBValue2CellValue(row[field.dbFieldName]) as string; @@ -770,11 +766,9 @@ export class FieldConvertingService { .toSQL() .toNative(); - const result = await this.prismaService - .txClient() - .$queryRawUnsafe< - { __id: string; [dbFieldName: string]: string }[] - >(nativeSql.sql, ...nativeSql.bindings); + const result = await this.databaseRouter.queryDataPrismaForTable< + { __id: string; [dbFieldName: string]: string }[] + >(tableId, nativeSql.sql, { useTransaction: true }, ...nativeSql.bindings); for (const row of result) { let oldCellValue = field.convertDBValue2CellValue(row[dbFieldName]) as number; @@ -821,9 +815,9 @@ export class FieldConvertingService { const opsMap: { [recordId: string]: IOtOperation[] } = {}; const nativeSql = this.knex(dbTableName).select('__id', dbFieldName).whereNotNull(dbFieldName); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ __id: string; [dbFieldName: string]: string }[]>(nativeSql.toQuery()); + const result = await this.databaseRouter.queryDataPrismaForTable< + { __id: string; [dbFieldName: string]: string }[] + >(tableId, nativeSql.toQuery(), { useTransaction: true }); for (const row of result) { const oldCellValue = field.convertDBValue2CellValue(row[dbFieldName]); @@ -870,9 +864,9 @@ export class FieldConvertingService { .select('__id', field.dbFieldName) .whereNotNull(field.dbFieldName); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ __id: string; [dbFieldName: string]: string }[]>(nativeSql.toQuery()); + const result = await this.databaseRouter.queryDataPrismaForTable< + { __id: string; [dbFieldName: string]: string }[] + >(tableId, nativeSql.toQuery(), { useTransaction: true }); for (const row of result) { const oldCellValue = field.convertDBValue2CellValue(row[field.dbFieldName]); opsMap[row.__id] = [ @@ -1468,9 +1462,15 @@ export class FieldConvertingService { async deleteOrCreateSupplementLink( tableId: string, newField: IFieldInstance, - oldField: IFieldInstance + oldField: IFieldInstance, + skipDestructive?: boolean ) { - await this.fieldConvertingLinkService.deleteOrCreateSupplementLink(tableId, newField, oldField); + await this.fieldConvertingLinkService.deleteOrCreateSupplementLink( + tableId, + newField, + oldField, + skipDestructive + ); } private needTempleCloseFieldConstraint(newField: IFieldInstance, oldField: IFieldInstance) { @@ -1510,7 +1510,10 @@ export class FieldConvertingService { .toQuery(); await handleDBValidationErrors({ - fn: () => this.dataPrismaService.txClient().$executeRawUnsafe(fieldValidationQuery), + fn: () => + this.databaseRouter.executeDataPrismaForTable(tableId, fieldValidationQuery, { + useTransaction: true, + }), handleUniqueError: () => { throw new CustomHttpException( `Field ${oldField.id} unique validation failed`, @@ -1553,6 +1556,7 @@ export class FieldConvertingService { } const matchedIndexes = await this.fieldService.findUniqueIndexesForField( + tableId, dbTableName, dbFieldName ); @@ -1571,7 +1575,9 @@ export class FieldConvertingService { .map(({ sql }) => sql); for (const sql of executeSqls) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql, { + useTransaction: true, + }); } } diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-creating.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-creating.service.ts index f09058a736..a371f2b662 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-creating.service.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-creating.service.ts @@ -2,6 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'; import type { IColumn, IColumnMeta } from '@teable/core'; import { FieldType } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import type { IDataDbRoutingOptions } from '../../../global/data-db-client-manager.service'; import { ViewService } from '../../view/view.service'; import { FieldService } from '../field.service'; import type { IFieldInstance } from '../model/factory'; @@ -23,14 +24,15 @@ export class FieldCreatingService { tableId: string, field: IFieldInstance, initViewColumnMap?: Record, - isSymmetricField?: boolean + isSymmetricField?: boolean, + routingOptions?: IDataDbRoutingOptions ) { const fieldId = field.id; await this.fieldSupplementService.createReference(field); await this.fieldSupplementService.createFieldTaskReference(tableId, field); - const dbTableName = await this.fieldService.getDbTableName(tableId); + const dbTableName = await this.fieldService.getDbTableName(tableId, routingOptions); await this.fieldService.batchCreateFields(tableId, dbTableName, [field], isSymmetricField); @@ -45,11 +47,12 @@ export class FieldCreatingService { tableId: string, fieldInstances: IFieldInstance[], initViewColumnMapList?: Array | undefined>, - isSymmetricField?: boolean + isSymmetricField?: boolean, + routingOptions?: IDataDbRoutingOptions ) { if (!fieldInstances.length) return; - const dbTableName = await this.fieldService.getDbTableName(tableId); + const dbTableName = await this.fieldService.getDbTableName(tableId, routingOptions); for (const field of fieldInstances) { await this.fieldSupplementService.createReference(field); @@ -77,9 +80,10 @@ export class FieldCreatingService { async createFields( tableId: string, fieldInstances: IFieldInstance[], - initViewColumnMap?: Record + initViewColumnMap?: Record, + routingOptions?: IDataDbRoutingOptions ) { - const dbTableName = await this.fieldService.getDbTableName(tableId); + const dbTableName = await this.fieldService.getDbTableName(tableId, routingOptions); for (const field of fieldInstances) { await this.fieldSupplementService.createReference(field); @@ -97,14 +101,21 @@ export class FieldCreatingService { async alterCreateFieldsInExistingTable( tableId: string, - fields: Array<{ field: IFieldInstance; columnMeta?: Record }> + fields: Array<{ field: IFieldInstance; columnMeta?: Record }>, + routingOptions?: IDataDbRoutingOptions ) { if (!fields.length) return [] as { tableId: string; field: IFieldInstance }[]; const baseFieldInstances = fields.map(({ field }) => field); const initViewColumnMapList = fields.map(({ columnMeta }) => columnMeta); - await this.createFieldItemsBatch(tableId, baseFieldInstances, initViewColumnMapList); + await this.createFieldItemsBatch( + tableId, + baseFieldInstances, + initViewColumnMapList, + undefined, + routingOptions + ); const created: { tableId: string; field: IFieldInstance }[] = baseFieldInstances.map( (field) => ({ @@ -124,44 +135,64 @@ export class FieldCreatingService { if (!linkField.options.symmetricFieldId) continue; const symmetricField = await this.fieldSupplementService.generateSymmetricField( tableId, - linkField + linkField, + routingOptions ); const foreignTableId = linkField.options.foreignTableId; - await this.createFieldItemsBatch(foreignTableId, [symmetricField], undefined, true); + await this.createFieldItemsBatch( + foreignTableId, + [symmetricField], + undefined, + true, + routingOptions + ); created.push({ tableId: foreignTableId, field: symmetricField }); } return created; } - async alterCreateField(tableId: string, field: IFieldInstance, columnMeta?: IColumnMeta) { + async alterCreateField( + tableId: string, + field: IFieldInstance, + columnMeta?: IColumnMeta, + routingOptions?: IDataDbRoutingOptions + ) { const newFields: { tableId: string; field: IFieldInstance }[] = []; if (field.type === FieldType.Link && !field.isLookup) { // Foreign key creation is now handled by the visitor in createFieldItem - await this.createFieldItem(tableId, field, columnMeta); + await this.createFieldItem(tableId, field, columnMeta, undefined, routingOptions); newFields.push({ tableId, field }); if (field.options.symmetricFieldId) { const symmetricField = await this.fieldSupplementService.generateSymmetricField( tableId, - field + field, + routingOptions ); - await this.createFieldItem(field.options.foreignTableId, symmetricField, columnMeta, true); + await this.createFieldItem( + field.options.foreignTableId, + symmetricField, + columnMeta, + true, + routingOptions + ); newFields.push({ tableId: field.options.foreignTableId, field: symmetricField }); } return newFields; } - await this.createFieldItem(tableId, field, columnMeta); + await this.createFieldItem(tableId, field, columnMeta, undefined, routingOptions); return [{ tableId, field: field }]; } async alterCreateFields( tableId: string, fieldInstances: IFieldInstance[], - columnMeta?: IColumnMeta + columnMeta?: IColumnMeta, + routingOptions?: IDataDbRoutingOptions ) { const newFields: { tableId: string; field: IFieldInstance }[] = fieldInstances.map((field) => ({ tableId, @@ -170,7 +201,7 @@ export class FieldCreatingService { const primaryField = fieldInstances.find((field) => field.isPrimary)!; - await this.createFieldItem(tableId, primaryField, columnMeta); + await this.createFieldItem(tableId, primaryField, columnMeta, undefined, routingOptions); const linkFields = fieldInstances.filter( (field) => field.type === FieldType.Link && !field.isLookup @@ -180,7 +211,13 @@ export class FieldCreatingService { const initViewColumnMapList = columnMeta ? linkFields.map(() => columnMeta as unknown as Record) : undefined; - await this.createFieldItemsBatch(tableId, linkFields, initViewColumnMapList); + await this.createFieldItemsBatch( + tableId, + linkFields, + initViewColumnMapList, + undefined, + routingOptions + ); // Generate and create symmetric fields one-by-one to avoid duplicate // dbFieldName collisions when multiple links target the same foreign table. @@ -188,10 +225,17 @@ export class FieldCreatingService { if (!field.options.symmetricFieldId) continue; const symmetricField = await this.fieldSupplementService.generateSymmetricField( tableId, - field + field, + routingOptions ); const foreignTableId = field.options.foreignTableId; - await this.createFieldItemsBatch(foreignTableId, [symmetricField], undefined, true); + await this.createFieldItemsBatch( + foreignTableId, + [symmetricField], + undefined, + true, + routingOptions + ); newFields.push({ tableId: foreignTableId, field: symmetricField }); } } @@ -201,7 +245,7 @@ export class FieldCreatingService { (linkFields.length ? !linkFields.map(({ id }) => id).includes(id) : true) && !isPrimary ); - await this.createFields(tableId, otherFields, columnMeta); + await this.createFields(tableId, otherFields, columnMeta, routingOptions); return newFields; } } diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts index 7f3e7eeaea..03daf605d5 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts @@ -1,5 +1,5 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { BadRequestException, Injectable } from '@nestjs/common'; +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; import { AttachmentFieldCore, AutoNumberFieldCore, @@ -58,7 +58,6 @@ import type { INumberFieldOptions, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { Knex } from 'knex'; import { uniq, keyBy, mergeWith } from 'lodash'; import { InjectModel } from 'nest-knexjs'; @@ -67,12 +66,15 @@ import { fromZodError } from 'zod-validation-error'; import { CustomHttpException } from '../../../custom.exception'; import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import type { IDataDbRoutingOptions } from '../../../global/data-db-client-manager.service'; +import { DatabaseRouter } from '../../../global/database-router.service'; import { DATA_KNEX } from '../../../global/knex/knex.module'; import { extractFieldReferences } from '../../../utils'; import { majorFieldKeysChanged, NON_INFECT_OPTION_KEYS, } from '../../../utils/major-field-keys-changed'; +import { parseFieldJson } from '../../base/cross-space-detection.util'; import { ReferenceService } from '../../calculation/reference.service'; import { hasCycle } from '../../calculation/utils/dfs'; import { FieldService } from '../field.service'; @@ -83,17 +85,19 @@ import { FormulaFieldDto } from '../model/field-dto/formula-field.dto'; import type { LinkFieldDto } from '../model/field-dto/link-field.dto'; import { RollupFieldDto } from '../model/field-dto/rollup-field.dto'; -type LinkFieldReference = Pick & { +type ILinkFieldReference = Pick & { options: Pick & Partial>; }; @Injectable() export class FieldSupplementService { + private readonly logger = new Logger(FieldSupplementService.name); + constructor( private readonly fieldService: FieldService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly referenceService: ReferenceService, @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel(DATA_KNEX) private readonly knex: Knex @@ -107,6 +111,96 @@ export class FieldSupplementService { return tableMeta.dbTableName; } + async isCrossSpaceTarget(tableId: string, foreignTableId: string): Promise { + if (!foreignTableId || tableId === foreignTableId) return false; + const rows = await this.prismaService.txClient().tableMeta.findMany({ + where: { id: { in: [tableId, foreignTableId] }, deletedTime: null }, + select: { id: true, base: { select: { spaceId: true } } }, + }); + const spaceMap = new Map(rows.map((r) => [r.id, r.base.spaceId])); + const selfSpace = spaceMap.get(tableId); + const foreignSpace = spaceMap.get(foreignTableId); + return Boolean(selfSpace && foreignSpace && selfSpace !== foreignSpace); + } + + async assertSameSpaceLinkTarget(tableId: string, foreignTableId: string): Promise { + if (!(await this.isCrossSpaceTarget(tableId, foreignTableId))) return; + // Tag for post-deploy observability: count rejections to gauge how often + // clients try to create new cross-space refs after the forbid landed. + this.logger.warn( + `[cross-space] reject create: tableId=${tableId} foreignTableId=${foreignTableId}` + ); + throw new CustomHttpException( + `Cross-space link is no longer supported (foreignTableId: ${foreignTableId})`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.crossSpaceLinkForbidden', + context: { foreignTableId }, + }, + } + ); + } + + // Returns the field's foreign table reference: stored in `options` for + // Link / ConditionalRollup, in `lookupOptions` for Lookup / Rollup / + // ConditionalLookup. Accepts both parsed IFieldVo and raw Prisma rows + // (options/lookupOptions as JSON strings) — parseFieldJson short-circuits + // when already an object. + getForeignTableId(field: { + type: string; + isLookup?: boolean | null; + options?: unknown; + lookupOptions?: unknown; + }): string | undefined { + const readFt = (raw: unknown): string | undefined => { + const blob = parseFieldJson(raw); + const v = blob?.foreignTableId; + return typeof v === 'string' && v ? v : undefined; + }; + if (field.type === FieldType.Link && !field.isLookup) { + return readFt(field.options); + } + if (field.type === FieldType.ConditionalRollup) { + return readFt(field.options); + } + if (field.isLookup || field.type === FieldType.Rollup) { + return readFt(field.lookupOptions); + } + return undefined; + } + + // Composes getForeignTableId + isCrossSpaceTarget so callers don't have to + // re-derive the dual-location read every time. Returns true iff the field's + // foreignTableId resolves to a different space than `tableId`. + async isCrossSpaceField( + tableId: string, + field: { + type: string; + isLookup?: boolean | null; + options?: unknown; + lookupOptions?: unknown; + } + ): Promise { + const ft = this.getForeignTableId(field); + if (!ft) return false; + return this.isCrossSpaceTarget(tableId, ft); + } + + // Single guard at the public entry points: any new cross-space link / lookup / + // rollup / conditionalLookup / conditionalRollup is rejected. On update, keep + // legacy cross-space refs working as long as the foreignTableId is unchanged. + private async assertNoNewCrossSpaceField( + tableId: string, + newField: IFieldVo, + oldField?: IFieldVo + ): Promise { + const newFt = this.getForeignTableId(newField); + if (!newFt) return; + if (oldField && this.getForeignTableId(oldField) === newFt) return; + await this.assertSameSpaceLinkTarget(tableId, newFt); + } + private getForeignKeyFieldName(fieldId: string | undefined) { if (!fieldId) { return `__fk_rad${getRandomString(16)}`; @@ -506,7 +600,7 @@ export class FieldSupplementService { return { hasOrderColumn: Boolean(hasOrderColumn) }; } - private async prepareLookupOptions(field: IFieldRo, batchFieldVos?: IFieldVo[]) { + private async prepareLookupOptions(tableId: string, field: IFieldRo, batchFieldVos?: IFieldVo[]) { const { lookupOptions } = field; if (!lookupOptions) { throw new CustomHttpException(`lookupOptions is required`, HttpErrorCode.VALIDATION_ERROR, { @@ -530,11 +624,11 @@ export class FieldSupplementService { const batchLinkField = batchFieldVos?.find( (candidate) => candidate.id === linkFieldId && candidate.type === FieldType.Link ); - const linkFieldOptions: LinkFieldReference['options'] | undefined = + const linkFieldOptions: ILinkFieldReference['options'] | undefined = (optionsRaw && (JSON.parse(optionsRaw as string) as ILinkFieldOptions)) || (batchLinkField?.options as ILinkFieldOptions | ILinkFieldOptionsRo | undefined); - const linkFieldReference: LinkFieldReference | undefined = + const linkFieldReference: ILinkFieldReference | undefined = linkFieldRaw && linkFieldOptions ? { name: linkFieldRaw.name, @@ -646,12 +740,13 @@ export class FieldSupplementService { }; } - private async prepareLookupField(fieldRo: IFieldRo, batchFieldVos?: IFieldVo[]) { + private async prepareLookupField(tableId: string, fieldRo: IFieldRo, batchFieldVos?: IFieldVo[]) { if (fieldRo.isConditionalLookup) { - return this.prepareConditionalLookupField(fieldRo); + return this.prepareConditionalLookupField(tableId, fieldRo); } const { lookupOptions, lookupFieldRaw, linkField } = await this.prepareLookupOptions( + tableId, fieldRo, batchFieldVos ); @@ -693,20 +788,20 @@ export class FieldSupplementService { }; } - private async prepareUpdateLookupField(fieldRo: IFieldRo, oldFieldVo: IFieldVo) { + private async prepareUpdateLookupField(tableId: string, fieldRo: IFieldRo, oldFieldVo: IFieldVo) { if (fieldRo.isConditionalLookup) { - return this.prepareConditionalLookupField(fieldRo); + return this.prepareConditionalLookupField(tableId, fieldRo); } const newLookupOptions = fieldRo.lookupOptions as ILookupOptionsRo | undefined; const oldLookupOptions = oldFieldVo.lookupOptions as ILookupOptionsVo | undefined; if (!newLookupOptions || !isLinkLookupOptions(newLookupOptions)) { - return this.prepareLookupField(fieldRo); + return this.prepareLookupField(tableId, fieldRo); } if (!oldLookupOptions || !isLinkLookupOptions(oldLookupOptions)) { - return this.prepareLookupField(fieldRo); + return this.prepareLookupField(tableId, fieldRo); } if ( oldFieldVo.isLookup && @@ -729,7 +824,7 @@ export class FieldSupplementService { }; } - return this.prepareLookupField(fieldRo); + return this.prepareLookupField(tableId, fieldRo); } private async prepareFormulaField(fieldRo: IFieldRo, batchFieldVos?: IFieldVo[]) { @@ -872,8 +967,9 @@ export class FieldSupplementService { return this.prepareFormulaField(mergedFieldRo); } - private async prepareRollupField(field: IFieldRo, batchFieldVos?: IFieldVo[]) { + private async prepareRollupField(tableId: string, field: IFieldRo, batchFieldVos?: IFieldVo[]) { const { lookupOptions, linkField, lookupFieldRaw } = await this.prepareLookupOptions( + tableId, field, batchFieldVos ); @@ -935,7 +1031,7 @@ export class FieldSupplementService { } // eslint-disable-next-line sonarjs/cognitive-complexity - private async prepareConditionalRollupField(field: IFieldRo) { + private async prepareConditionalRollupField(tableId: string, field: IFieldRo) { const rawOptions = field.options as IConditionalRollupFieldOptions | undefined; const options = { ...(rawOptions || {}) } as IConditionalRollupFieldOptions | undefined; if (!options) { @@ -1081,7 +1177,7 @@ export class FieldSupplementService { }; } - private async prepareConditionalLookupField(field: IFieldRo) { + private async prepareConditionalLookupField(tableId: string, field: IFieldRo) { const lookupOptions = field.lookupOptions as ILookupOptionsRo | undefined; const conditionalLookup = isConditionalLookupOptions(lookupOptions) ? (lookupOptions as IConditionalLookupOptions) @@ -1207,7 +1303,7 @@ export class FieldSupplementService { }; } - private async prepareUpdateRollupField(fieldRo: IFieldRo, oldFieldVo: IFieldVo) { + private async prepareUpdateRollupField(tableId: string, fieldRo: IFieldRo, oldFieldVo: IFieldVo) { const newOptions = fieldRo.options as IRollupFieldOptions; const oldOptions = oldFieldVo.options as IRollupFieldOptions; @@ -1224,7 +1320,7 @@ export class FieldSupplementService { !isLinkLookupOptions(newLookupOptions) || !isLinkLookupOptions(oldLookupOptions) ) { - return this.prepareRollupField(fieldRo); + return this.prepareRollupField(tableId, fieldRo); } if ( newOptions.expression === oldOptions.expression && @@ -1244,7 +1340,7 @@ export class FieldSupplementService { }; } - return this.prepareRollupField(fieldRo); + return this.prepareRollupField(tableId, fieldRo); } private prepareSingleTextField(field: IFieldRo) { @@ -1517,16 +1613,16 @@ export class FieldSupplementService { batchFieldVos?: IFieldVo[] ) { if (fieldRo.isLookup) { - return this.prepareLookupField(fieldRo, batchFieldVos); + return this.prepareLookupField(tableId, fieldRo, batchFieldVos); } switch (fieldRo.type) { case FieldType.Link: return this.prepareLinkField(tableId, fieldRo); case FieldType.Rollup: - return this.prepareRollupField(fieldRo, batchFieldVos); + return this.prepareRollupField(tableId, fieldRo, batchFieldVos); case FieldType.ConditionalRollup: - return this.prepareConditionalRollupField(fieldRo); + return this.prepareConditionalRollupField(tableId, fieldRo); case FieldType.Formula: return this.prepareFormulaField(fieldRo, batchFieldVos); case FieldType.SingleLineText: @@ -1629,7 +1725,7 @@ export class FieldSupplementService { } if (fieldRo.isLookup && hasMajorChange) { - return this.prepareUpdateLookupField(fieldRo, oldFieldVo); + return this.prepareUpdateLookupField(tableId, fieldRo, oldFieldVo); } switch (fieldRo.type) { @@ -1637,9 +1733,9 @@ export class FieldSupplementService { return this.prepareUpdateLinkField(tableId, fieldRo, oldFieldVo); } case FieldType.Rollup: - return this.prepareUpdateRollupField(fieldRo, oldFieldVo); + return this.prepareUpdateRollupField(tableId, fieldRo, oldFieldVo); case FieldType.ConditionalRollup: - return this.prepareConditionalRollupField(fieldRo); + return this.prepareConditionalRollupField(tableId, fieldRo); case FieldType.Formula: return this.prepareUpdateFormulaField(fieldRo, oldFieldVo); case FieldType.SingleLineText: @@ -1730,14 +1826,20 @@ export class FieldSupplementService { * prepare properties for computed field to make sure it's valid * this method do not do any db update */ - async prepareCreateField(tableId: string, fieldRo: IFieldRo, batchFieldVos?: IFieldVo[]) { + async prepareCreateField( + tableId: string, + fieldRo: IFieldRo, + batchFieldVos?: IFieldVo[], + routingOptions?: IDataDbRoutingOptions + ) { const field = (await this.prepareCreateFieldInner(tableId, fieldRo, batchFieldVos)) as IFieldVo; const fieldId = field.id || generateFieldId(); const fieldName = await this.uniqFieldName(tableId, field.name); const dbFieldName = - fieldRo.dbFieldName ?? (await this.fieldService.generateDbFieldName(tableId, fieldName)); + fieldRo.dbFieldName ?? + (await this.fieldService.generateDbFieldName(tableId, fieldName, routingOptions)); if (fieldRo.dbFieldName) { const existField = await this.prismaService.txClient().field.findFirst({ @@ -1769,6 +1871,7 @@ export class FieldSupplementService { this.validateFormattingShowAs(fieldVo); this.validateAiConfig(fieldVo); await this.validatePrimaryConfigurations(tableId, [fieldVo]); + await this.assertNoNewCrossSpaceField(tableId, fieldVo); return fieldVo; } @@ -1832,7 +1935,12 @@ export class FieldSupplementService { } } - async prepareCreateFields(tableId: string, fieldRos: IFieldRo[], batchFieldVos?: IFieldVo[]) { + async prepareCreateFields( + tableId: string, + fieldRos: IFieldRo[], + batchFieldVos?: IFieldVo[], + routingOptions?: IDataDbRoutingOptions + ) { // throw error when dbFieldName is duplicated const fieldRoDbFieldNames = fieldRos .map((field) => field.dbFieldName) @@ -1869,7 +1977,11 @@ export class FieldSupplementService { fields.map((field) => field.name) ); - const dbFieldNames = await this.fieldService.generateDbFieldNames(tableId, uniqFieldNames); + const dbFieldNames = await this.fieldService.generateDbFieldNames( + tableId, + uniqFieldNames, + routingOptions + ); const fieldVos = fieldRos.map((fieldRo, index) => { const field = fields[index]; @@ -1888,6 +2000,9 @@ export class FieldSupplementService { return fieldVo; }); await this.validatePrimaryConfigurations(tableId, fieldVos); + for (const fieldVo of fieldVos) { + await this.assertNoNewCrossSpaceField(tableId, fieldVo); + } return fieldVos; } @@ -1916,6 +2031,7 @@ export class FieldSupplementService { )) as IFieldVo; this.validateFormattingShowAs(fieldVo); this.validateAiConfig(fieldVo); + await this.assertNoNewCrossSpaceField(tableId, fieldVo, oldFieldVo); return { ...fieldVo, @@ -1953,7 +2069,11 @@ export class FieldSupplementService { }); } - async generateSymmetricField(tableId: string, field: LinkFieldDto) { + async generateSymmetricField( + tableId: string, + field: LinkFieldDto, + routingOptions?: IDataDbRoutingOptions + ) { if (!field.options.symmetricFieldId) { throw new CustomHttpException( 'symmetricFieldId is required', @@ -1984,7 +2104,8 @@ export class FieldSupplementService { const isMultipleCellValue = isMultiValueLink(relationship) || undefined; const dbFieldName = await this.fieldService.generateDbFieldName( field.options.foreignTableId, - fieldName + fieldName, + routingOptions ); return createFieldInstanceByVo({ @@ -2013,26 +2134,28 @@ export class FieldSupplementService { async cleanForeignKey(options: ILinkFieldOptions) { const { fkHostTableName, relationship, selfKeyName, foreignKeyName, isOneWay } = options; + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForTable( + options.foreignTableId, + { + useTransaction: true, + } + ); const dropTable = async (tableName: string) => { // Use provider to generate dialect-correct DROP TABLE SQL const sql = this.dbProvider.dropTable(tableName); - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await dataPrisma.$executeRawUnsafe(sql); }; const dropColumn = async (tableName: string, columnName: string) => { const sqls = this.dbProvider.dropColumnAndIndex(tableName, columnName, `index_${columnName}`); for (const sql of sqls) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await dataPrisma.$executeRawUnsafe(sql); } // Drop the associated order column if it exists const orderColumn = `${columnName}_order`; - const exists = await this.dbProvider.checkColumnExist( - tableName, - orderColumn, - this.dataPrismaService.txClient() - ); + const exists = await this.dbProvider.checkColumnExist(tableName, orderColumn, dataPrisma); if (exists) { const dropOrderSqls = this.dbProvider.dropColumnAndIndex( tableName, @@ -2040,7 +2163,7 @@ export class FieldSupplementService { `index_${orderColumn}` ); for (const sql of dropOrderSqls) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await dataPrisma.$executeRawUnsafe(sql); } } }; diff --git a/apps/nestjs-backend/src/features/field/field-duplicate/field-duplicate.service.ts b/apps/nestjs-backend/src/features/field/field-duplicate/field-duplicate.service.ts index 5ba84f247e..22f45640c6 100644 --- a/apps/nestjs-backend/src/features/field/field-duplicate/field-duplicate.service.ts +++ b/apps/nestjs-backend/src/features/field/field-duplicate/field-duplicate.service.ts @@ -18,7 +18,6 @@ import { isLinkLookupOptions, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { IBaseJson, IFieldJson, IFieldWithTableIdJson } from '@teable/openapi'; import { Knex } from 'knex'; import { pick, get } from 'lodash'; @@ -26,6 +25,8 @@ import { InjectModel } from 'nest-knexjs'; import { CustomHttpException } from '../../../custom.exception'; import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import type { IDataDbRoutingOptions } from '../../../global/data-db-client-manager.service'; +import { DatabaseRouter } from '../../../global/database-router.service'; import { DATA_KNEX } from '../../../global/knex/knex.module'; import { extractFieldReferences } from '../../../utils'; import { DEFAULT_EXPRESSION } from '../../base/constant'; @@ -42,7 +43,7 @@ export class FieldDuplicateService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly fieldOpenApiService: FieldOpenApiService, private readonly linkFieldQueryService: LinkFieldQueryService, @InjectModel(DATA_KNEX) private readonly knex: Knex, @@ -50,7 +51,11 @@ export class FieldDuplicateService { private readonly tableDomainQueryService: TableDomainQueryService ) {} - async createCommonFields(fields: IFieldWithTableIdJson[], fieldMap: Record) { + async createCommonFields( + fields: IFieldWithTableIdJson[], + fieldMap: Record, + routingOptions?: IDataDbRoutingOptions + ) { const byTable = new Map(); for (const field of fields) { const list = byTable.get(field.targetTableId) ?? []; @@ -69,23 +74,38 @@ export class FieldDuplicateService { }) ); - const newFieldVos = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + const newFieldVos = await this.fieldOpenApiService.createFieldsByRo( + targetTableId, + fieldRos, + routingOptions + ); for (let index = 0; index < tableFields.length; index++) { const original = tableFields[index]; const newFieldVo = newFieldVos[index]; - await this.replenishmentConstraint(newFieldVo.id, targetTableId, original.order, { - notNull: original.notNull, - unique: original.unique, - dbFieldName: newFieldVo.dbFieldName, - isPrimary: original.isPrimary, - }); + await this.replenishmentConstraint( + newFieldVo.id, + targetTableId, + original.order, + { + notNull: original.notNull, + unique: original.unique, + dbFieldName: newFieldVo.dbFieldName, + isPrimary: original.isPrimary, + }, + undefined, + routingOptions + ); fieldMap[original.id] = newFieldVo.id; } } } - async createButtonFields(fields: IFieldWithTableIdJson[], fieldMap: Record) { + async createButtonFields( + fields: IFieldWithTableIdJson[], + fieldMap: Record, + routingOptions?: IDataDbRoutingOptions + ) { const newFields = fields.map((field) => { const { options } = field; return { @@ -96,12 +116,13 @@ export class FieldDuplicateService { }, }; }) as IFieldWithTableIdJson[]; - return await this.createCommonFields(newFields, fieldMap); + return await this.createCommonFields(newFields, fieldMap, routingOptions); } async createTmpPrimaryFormulaFields( primaryFormulaFields: IFieldWithTableIdJson[], - fieldMap: Record + fieldMap: Record, + routingOptions?: IDataDbRoutingOptions ) { const byTable = new Map(); for (const field of primaryFormulaFields) { @@ -124,7 +145,11 @@ export class FieldDuplicateService { }) ); - const newFields = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + const newFields = await this.fieldOpenApiService.createFieldsByRo( + targetTableId, + fieldRos, + routingOptions + ); for (let index = 0; index < tableFields.length; index++) { const original = tableFields[index]; @@ -140,12 +165,19 @@ export class FieldDuplicateService { }); } - await this.replenishmentConstraint(newField.id, targetTableId, original.order, { - notNull: original.notNull, - unique: original.unique, - dbFieldName: original.dbFieldName, - isPrimary: original.isPrimary, - }); + await this.replenishmentConstraint( + newField.id, + targetTableId, + original.order, + { + notNull: original.notNull, + unique: original.unique, + dbFieldName: original.dbFieldName, + isPrimary: original.isPrimary, + }, + undefined, + routingOptions + ); fieldMap[original.id] = newField.id; if (original.hasError) { @@ -223,7 +255,9 @@ export class FieldDuplicateService { this.logger.debug( "Executing SQL to modify primary formula field's column: " + alterTableQuery ); - await this.dataPrismaService.txClient().$executeRawUnsafe(alterTableQuery); + await this.databaseRouter.executeDataPrismaForTable(targetTableId, alterTableQuery, { + useTransaction: true, + }); } await this.prismaService.txClient().field.update({ where: { @@ -281,7 +315,8 @@ export class FieldDuplicateService { linkFields: IFieldWithTableIdJson[], tableIdMap: Record, fieldMap: Record, - fkMap: Record + fkMap: Record, + routingOptions?: IDataDbRoutingOptions ) { const selfLinkFields = linkFields.filter( ({ options, sourceTableId }) => @@ -310,19 +345,34 @@ export class FieldDuplicateService { ({ id }) => ![...selfLinkFields, ...crossBaseLinkFields].map(({ id }) => id).includes(id) ); - await this.createSelfLinkFields(selfLinkFields, fieldMap, fkMap); + await this.createSelfLinkFields(selfLinkFields, fieldMap, fkMap, routingOptions); // deal with cross base link fields - await this.createCommonLinkFields(crossBaseLinkFields, tableIdMap, fieldMap, fkMap, true); + await this.createCommonLinkFields( + crossBaseLinkFields, + tableIdMap, + fieldMap, + fkMap, + true, + routingOptions + ); - await this.createCommonLinkFields(commonLinkFields, tableIdMap, fieldMap, fkMap); + await this.createCommonLinkFields( + commonLinkFields, + tableIdMap, + fieldMap, + fkMap, + false, + routingOptions + ); } // eslint-disable-next-line sonarjs/cognitive-complexity async createSelfLinkFields( fields: IFieldWithTableIdJson[], fieldMap: Record, - fkMap: Record + fkMap: Record, + routingOptions?: IDataDbRoutingOptions ) { const twoWaySelfLinkFields = fields.filter( ({ options }) => !(options as ILinkFieldOptions).isOneWay @@ -366,7 +416,11 @@ export class FieldDuplicateService { }) ); - const newFieldVos = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + const newFieldVos = await this.fieldOpenApiService.createFieldsByRo( + targetTableId, + fieldRos, + routingOptions + ); const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ where: { @@ -438,7 +492,11 @@ export class FieldDuplicateService { }; }); - const newFieldVos = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + const newFieldVos = await this.fieldOpenApiService.createFieldsByRo( + targetTableId, + fieldRos, + routingOptions + ); const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ where: { @@ -483,7 +541,8 @@ export class FieldDuplicateService { tableIdMap: Record, fieldMap: Record, fkMap: Record, - allowCrossBase: boolean = false + allowCrossBase: boolean = false, + routingOptions?: IDataDbRoutingOptions ) { const oneWayFields = fields.filter(({ options }) => (options as ILinkFieldOptions).isOneWay); const twoWayFields = fields.filter(({ options }) => !(options as ILinkFieldOptions).isOneWay); @@ -513,7 +572,11 @@ export class FieldDuplicateService { } ); - const newFieldVos = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + const newFieldVos = await this.fieldOpenApiService.createFieldsByRo( + targetTableId, + fieldRos, + routingOptions + ); const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ where: { @@ -592,7 +655,11 @@ export class FieldDuplicateService { }; }); - const newFieldVos = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + const newFieldVos = await this.fieldOpenApiService.createFieldsByRo( + targetTableId, + fieldRos, + routingOptions + ); const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ where: { @@ -677,10 +744,13 @@ export class FieldDuplicateService { }); if (genDbFieldName !== dbFieldName) { + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForTable(targetTableId, { + useTransaction: true, + }); const exists = await this.dbProvider.checkColumnExist( resolvedDbTableName, genDbFieldName, - this.dataPrismaService.txClient() + dataPrisma ); if (exists) { // Debug logging for rename operation to diagnose failures @@ -700,7 +770,9 @@ export class FieldDuplicateService { for (const sql of alterTableSql) { // eslint-disable-next-line no-console console.log('[repairSymmetricField] executing SQL', sql); - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(targetTableId, sql, { + useTransaction: true, + }); } } } @@ -820,7 +892,8 @@ export class FieldDuplicateService { dependFields: IFieldWithTableIdJson[], tableIdMap: Record, fieldMap: Record, - scope: 'base' | 'table' = 'base' + scope: 'base' | 'table' = 'base', + routingOptions?: IDataDbRoutingOptions ): Promise { if (!dependFields.length) return; @@ -847,7 +920,9 @@ export class FieldDuplicateService { curField, tableIdMap, fieldMap, - scope + scope, + false, + routingOptions ); continue; } @@ -861,7 +936,8 @@ export class FieldDuplicateService { tableIdMap, fieldMap, scope, - true + true, + routingOptions ); } else if (!countMap[curField.id] || countMap[curField.id] < maxCount) { dependFields.push(curField); @@ -891,7 +967,8 @@ export class FieldDuplicateService { async bootstrapPrimaryDependencyFields( fields: IFieldWithTableIdJson[], - sourceToTargetFieldMap: Record + sourceToTargetFieldMap: Record, + routingOptions?: IDataDbRoutingOptions ) { for (const field of fields) { if (!field.isPrimary || !field.aiConfig || field.isLookup) continue; @@ -909,13 +986,18 @@ export class FieldDuplicateService { order, } = field; - const newField = await this.fieldOpenApiService.createField(targetTableId, { - type, - dbFieldName, - description, - options, - name, - }); + const newField = await this.fieldOpenApiService.createField( + targetTableId, + { + type, + dbFieldName, + description, + options, + name, + }, + undefined, + routingOptions + ); await this.replenishmentConstraint(newField.id, targetTableId, order, { notNull, @@ -935,7 +1017,8 @@ export class FieldDuplicateService { tableIdMap: Record, sourceToTargetFieldMap: Record, scope: 'base' | 'table' = 'base', - hasError = false + hasError = false, + routingOptions?: IDataDbRoutingOptions ) { const hasFieldError = Boolean(field.hasError); const isAiConfig = field.aiConfig && !field.isLookup; @@ -948,7 +1031,12 @@ export class FieldDuplicateService { if (shouldConvertErroredComputed) { // During base import, persist errored computed fields as plain text so users keep the data. - await this.duplicateErroredComputedFieldAsText(targetTableId, field, sourceToTargetFieldMap); + await this.duplicateErroredComputedFieldAsText( + targetTableId, + field, + sourceToTargetFieldMap, + routingOptions + ); return; } @@ -968,14 +1056,16 @@ export class FieldDuplicateService { targetTableId, field, tableIdMap, - sourceToTargetFieldMap + sourceToTargetFieldMap, + routingOptions ); break; case isAiConfig: await this.duplicateFieldAiConfig( targetTableId, field as unknown as IFieldInstance, - sourceToTargetFieldMap + sourceToTargetFieldMap, + routingOptions ); break; case isRollup: @@ -984,7 +1074,8 @@ export class FieldDuplicateService { targetTableId, field, tableIdMap, - sourceToTargetFieldMap + sourceToTargetFieldMap, + routingOptions ); break; case isConditionalRollup: @@ -993,7 +1084,8 @@ export class FieldDuplicateService { targetTableId, field, tableIdMap, - sourceToTargetFieldMap + sourceToTargetFieldMap, + routingOptions ); break; case isFormula: @@ -1001,7 +1093,8 @@ export class FieldDuplicateService { targetTableId, field, sourceToTargetFieldMap, - hasError || hasFieldError + hasError || hasFieldError, + routingOptions ); } } @@ -1009,7 +1102,8 @@ export class FieldDuplicateService { private async duplicateErroredComputedFieldAsText( targetTableId: string, field: IFieldWithTableIdJson, - sourceToTargetFieldMap: Record + sourceToTargetFieldMap: Record, + routingOptions?: IDataDbRoutingOptions ) { const { id, name, description, dbFieldName, order, notNull, unique, isPrimary } = field; @@ -1023,7 +1117,12 @@ export class FieldDuplicateService { createFieldRo.dbFieldName = dbFieldName; } - const newField = await this.fieldOpenApiService.createField(targetTableId, createFieldRo); + const newField = await this.fieldOpenApiService.createField( + targetTableId, + createFieldRo, + undefined, + routingOptions + ); await this.replenishmentConstraint(newField.id, targetTableId, order, { notNull, @@ -1040,7 +1139,8 @@ export class FieldDuplicateService { targetTableId: string, field: IFieldWithTableIdJson, tableIdMap: Record, - sourceToTargetFieldMap: Record + sourceToTargetFieldMap: Record, + routingOptions?: IDataDbRoutingOptions ) { const { dbFieldName, @@ -1101,23 +1201,28 @@ export class FieldDuplicateService { const effectiveLookupFieldId = hasError ? mockFieldId : (mappedLookupFieldId as string); - newField = await this.fieldOpenApiService.createField(targetTableId, { - type: (hasError ? mockType : lookupFieldType) as FieldType, - dbFieldName, - description, - isLookup: true, - isConditionalLookup: true, - name, - options, - lookupOptions: { - baseId: remappedLookupOptions?.baseId ?? conditionalOptions?.baseId, - foreignTableId: remappedLookupOptions?.foreignTableId ?? mappedForeignTableId, - lookupFieldId: effectiveLookupFieldId, - filter: remappedLookupOptions?.filter ?? conditionalOptions?.filter ?? null, - sort: remappedLookupOptions?.sort ?? conditionalOptions?.sort ?? undefined, - limit: remappedLookupOptions?.limit ?? conditionalOptions?.limit ?? undefined, + newField = await this.fieldOpenApiService.createField( + targetTableId, + { + type: (hasError ? mockType : lookupFieldType) as FieldType, + dbFieldName, + description, + isLookup: true, + isConditionalLookup: true, + name, + options, + lookupOptions: { + baseId: remappedLookupOptions?.baseId ?? conditionalOptions?.baseId, + foreignTableId: remappedLookupOptions?.foreignTableId ?? mappedForeignTableId, + lookupFieldId: effectiveLookupFieldId, + filter: remappedLookupOptions?.filter ?? conditionalOptions?.filter ?? null, + sort: remappedLookupOptions?.sort ?? conditionalOptions?.sort ?? undefined, + limit: remappedLookupOptions?.limit ?? conditionalOptions?.limit ?? undefined, + }, }, - }); + undefined, + routingOptions + ); if (hasError) { await this.prismaService.txClient().field.update({ @@ -1148,25 +1253,30 @@ export class FieldDuplicateService { const { foreignTableId, linkFieldId, lookupFieldId } = lookupOptionsRo; const isSelfLink = foreignTableId === sourceTableId; - newField = await this.fieldOpenApiService.createField(targetTableId, { - type: (hasError ? mockType : lookupFieldType) as FieldType, - dbFieldName, - description, - isLookup: true, - lookupOptions: { - foreignTableId: - (isSelfLink ? targetTableId : tableIdMap[foreignTableId]) || foreignTableId, - linkFieldId: sourceToTargetFieldMap[linkFieldId], - lookupFieldId: isSelfLink - ? hasError - ? mockFieldId - : sourceToTargetFieldMap[lookupFieldId] - : hasError - ? mockFieldId - : sourceToTargetFieldMap[lookupFieldId] || lookupFieldId, + newField = await this.fieldOpenApiService.createField( + targetTableId, + { + type: (hasError ? mockType : lookupFieldType) as FieldType, + dbFieldName, + description, + isLookup: true, + lookupOptions: { + foreignTableId: + (isSelfLink ? targetTableId : tableIdMap[foreignTableId]) || foreignTableId, + linkFieldId: sourceToTargetFieldMap[linkFieldId], + lookupFieldId: isSelfLink + ? hasError + ? mockFieldId + : sourceToTargetFieldMap[lookupFieldId] + : hasError + ? mockFieldId + : sourceToTargetFieldMap[lookupFieldId] || lookupFieldId, + }, + name, }, - name, - }); + undefined, + routingOptions + ); if (hasError) { await this.prismaService.txClient().field.update({ @@ -1199,7 +1309,8 @@ export class FieldDuplicateService { targetTableId: string, fieldInstance: IFieldWithTableIdJson, tableIdMap: Record, - sourceToTargetFieldMap: Record + sourceToTargetFieldMap: Record, + routingOptions?: IDataDbRoutingOptions ) { const { dbFieldName, @@ -1221,25 +1332,31 @@ export class FieldDuplicateService { const isSelfLink = foreignTableId === sourceTableId; const mockFieldId = Object.values(sourceToTargetFieldMap)[0]; - const newField = await this.fieldOpenApiService.createField(targetTableId, { - type: FieldType.Rollup, - dbFieldName, - description, - lookupOptions: { - // foreignTableId may are cross base table id, so we need to use tableIdMap to get the target table id - foreignTableId: (isSelfLink ? targetTableId : tableIdMap[foreignTableId]) || foreignTableId, - linkFieldId: sourceToTargetFieldMap[linkFieldId], - lookupFieldId: isSelfLink - ? hasError - ? mockFieldId - : sourceToTargetFieldMap[lookupFieldId] - : hasError - ? mockFieldId - : sourceToTargetFieldMap[lookupFieldId] || lookupFieldId, + const newField = await this.fieldOpenApiService.createField( + targetTableId, + { + type: FieldType.Rollup, + dbFieldName, + description, + lookupOptions: { + // foreignTableId may are cross base table id, so we need to use tableIdMap to get the target table id + foreignTableId: + (isSelfLink ? targetTableId : tableIdMap[foreignTableId]) || foreignTableId, + linkFieldId: sourceToTargetFieldMap[linkFieldId], + lookupFieldId: isSelfLink + ? hasError + ? mockFieldId + : sourceToTargetFieldMap[lookupFieldId] + : hasError + ? mockFieldId + : sourceToTargetFieldMap[lookupFieldId] || lookupFieldId, + }, + options, + name, }, - options, - name, - }); + undefined, + routingOptions + ); await this.replenishmentConstraint(newField.id, targetTableId, fieldInstance.order, { notNull, unique, @@ -1270,7 +1387,8 @@ export class FieldDuplicateService { targetTableId: string, fieldInstance: IFieldWithTableIdJson, tableIdMap: Record, - sourceToTargetFieldMap: Record + sourceToTargetFieldMap: Record, + routingOptions?: IDataDbRoutingOptions ) { const { dbFieldName, @@ -1302,13 +1420,18 @@ export class FieldDuplicateService { false ) as IConditionalRollupFieldOptions; - const newField = await this.fieldOpenApiService.createField(targetTableId, { - type: FieldType.ConditionalRollup, - dbFieldName, - description, - options: remappedOptions, - name, - }); + const newField = await this.fieldOpenApiService.createField( + targetTableId, + { + type: FieldType.ConditionalRollup, + dbFieldName, + description, + options: remappedOptions, + name, + }, + undefined, + routingOptions + ); await this.replenishmentConstraint(newField.id, targetTableId, fieldInstance.order, { notNull, @@ -1335,7 +1458,8 @@ export class FieldDuplicateService { targetTableId: string, fieldInstance: IFieldWithTableIdJson, sourceToTargetFieldMap: Record, - hasError: boolean = false + hasError: boolean = false, + routingOptions?: IDataDbRoutingOptions ) { const { type, @@ -1353,20 +1477,25 @@ export class FieldDuplicateService { } = fieldInstance; const { expression } = options as IFormulaFieldOptions; const newExpression = replaceStringByMap(expression, { sourceToTargetFieldMap }); - const newField = await this.fieldOpenApiService.createField(targetTableId, { - type, - dbFieldName, - description, - options: { - ...options, - expression: hasError - ? DEFAULT_EXPRESSION - : newExpression - ? JSON.parse(newExpression) - : undefined, + const newField = await this.fieldOpenApiService.createField( + targetTableId, + { + type, + dbFieldName, + description, + options: { + ...options, + expression: hasError + ? DEFAULT_EXPRESSION + : newExpression + ? JSON.parse(newExpression) + : undefined, + }, + name, }, - name, - }); + undefined, + routingOptions + ); await this.replenishmentConstraint(newField.id, targetTableId, fieldInstance.order, { notNull, unique, @@ -1426,7 +1555,9 @@ export class FieldDuplicateService { ); for (const alterTableQuery of modifyColumnSql) { - await this.dataPrismaService.txClient().$executeRawUnsafe(alterTableQuery); + await this.databaseRouter.executeDataPrismaForTable(targetTableId, alterTableQuery, { + useTransaction: true, + }); } await this.prismaService.txClient().field.update({ @@ -1464,7 +1595,8 @@ export class FieldDuplicateService { private async duplicateFieldAiConfig( targetTableId: string, fieldInstance: IFieldInstance, - sourceToTargetFieldMap: Record + sourceToTargetFieldMap: Record, + routingOptions?: IDataDbRoutingOptions ) { if (!fieldInstance.aiConfig) return; @@ -1472,14 +1604,19 @@ export class FieldDuplicateService { fieldInstance; const aiConfig = this.mapAiConfigForDuplicate(fieldInstance.aiConfig, sourceToTargetFieldMap); - const newField = await this.fieldOpenApiService.createField(targetTableId, { - type, - dbFieldName, - description, - options, - aiConfig, - name, - }); + const newField = await this.fieldOpenApiService.createField( + targetTableId, + { + type, + dbFieldName, + description, + options, + aiConfig, + name, + }, + undefined, + routingOptions + ); await this.replenishmentConstraint(newField.id, targetTableId, 1, { notNull, @@ -1520,7 +1657,8 @@ export class FieldDuplicateService { dbFieldName, isPrimary, }: { notNull?: boolean; unique?: boolean; dbFieldName: string; isPrimary?: boolean }, - dbTableName?: string + dbTableName?: string, + routingOptions?: IDataDbRoutingOptions ) { await this.prismaService.txClient().field.update({ where: { @@ -1574,7 +1712,11 @@ export class FieldDuplicateService { .toSQL(); for (const sql of fieldValidationSqls) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql.sql); + await this.databaseRouter.executeDataPrismaForTable( + targetTableId, + sql.sql, + routingOptions ?? { useTransaction: true } + ); } } } diff --git a/apps/nestjs-backend/src/features/field/field.service.spec.ts b/apps/nestjs-backend/src/features/field/field.service.spec.ts index 79635f98f0..c367a66542 100644 --- a/apps/nestjs-backend/src/features/field/field.service.spec.ts +++ b/apps/nestjs-backend/src/features/field/field.service.spec.ts @@ -3,6 +3,7 @@ import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; import { CellValueType, DbFieldType, FieldType, OpName } from '@teable/core'; import type { IFieldVo, INumberFormatting, ISetFieldPropertyOpContext } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; import { GlobalModule } from '../../global/global.module'; import { FieldModule } from './field.module'; import { FieldService } from './field.service'; @@ -23,6 +24,33 @@ describe('FieldService', () => { expect(service).toBeDefined(); }); + it('reads table metadata from the active transaction when routing asks for it', async () => { + const txFindUnique = vi.fn().mockResolvedValue({ dbTableName: 'bse_test.tbl_tx' }); + const rootFindUnique = vi.fn(); + const service = Object.create(FieldService.prototype) as FieldService; + Object.assign(service, { + prismaService: { + txClient: vi.fn(() => ({ + tableMeta: { + findUnique: txFindUnique, + }, + })), + tableMeta: { + findUnique: rootFindUnique, + }, + } as unknown as PrismaService, + }); + + await expect(service.getDbTableName('tbl_tx', { useTransaction: true })).resolves.toBe( + 'bse_test.tbl_tx' + ); + expect(txFindUnique).toHaveBeenCalledWith({ + where: { id: 'tbl_tx' }, + select: { dbTableName: true }, + }); + expect(rootFindUnique).not.toHaveBeenCalled(); + }); + describe('applyFieldPropertyOpsAndCreateInstance', () => { it('should apply field property operations and return field instance', () => { // Create a mock field VO diff --git a/apps/nestjs-backend/src/features/field/field.service.ts b/apps/nestjs-backend/src/features/field/field.service.ts index 4b9ec2ffe4..4fd74ed933 100644 --- a/apps/nestjs-backend/src/features/field/field.service.ts +++ b/apps/nestjs-backend/src/features/field/field.service.ts @@ -22,7 +22,6 @@ import type { } from '@teable/core'; import type { Field as RawField, Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { instanceToPlain } from 'class-transformer'; import { Knex } from 'knex'; import { keyBy, sortBy, omit } from 'lodash'; @@ -32,6 +31,8 @@ import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; import { DropColumnOperationType } from '../../db-provider/drop-database-column-query/drop-database-column-field-visitor.interface'; +import type { IDataDbRoutingOptions } from '../../global/data-db-client-manager.service'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import type { IReadonlyAdapterService } from '../../share-db/interface'; import { RawOpType } from '../../share-db/interface'; @@ -64,7 +65,7 @@ export class FieldService implements IReadonlyAdapterService { constructor( private readonly batchService: BatchService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly dataLoaderService: DataLoaderService, private readonly cls: ClsService, @InjectDbProvider() private readonly dbProvider: IDbProvider, @@ -83,13 +84,19 @@ export class FieldService implements IReadonlyAdapterService { this.dataLoaderService.field.invalidateTables(ids); } - async generateDbFieldName(tableId: string, name: string): Promise { + async generateDbFieldName( + tableId: string, + name: string, + routingOptions?: IDataDbRoutingOptions + ): Promise { let dbFieldName = convertNameToValidCharacter(name, 40); - const query = this.dbProvider.columnInfo(await this.getDbTableName(tableId)); - const columns = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ name: string }[]>(query); + const query = this.dbProvider.columnInfo(await this.getDbTableName(tableId, routingOptions)); + const columns = await this.databaseRouter.queryDataPrismaForTable<{ name: string }[]>( + tableId, + query, + routingOptions + ); // fallback logic if (columns.some((column) => column.name === dbFieldName)) { dbFieldName += new Date().getTime(); @@ -97,11 +104,17 @@ export class FieldService implements IReadonlyAdapterService { return dbFieldName; } - async generateDbFieldNames(tableId: string, names: string[]) { - const query = this.dbProvider.columnInfo(await this.getDbTableName(tableId)); - const columns = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ name: string }[]>(query); + async generateDbFieldNames( + tableId: string, + names: string[], + routingOptions?: IDataDbRoutingOptions + ) { + const query = this.dbProvider.columnInfo(await this.getDbTableName(tableId, routingOptions)); + const columns = await this.databaseRouter.queryDataPrismaForTable<{ name: string }[]>( + tableId, + query, + routingOptions + ); return names .map((name) => convertNameToValidCharacter(name, 40)) .map((dbFieldName) => { @@ -463,7 +476,9 @@ export class FieldService implements IReadonlyAdapterService { // Execute all queries (main table alteration + any additional queries like junction tables) for (const query of alterTableQueries) { this.logger.debug(`Executing alter table query: ${query}`); - await this.dataPrismaService.txClient().$executeRawUnsafe(query); + await this.databaseRouter.executeDataPrismaForTable(tableId, query, { + useTransaction: true, + }); } if (unique) { @@ -487,7 +502,9 @@ export class FieldService implements IReadonlyAdapterService { }); }) .toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(fieldValidationQuery); + await this.databaseRouter.executeDataPrismaForTable(tableId, fieldValidationQuery, { + useTransaction: true, + }); } if (notNull) { @@ -537,7 +554,7 @@ export class FieldService implements IReadonlyAdapterService { ); for (const alterTableQuery of alterTableSql) { - await this.dataPrismaService.txClient().$executeRawUnsafe(alterTableQuery); + await this.databaseRouter.executeDataPrismaForTable(tableId, alterTableQuery); } } } @@ -575,9 +592,10 @@ export class FieldService implements IReadonlyAdapterService { // Link fields in Teable maintain a persisted display column on the host table; skipping // the physical rename causes mismatches during computed updates (e.g., UPDATE ... FROM ...). const columnInfoQuery = this.dbProvider.columnInfo(table.dbTableName); - const columns = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ name: string }[]>(columnInfoQuery); + const columns = await this.databaseRouter.queryDataPrismaForTable<{ name: string }[]>( + table.id, + columnInfoQuery + ); const columnNames = new Set(columns.map((column) => column.name)); if (columnNames.has(newDbFieldName)) { @@ -601,7 +619,7 @@ export class FieldService implements IReadonlyAdapterService { ); for (const alterTableQuery of alterTableSql) { - await this.dataPrismaService.txClient().$executeRawUnsafe(alterTableQuery); + await this.databaseRouter.executeDataPrismaForTable(table.id, alterTableQuery); } } @@ -666,11 +684,11 @@ export class FieldService implements IReadonlyAdapterService { await handleDBValidationErrors({ fn: async () => { if (resetFieldQuery) { - await this.dataPrismaService.txClient().$executeRawUnsafe(resetFieldQuery); + await this.databaseRouter.executeDataPrismaForTable(tableId, resetFieldQuery); } for (const alterTableQuery of modifyColumnSql) { - await this.dataPrismaService.txClient().$executeRawUnsafe(alterTableQuery); + await this.databaseRouter.executeDataPrismaForTable(tableId, alterTableQuery); } }, handleUniqueError: () => { @@ -700,11 +718,22 @@ export class FieldService implements IReadonlyAdapterService { }); } - async findUniqueIndexesForField(dbTableName: string, dbFieldName: string) { + async findUniqueIndexesForField( + tableIdOrDbTableName: string, + dbTableNameOrDbFieldName: string, + maybeDbFieldName?: string + ) { + const tableId = maybeDbFieldName ? tableIdOrDbTableName : undefined; + const dbTableName = maybeDbFieldName ? dbTableNameOrDbFieldName : tableIdOrDbTableName; + const dbFieldName = maybeDbFieldName ?? dbTableNameOrDbFieldName; const indexesQuery = this.dbProvider.getTableIndexes(dbTableName); - const indexes = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ name: string; columns: string; isUnique: boolean }[]>(indexesQuery); + const indexes = tableId + ? await this.databaseRouter.queryDataPrismaForTable< + { name: string; columns: string; isUnique: boolean }[] + >(tableId, indexesQuery) + : await this.databaseRouter.queryDataPrismaForBase< + { name: string; columns: string; isUnique: boolean }[] + >(dbTableName.split('.')[0], indexesQuery); return indexes .filter((index) => { @@ -729,7 +758,7 @@ export class FieldService implements IReadonlyAdapterService { dbFieldName: true, type: true, isLookup: true, - table: { select: { dbTableName: true, name: true } }, + table: { select: { id: true, dbTableName: true, name: true } }, }, }); @@ -747,7 +776,7 @@ export class FieldService implements IReadonlyAdapterService { } const dbTableName = table.dbTableName; - const matchedIndexes = await this.findUniqueIndexesForField(dbTableName, dbFieldName); + const matchedIndexes = await this.findUniqueIndexesForField(table.id, dbTableName, dbFieldName); const fieldValidationSqls = this.knex.schema .alterTable(dbTableName, (table) => { @@ -772,7 +801,7 @@ export class FieldService implements IReadonlyAdapterService { await handleDBValidationErrors({ fn: () => { return Promise.all( - executeSqls.map((sql) => this.dataPrismaService.txClient().$executeRawUnsafe(sql)) + executeSqls.map((sql) => this.databaseRouter.executeDataPrismaForTable(table.id, sql)) ); }, handleUniqueError: () => { @@ -891,7 +920,18 @@ export class FieldService implements IReadonlyAdapterService { return fields.map((field) => createFieldInstanceByVo(field)); } - async getDbTableName(tableId: string) { + async getDbTableName(tableId: string, routingOptions?: IDataDbRoutingOptions) { + if (routingOptions?.useTransaction) { + const tableMeta = await this.prismaService.txClient().tableMeta.findUnique({ + where: { id: tableId }, + select: { dbTableName: true }, + }); + if (!tableMeta) { + throw new NotFoundException(`Table not found: ${tableId}`); + } + return tableMeta.dbTableName; + } + const [tableMeta] = await this.dataLoaderService.table.loadByIds([tableId]); if (!tableMeta) { throw new NotFoundException(`Table not found: ${tableId}`); @@ -1064,7 +1104,7 @@ export class FieldService implements IReadonlyAdapterService { tableDomain ); for (const sql of sqls) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } } catch (e) { this.logger.warn( @@ -1499,7 +1539,7 @@ export class FieldService implements IReadonlyAdapterService { tableDomain ); for (const sql of modifyColumnSql) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } return; } @@ -1520,7 +1560,7 @@ export class FieldService implements IReadonlyAdapterService { tableDomain ); for (const sql of modifyColumnSql) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } return; } @@ -1557,7 +1597,7 @@ export class FieldService implements IReadonlyAdapterService { // Execute the column modification for (const sql of modifyColumnSql) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } } @@ -1649,7 +1689,7 @@ export class FieldService implements IReadonlyAdapterService { // Execute the column modification for (const sql of modifyColumnSql) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(dependentTableId, sql); } } } catch (error) { diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts index 5a0f1b3cf2..52e3b8dd32 100644 --- a/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts @@ -75,6 +75,8 @@ const createService = () => {} as never, {} as never, {} as never, + {} as never, + {} as never, {} as never ) as unknown as ITestFieldOpenApiV2Service; @@ -1031,6 +1033,8 @@ describe('FieldOpenApiV2Service normalizeFieldVo', () => { {} as never, {} as never, {} as never, + {} as never, + {} as never, {} as never ) as unknown as ITestFieldOpenApiV2Service; @@ -1429,6 +1433,8 @@ describe('FieldOpenApiV2Service createField', () => { { field: { invalidateTables: vi.fn() } } as never, {} as never, {} as never, + {} as never, + {} as never, {} as never ) as unknown as ITestFieldOpenApiV2Service; @@ -1497,6 +1503,8 @@ describe('FieldOpenApiV2Service createField', () => { { field: { invalidateTables: vi.fn() } } as never, {} as never, {} as never, + {} as never, + {} as never, {} as never ) as unknown as ITestFieldOpenApiV2Service; @@ -1582,6 +1590,8 @@ describe('FieldOpenApiV2Service createFields', () => { { field: { invalidateTables: vi.fn() } } as never, {} as never, {} as never, + {} as never, + {} as never, {} as never ) as unknown as ITestFieldOpenApiV2Service; diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts index 7c63670231..f9b494905b 100644 --- a/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts @@ -21,6 +21,7 @@ import { type IUpdateFieldRo, type IViewVo, } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; import type { IDuplicateFieldRo, IPlanFieldVo } from '@teable/openapi'; import { mapDomainErrorToHttpError, @@ -74,6 +75,7 @@ import { } from '../../v2/v2-undo-redo.constants'; import { adjustFrozenField } from '../../view/utils/derive-frozen-fields'; import { ViewService } from '../../view/view.service'; +import { FieldSupplementService } from '../field-calculate/field-supplement.service'; import { FieldOpenApiService } from './field-open-api.service'; const internalServerError = 'Internal server error'; @@ -108,9 +110,30 @@ export class FieldOpenApiV2Service { private readonly dataLoaderService: DataLoaderService, private readonly fieldOpenApiService: FieldOpenApiService, private readonly viewService: ViewService, - private readonly cls: ClsService + private readonly cls: ClsService, + private readonly fieldSupplementService: FieldSupplementService, + private readonly prismaService: PrismaService ) {} + private async assertCrossSpaceForV2Field( + tableId: string, + v2Field: Record + ): Promise { + const readForeignTableId = (raw: unknown): string | undefined => { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; + const value = (raw as Record).foreignTableId; + return typeof value === 'string' ? value : undefined; + }; + const candidates = [ + readForeignTableId(v2Field.options), + readForeignTableId(v2Field.config), + readForeignTableId(v2Field.lookupOptions), + ].filter((x): x is string => Boolean(x)); + for (const foreignTableId of candidates) { + await this.fieldSupplementService.assertSameSpaceLinkTarget(tableId, foreignTableId); + } + } + private stripUndefinedDeep(value: unknown): unknown { if (Array.isArray(value)) { return value.map((item) => this.stripUndefinedDeep(item)); @@ -481,7 +504,7 @@ export class FieldOpenApiV2Service { fieldId: string, context?: IExecutionContext ): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); const tableMapper = container.resolve(v2CoreTokens.tableMapper); const tableIdResult = TableId.create(tableId); @@ -489,7 +512,7 @@ export class FieldOpenApiV2Service { throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); } - const queryContext = context ?? (await this.v2ContextFactory.createContext()); + const queryContext = context ?? (await this.v2ContextFactory.createContext(container)); const tableResult = await tableQueryService.getById(queryContext, tableIdResult.value); if (tableResult.isErr()) { const errMsg = tableResult.error.message ?? 'Table not found'; @@ -667,10 +690,10 @@ export class FieldOpenApiV2Service { context: IExecutionContext; table: Table; }> { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const tableIdResult = TableId.create(tableId); if (tableIdResult.isErr()) { throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); @@ -768,7 +791,8 @@ export class FieldOpenApiV2Service { } async getField(tableId: string, fieldId: string): Promise { - const context = await this.v2ContextFactory.createContext(); + const container = await this.v2ContainerService.getContainerForTable(tableId); + const context = await this.v2ContextFactory.createContext(container); return this.getFieldFromV2(tableId, fieldId, context); } @@ -1268,6 +1292,7 @@ export class FieldOpenApiV2Service { context ); const { hasAiConfig, nextAiConfig, v2Field } = preparedField; + await this.assertCrossSpaceForV2Field(tableId, v2Field); const legacyViewId = fieldRo && typeof fieldRo === 'object' && 'viewId' in fieldRo ? (fieldRo.viewId as string | undefined) @@ -1442,12 +1467,12 @@ export class FieldOpenApiV2Service { tableId: string, fieldId: string, duplicateFieldRo: IDuplicateFieldRo, - _windowId?: string + windowId?: string ): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const tableIdResult = TableId.create(tableId); if (tableIdResult.isErr()) { @@ -1465,6 +1490,39 @@ export class FieldOpenApiV2Service { ); } + // If the source field's foreign table lives in a different space, the v2 + // duplicate command would happily clone the cross-space relationship — + // bypassing the field-supplement rejection. Detect this up front and ask + // the caller to confirm before degrading the duplicate to single line text. + const sourceFieldRaw = await this.prismaService.txClient().field.findUnique({ + where: { id: fieldId, deletedTime: null }, + select: { + id: true, + name: true, + type: true, + isLookup: true, + options: true, + lookupOptions: true, + }, + }); + if (sourceFieldRaw) { + const isCrossSpace = await this.fieldSupplementService.isCrossSpaceField( + tableId, + sourceFieldRaw + ); + if (isCrossSpace) { + // Delegate to v1: it creates the new field as single line text and + // copies the source link/lookup values across as title text. Keeping + // the downgrade in one place avoids drift between v1 and v2. + return this.fieldOpenApiService.duplicateField( + tableId, + fieldId, + duplicateFieldRo, + windowId + ); + } + } + const duplicateResult = await executeDuplicateFieldEndpoint( context, { @@ -1493,10 +1551,10 @@ export class FieldOpenApiV2Service { } async deleteField(tableId: string, fieldId: string): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const tableIdResult = TableId.create(tableId); if (tableIdResult.isErr()) { throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); @@ -1548,10 +1606,10 @@ export class FieldOpenApiV2Service { } async deleteFields(tableId: string, fieldIds: string[]): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const tableIdResult = TableId.create(tableId); if (tableIdResult.isErr()) { throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); @@ -1614,9 +1672,9 @@ export class FieldOpenApiV2Service { } async updateField(tableId: string, fieldId: string, updateFieldRo: IUpdateFieldRo) { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const currentField = await this.getFieldFromV2(tableId, fieldId, context); const v2Input = { @@ -1656,9 +1714,9 @@ export class FieldOpenApiV2Service { convertFieldRo: IConvertFieldRo, executionOptions?: ConvertFieldExecutionOptions ) { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const shouldTrackUndoContext = executionOptions?.emitOperation !== false && Boolean(context.windowId && context.actorId); if (executionOptions?.undoRedoMode) { @@ -1688,6 +1746,7 @@ export class FieldOpenApiV2Service { replaceOptions: true, }, }; + await this.assertCrossSpaceForV2Field(tableId, v2Input.field as Record); const result = await executeUpdateFieldEndpoint(context, v2Input, commandBus); @@ -1738,13 +1797,13 @@ export class FieldOpenApiV2Service { direction: 'old' | 'new', undoRedoMode: 'undo' | 'redo' ): Promise { - const container = await this.v2ContainerService.getContainer(); - const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); - context.undoRedo = { mode: undoRedoMode }; - delete context.windowId; - for (const [tableId, opsByRecordId] of Object.entries(modifiedOps)) { + const container = await this.v2ContainerService.getContainerForTable(tableId); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(container); + context.undoRedo = { mode: undoRedoMode }; + delete context.windowId; + for (const [recordId, ops] of Object.entries(opsByRecordId)) { const fields: Record = {}; for (const op of ops) { diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts index 3769b8277d..38f19a3073 100644 --- a/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts @@ -7,6 +7,7 @@ import { FieldKeyType, FieldOpBuilder, FieldType, + HttpErrorCode, ViewType, generateFieldId, generateOperationId, @@ -53,11 +54,13 @@ import { Knex } from 'knex'; import { groupBy, isEqual, omit, pick } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { ThresholdConfig, IThresholdConfig } from '../../../configs/threshold.config'; +import { CustomHttpException } from '../../../custom.exception'; import { FieldReferenceCompatibilityException } from '../../../db-provider/filter-query/cell-value-filter.abstract'; import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; import { Events } from '../../../event-emitter/events'; +import { IDataDbRoutingOptions } from '../../../global/data-db-client-manager.service'; +import { DatabaseRouter } from '../../../global/database-router.service'; import type { IClsStore } from '../../../types/cls'; import { Timing } from '../../../utils/timing'; import { FieldCalculationService } from '../../calculation/field-calculation.service'; @@ -109,13 +112,17 @@ export type ILegacyDeleteFieldsPayloadSnapshot = { records: Awaited> | undefined; }; +type CreateFieldsOptions = { + restoreViewOrder?: boolean; +}; + @Injectable() export class FieldOpenApiService { private logger = new Logger(FieldOpenApiService.name); constructor( private readonly graphService: GraphService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly fieldService: FieldService, private readonly viewService: ViewService, private readonly viewOpenApiService: ViewOpenApiService, @@ -1149,7 +1156,9 @@ export class FieldOpenApiService { @Timing() async createFields( tableId: string, - fields: (IFieldVo & { columnMeta?: IColumnMeta; references?: string[] })[] + fields: (IFieldVo & { columnMeta?: IColumnMeta; references?: string[] })[], + routingOptions?: IDataDbRoutingOptions, + options: CreateFieldsOptions = {} ) { if (!fields.length) return; @@ -1183,6 +1192,7 @@ export class FieldOpenApiService { set.add(fieldId); }; + const columnMetaByFieldId = new Map(fields.map((field) => [field.id, field.columnMeta])); const createPayload = orderedFields.map((field) => { const { columnMeta, references, ...fieldVo } = field; if (references?.length) { @@ -1191,7 +1201,10 @@ export class FieldOpenApiService { return { field: createFieldInstanceByVo(fieldVo), - columnMeta: columnMeta as unknown as Record, + columnMeta: (options.restoreViewOrder ? undefined : columnMeta) as unknown as Record< + string, + IColumn + >, }; }); @@ -1200,7 +1213,8 @@ export class FieldOpenApiService { async () => { const createResult = await this.fieldCreatingService.alterCreateFieldsInExistingTable( tableId, - createPayload + createPayload, + routingOptions ); created.push(...createResult); @@ -1215,6 +1229,19 @@ export class FieldOpenApiService { await this.restoreReference(Array.from(referencesToRestore)); } + if (options.restoreViewOrder) { + for (const { tableId: tid, field } of createResult) { + const columnMeta = columnMetaByFieldId.get(field.id); + if (columnMeta) { + await this.viewService.initViewColumnMeta( + tid, + [field.id], + [columnMeta as unknown as Record] + ); + } + } + } + const skipComputation = this.cls.get('skipFieldComputation'); if (!skipComputation) { @@ -1253,15 +1280,24 @@ export class FieldOpenApiService { // Recreate search indexes after schema changes (outside tx boundaries) for (const { tableId: tid, field } of createdFields) { - await this.tableIndexService.createSearchFieldSingleIndex(tid, field); + await this.tableIndexService.createSearchFieldSingleIndex(tid, field, routingOptions); } } @Timing() - async createFieldsByRo(tableId: string, fieldRos: IFieldRo[]): Promise { + async createFieldsByRo( + tableId: string, + fieldRos: IFieldRo[], + routingOptions?: IDataDbRoutingOptions + ): Promise { if (!fieldRos.length) return []; - const fieldVos = await this.fieldSupplementService.prepareCreateFields(tableId, fieldRos); - await this.createFields(tableId, fieldVos); + const fieldVos = await this.fieldSupplementService.prepareCreateFields( + tableId, + fieldRos, + undefined, + routingOptions + ); + await this.createFields(tableId, fieldVos, routingOptions); return fieldVos; } @@ -1327,8 +1363,18 @@ export class FieldOpenApiService { } @Timing() - async createField(tableId: string, fieldRo: IFieldRo, windowId?: string) { - const fieldVo = await this.fieldSupplementService.prepareCreateField(tableId, fieldRo); + async createField( + tableId: string, + fieldRo: IFieldRo, + windowId?: string, + routingOptions?: IDataDbRoutingOptions + ) { + const fieldVo = await this.fieldSupplementService.prepareCreateField( + tableId, + fieldRo, + undefined, + routingOptions + ); const fieldInstance = createFieldInstanceByVo(fieldVo); const columnMeta = fieldRo.order && { [fieldRo.order.viewId]: { order: fieldRo.order.orderIndex }, @@ -1344,7 +1390,8 @@ export class FieldOpenApiService { created = await this.fieldCreatingService.alterCreateField( tableId, fieldInstance, - columnMeta + columnMeta, + routingOptions ); for (const { tableId: tid, field } of created) { let entry = sourceEntries.find((s) => s.tableId === tid); @@ -1367,7 +1414,7 @@ export class FieldOpenApiService { ); for (const { tableId: tid, field } of newFields) { - await this.tableIndexService.createSearchFieldSingleIndex(tid, field); + await this.tableIndexService.createSearchFieldSingleIndex(tid, field, routingOptions); } const referenceMap = await this.getFieldReferenceMap([fieldVo.id]); @@ -1578,6 +1625,7 @@ export class FieldOpenApiService { modifiedOps, supplementChange, dependentFieldIds, + skipLinkDestructive, }: { tableId: string; newField: IFieldInstance; @@ -1589,6 +1637,13 @@ export class FieldOpenApiService { oldField: IFieldInstance; }; dependentFieldIds?: string[]; + /** + * Internal flag set by `convertCrossSpaceLinkToText`: when true, the link + * cleanup path skips dropping the junction/FK and skips the symmetric + * partner cascade, so both sides of a two-way link can be converted + * independently and each preserves its own values. + */ + skipLinkDestructive?: boolean; }): Promise<{ compatibilityIssue: boolean }> { let encounteredCompatibilityIssue = false; @@ -1662,7 +1717,12 @@ export class FieldOpenApiService { const { newField: sNew, oldField: sOld } = supplementChange; await this.syncConditionalFiltersByFieldChanges(sNew, sOld); } - await this.fieldConvertingService.deleteOrCreateSupplementLink(tableId, newField, oldField); + await this.fieldConvertingService.deleteOrCreateSupplementLink( + tableId, + newField, + oldField, + skipLinkDestructive + ); await this.fieldConvertingService.stageAlter(tableId, newField, oldField); if (supplementChange) { const { tableId: sTid, newField: sNew, oldField: sOld } = supplementChange; @@ -1876,6 +1936,82 @@ export class FieldOpenApiService { return omit(newFieldVo, ['meta']) as IFieldVo; } + /** + * Cross-space variant of convertField for Link fields only. Performs the + * same Link → SingleLineText conversion that convertField would, but skips + * two destructive steps inside `linkToOther`: + * + * - `cleanForeignKey` (which would drop the junction/FK and break the + * symmetric partner's read path) + * - the symmetric-partner cascade delete (which would remove the partner + * field outright in the other base) + * + * The result is that each Link field can be converted independently while + * preserving its own current display values. The orchestrator (moveBase) + * is expected to walk affected fields in deepest-first order so dependent + * lookup/rollup fields are already converted via the regular `convertField` + * path before any Link gets here — that's why `dependentFieldIds` is left + * empty. + * + * Returns the original link options so the orchestrator can call + * `cleanOrphanCrossSpaceLinkStorage` after every paired field has been + * converted, dropping the now-orphaned junction/FK in a single sweep. + * + * Not exposed via REST; called internally from base.service.ts. + */ + async convertCrossSpaceLinkToText( + tableId: string, + fieldId: string + ): Promise<{ field: IFieldVo; oldLinkOptions: ILinkFieldOptions }> { + return this.prismaService.$tx( + async () => { + // Pass `options: {}` explicitly so stageAnalysis assigns the empty + // default to the new SingleLineText field. Without this the new field + // ends up with `options: null`, which crashes any UI/grid code that + // does `const { showAs } = field.options`. + const analysisResult = await this.fieldConvertingService.stageAnalysis(tableId, fieldId, { + type: FieldType.SingleLineText, + options: {}, + }); + const { newField, oldField } = analysisResult; + const oldLinkOptions = oldField.options as ILinkFieldOptions; + + await this.performConvertField({ + tableId, + newField, + oldField, + modifiedOps: analysisResult.modifiedOps, + supplementChange: analysisResult.supplementChange, + dependentFieldIds: [], + skipLinkDestructive: true, + }); + + const newFieldVo = instanceToPlain(newField, { excludePrefixes: ['_'] }) as IFieldVo; + return { + field: omit(newFieldVo, ['meta']) as IFieldVo, + oldLinkOptions, + }; + }, + { timeout: this.thresholdConfig.bigTransactionTimeout } + ); + } + + /** + * Drops the junction table (M:N) or FK column (N:1 / 1:1 / one-way variants) + * that backed a previously-converted cross-space Link. Intended to run after + * every paired Link in a move has been converted via + * `convertCrossSpaceLinkToText`, since at that point no field references the + * junction/FK any more. + * + * Idempotent in the typical case where both sides of a symmetric pair pass + * options pointing at the same storage — callers should still dedupe by + * storage key to avoid second-call errors. Errors are propagated so the + * caller can decide whether to log-and-continue or fail. + */ + async cleanOrphanCrossSpaceLinkStorage(options: ILinkFieldOptions): Promise { + await this.fieldSupplementService.cleanForeignKey(options); + } + async getFilterLinkRecords(tableId: string, fieldId: string) { const field = await this.fieldService.getField(tableId, fieldId); @@ -1902,7 +2038,6 @@ export class FieldOpenApiService { return []; } - // eslint-disable-next-line sonarjs/cognitive-complexity async duplicateField( sourceTableId: string, fieldId: string, @@ -1911,125 +2046,36 @@ export class FieldOpenApiService { ) { const { name, viewId } = duplicateFieldRo; const { newField } = await this.prismaService.$tx( - async () => { - const prisma = this.prismaService.txClient(); - - // throw error if field not found + async (prisma) => { const fieldRaw = await prisma.field.findUniqueOrThrow({ - where: { - id: fieldId, - deletedTime: null, - }, + where: { id: fieldId, deletedTime: null }, }); - const fieldName = await this.fieldSupplementService.uniqFieldName(sourceTableId, name); - - const dbFieldName = await this.fieldService.generateDbFieldName(sourceTableId, fieldName); - const fieldInstance = createFieldInstanceByRaw(fieldRaw); - const newFieldInstance = { - ...fieldInstance, - name: fieldName, - dbFieldName, - id: generateFieldId(), - } as IFieldInstance; - - delete newFieldInstance.isPrimary; - if (newFieldInstance.type === FieldType.Formula) { - newFieldInstance.meta = undefined; - } - - if (viewId) { - const view = await prisma.view.findUniqueOrThrow({ - where: { id: viewId, deletedTime: null }, - select: { - id: true, - columnMeta: true, - }, - }); - const columnMeta = (view.columnMeta ? JSON.parse(view.columnMeta) : {}) as IColumnMeta; - const fieldViewOrder = columnMeta[fieldId]?.order; - - const getterFieldViewOrders = Object.values(columnMeta) - .filter(({ order }) => order > fieldViewOrder) - .map(({ order }) => order) - .sort(); - - const targetFieldViewOrder = getterFieldViewOrders?.length - ? (getterFieldViewOrders[0] + fieldViewOrder) / 2 - : fieldViewOrder + 1; - - (newFieldInstance as IFieldRo).order = { - viewId, - orderIndex: targetFieldViewOrder, - }; - } - - // create field may not support notNull and unique validate - delete newFieldInstance.notNull; - delete newFieldInstance.unique; - - if (fieldInstance.type === FieldType.Button) { - newFieldInstance.options = omit(fieldInstance.options, ['workflow']); - } - - if (FieldType.Link === fieldInstance.type && !fieldInstance.isLookup) { - newFieldInstance.options = { - ...pick(fieldInstance.options, [ - 'filter', - 'filterByViewId', - 'foreignTableId', - 'relationship', - 'visibleFieldIds', - 'baseId', - ]), - // all link field should be one way link - isOneWay: true, - } as ILinkFieldOptions; - } - - if ( - fieldInstance.isLookup || - fieldInstance.type === FieldType.Rollup || - fieldInstance.type === FieldType.ConditionalRollup - ) { - const sourceLookupOptions = fieldInstance.lookupOptions; - if (sourceLookupOptions) { - const normalizedLookupOptions = pick(sourceLookupOptions, [ - 'foreignTableId', - 'lookupFieldId', - 'linkFieldId', - 'filter', - 'sort', - 'limit', - ]); - if (Object.keys(normalizedLookupOptions).length > 0) { - newFieldInstance.lookupOptions = - normalizedLookupOptions as IFieldInstance['lookupOptions']; - } else { - delete newFieldInstance.lookupOptions; - } - } else { - delete newFieldInstance.lookupOptions; - } - } + const newFieldInstance = await this.buildDuplicateFieldInstance({ + sourceTableId, + source: fieldInstance, + name, + viewId, + }); - // after create field, and add constraint relative + // remove all the constraints on the duplicated field + // re-apply constraints only when necessary in duplicateFieldData, which is aware of type changes and cross-space downgrades const newField = await this.createField(sourceTableId, { - ...omit(newFieldInstance, ['notNull', 'unique']), + ...omit(newFieldInstance, ['isPrimary', 'notNull', 'unique']), }); - if (!fieldInstance.isComputed && fieldInstance.type !== FieldType.Button) { - // Duplicate records synchronously to avoid cross-transaction CLS leaks - await this.duplicateFieldData( - sourceTableId, - newField.id, - fieldRaw.dbFieldName, - omit(newFieldInstance, 'order') as IFieldInstance, - { sourceFieldId: fieldRaw.id } - ); - } + // duplicateFieldData decides internally whether to skip (computed/Button + // target), whether to stringify cellValues (type changed, e.g. cross-space + // downgrade), and whether to re-apply notNull/unique constraints. + await this.duplicateFieldData( + sourceTableId, + newField.id, + fieldRaw.dbFieldName, + fieldInstance, + omit(newFieldInstance, 'order') as IFieldInstance + ); return { newField }; }, @@ -2047,70 +2093,211 @@ export class FieldOpenApiService { return newField; } + // Builds the IFieldInstance to be passed to createField when duplicating: + // - resolves a unique field name and dbFieldName within the target table + // - spreads source + overrides (name/dbFieldName/fresh id) + // - clears formula `meta` so the column-creation visitor decides + // persistedAsGeneratedColumn fresh for the new field + // - resolves orderIndex within the target view when viewId is provided + // - delegates type-specific options/lookupOptions shaping (incl. cross-space + // downgrade) to shapeDuplicateFieldOptions + // Constraints (isPrimary/notNull/unique) are stripped at the createField call + // site and re-applied later by duplicateFieldData when appropriate. + private async buildDuplicateFieldInstance(params: { + sourceTableId: string; + source: IFieldInstance; + name: string; + viewId?: string; + }): Promise { + const { sourceTableId, source, name, viewId } = params; + + const fieldName = await this.fieldSupplementService.uniqFieldName(sourceTableId, name); + const dbFieldName = await this.fieldService.generateDbFieldName(sourceTableId, fieldName); + + const base = { + ...source, + name: fieldName, + dbFieldName, + id: generateFieldId(), + } as IFieldInstance; + + if (base.type === FieldType.Formula) { + base.meta = undefined; + } + + if (viewId) { + (base as IFieldRo).order = await this.resolveDuplicateFieldOrder(source.id, viewId); + } + + return this.shapeDuplicateFieldOptions(sourceTableId, source, base); + } + + // Computes the orderIndex for a duplicated field within the target view: + // inserted immediately after the source field, midway to the next column. + // If the source has no entry in this view's columnMeta (sparse columnMeta on + // legacy views, etc.), falls back to placing at the rightmost end so the + // duplicate never lands ahead of unrelated fields (NaN → null after JSON + // serialization would otherwise sort to position 0). + private async resolveDuplicateFieldOrder( + sourceFieldId: string, + viewId: string + ): Promise<{ viewId: string; orderIndex: number }> { + const prisma = this.prismaService.txClient(); + const view = await prisma.view.findUniqueOrThrow({ + where: { id: viewId, deletedTime: null }, + select: { id: true, columnMeta: true }, + }); + const columnMeta = (view.columnMeta ? JSON.parse(view.columnMeta) : {}) as IColumnMeta; + const allOrders = Object.values(columnMeta) + .map((c) => c.order) + .filter((o): o is number => typeof o === 'number' && Number.isFinite(o)); + const sourceOrder = columnMeta[sourceFieldId]?.order; + + if (typeof sourceOrder !== 'number' || !Number.isFinite(sourceOrder)) { + const maxOrder = allOrders.length ? Math.max(...allOrders) : 0; + return { viewId, orderIndex: maxOrder + 1 }; + } + + const subsequentOrders = allOrders.filter((o) => o > sourceOrder).sort((a, b) => a - b); + const orderIndex = subsequentOrders.length + ? (subsequentOrders[0] + sourceOrder) / 2 + : sourceOrder + 1; + + return { viewId, orderIndex }; + } + + // Decides how to carry over options from the source field to the duplicate: + // - cross-space link/lookup/rollup → downgrade to plain SingleLineText + // - Button → strip workflow + // - Link (non-lookup) → keep relation, force one-way + // - Lookup / Rollup / ConditionalRollup → keep lookupOptions subset + // - other types → keep spread-copied options as-is + private async shapeDuplicateFieldOptions( + sourceTableId: string, + source: IFieldInstance, + target: IFieldInstance + ): Promise { + const crossSpaceDowngraded = await this.fieldSupplementService.isCrossSpaceField( + sourceTableId, + source + ); + + const isLookupOrRollup = + source.isLookup || + source.type === FieldType.Rollup || + source.type === FieldType.ConditionalRollup; + + switch (true) { + case crossSpaceDowngraded: { + const order = (target as IFieldRo).order; + return { + id: target.id, + name: target.name, + dbFieldName: target.dbFieldName, + description: target.description, + type: FieldType.SingleLineText, + ...(order ? { order } : {}), + } as unknown as IFieldInstance; + } + case source.type === FieldType.Button: + target.options = omit(source.options, ['workflow']); + return target; + case source.type === FieldType.Link && !source.isLookup: + target.options = { + ...pick(source.options, [ + 'filter', + 'filterByViewId', + 'foreignTableId', + 'relationship', + 'visibleFieldIds', + 'baseId', + ]), + // all link field should be one way link + isOneWay: true, + } as ILinkFieldOptions; + return target; + case isLookupOrRollup: { + const picked = pick(source.lookupOptions ?? {}, [ + 'foreignTableId', + 'lookupFieldId', + 'linkFieldId', + 'filter', + 'sort', + 'limit', + ]); + if (Object.keys(picked).length > 0) { + target.lookupOptions = picked as IFieldInstance['lookupOptions']; + } else { + delete target.lookupOptions; + } + return target; + } + default: + return target; + } + } + + // Copies values from `source` field column into the freshly-created `target` + // field. The function works out three things from the two field instances: + // - skip when target can't be written (computed / Button) + // - stringify via source.cellValue2String when the type changed + // (e.g. cross-space link/lookup/rollup → single line text) + // - re-apply notNull / unique on target via convertField after the copy async duplicateFieldData( sourceTableId: string, targetFieldId: string, sourceDbFieldName: string, - fieldInstance: IFieldInstance, - opts: { sourceFieldId: string } + source: IFieldInstance, + target: IFieldInstance ) { - const chunkSize = 1000; + if (target.isComputed || target.type === FieldType.Button) return; - const dbTableName = await this.fieldService.getDbTableName(sourceTableId); + const typeChanged = + source.type !== target.type || Boolean(source.isLookup) !== Boolean(target.isLookup); + const transformValue = typeChanged + ? (value: unknown) => (value == null ? null : source.cellValue2String(value)) + : undefined; - // Use the SOURCE field for filtering/counting so we only fetch rows where - // the original field has a value. The new field is empty at this point. - const sourceFieldId = opts.sourceFieldId; - const sourceFieldForFilter = { ...fieldInstance, id: sourceFieldId } as IFieldInstance; + const chunkSize = 1000; + const dbTableName = await this.fieldService.getDbTableName(sourceTableId); + const count = await this.getFieldRecordsCount(dbTableName, sourceTableId, source); - const count = await this.getFieldRecordsCount(dbTableName, sourceTableId, sourceFieldForFilter); + const reapplyConstraints = async () => { + if (target.notNull || target.unique) { + await this.convertField(sourceTableId, targetFieldId, target); + } + }; if (!count) { - if (fieldInstance.notNull || fieldInstance.unique) { - await this.convertField(sourceTableId, targetFieldId, { - ...fieldInstance, - notNull: fieldInstance.notNull, - unique: fieldInstance.unique, - }); - } + await reapplyConstraints(); return; } - const page = Math.ceil(count / chunkSize); - - for (let i = 0; i < page; i++) { + const pages = Math.ceil(count / chunkSize); + for (let i = 0; i < pages; i++) { const sourceRecords = await this.getFieldRecords( dbTableName, sourceTableId, - sourceFieldForFilter, + source, sourceDbFieldName, i, chunkSize ); - - if (!fieldInstance.isComputed && fieldInstance.type !== FieldType.Button) { - await this.prismaService.$tx(async () => { - await this.recordOpenApiService.simpleUpdateRecords(sourceTableId, { - fieldKeyType: FieldKeyType.Id, - typecast: true, - records: sourceRecords.map((record) => ({ - id: record.id, - fields: { - [targetFieldId]: record.value, - }, - })), - }); + await this.prismaService.$tx(async () => { + await this.recordOpenApiService.simpleUpdateRecords(sourceTableId, { + fieldKeyType: FieldKeyType.Id, + typecast: !transformValue, + records: sourceRecords.map((record) => ({ + id: record.id, + fields: { + [targetFieldId]: transformValue ? transformValue(record.value) : record.value, + }, + })), }); - } - } - - if (fieldInstance.notNull || fieldInstance.unique) { - await this.convertField(sourceTableId, targetFieldId, { - ...fieldInstance, - notNull: fieldInstance.notNull, - unique: fieldInstance.unique, }); } + + await reapplyConstraints(); } private async getFieldRecordsCount(dbTableName: string, tableId: string, field: IFieldInstance) { @@ -2147,9 +2334,11 @@ export class FieldOpenApiService { }); const query = qb.toQuery(); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ count: number }[]>(query); + const result = await this.databaseRouter.queryDataPrismaForTable<{ count: number }[]>( + tableId, + query, + { useTransaction: true } + ); return Number(result[0].count); } @@ -2189,9 +2378,9 @@ export class FieldOpenApiService { .limit(chunkSize) .offset(page * chunkSize) .toQuery(); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ __id: string; [key: string]: string }[]>(query); + const result = await this.databaseRouter.queryDataPrismaForTable< + { __id: string; [key: string]: string }[] + >(tableId, query, { useTransaction: true }); this.logger.debug('getFieldRecords: ', result); return result.map((item) => ({ id: item.__id, diff --git a/apps/nestjs-backend/src/features/graph/graph.service.ts b/apps/nestjs-backend/src/features/graph/graph.service.ts index 3ff3e549be..63e0bf5844 100644 --- a/apps/nestjs-backend/src/features/graph/graph.service.ts +++ b/apps/nestjs-backend/src/features/graph/graph.service.ts @@ -3,7 +3,6 @@ import type { IFieldRo, ILinkFieldOptions, IConvertFieldRo } from '@teable/core' import { FieldType, Relationship, isLinkLookupOptions } from '@teable/core'; import type { Field, TableMeta } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { IGraphEdge, IGraphNode, @@ -18,6 +17,7 @@ import { Knex } from 'knex'; import { groupBy, keyBy, uniq } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import { majorFieldKeysChanged } from '../../utils/major-field-keys-changed'; import { Timing } from '../../utils/timing'; @@ -61,7 +61,7 @@ export class GraphService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly fieldService: FieldService, private readonly referenceService: ReferenceService, private readonly fieldSupplementService: FieldSupplementService, @@ -197,7 +197,8 @@ export class GraphService { field.id, [field.id], { [field.id]: field }, - { [field.id]: tableMap[tableId].dbTableName } + { [field.id]: tableMap[tableId].dbTableName }, + { [field.id]: tableId } ); const estimateTime = field.isComputed ? this.getEstimateTime(updateCellCount) : 200; return { @@ -423,7 +424,8 @@ export class GraphService { fieldId, topoFieldIds, fieldMap, - fieldId2DbTableName + fieldId2DbTableName, + fieldId2TableId ); const resetLinkFieldLookupFieldIds = @@ -462,7 +464,8 @@ export class GraphService { hostFieldId: string, fieldIds: string[], fieldMap: IFieldMap, - fieldId2DbTableName: Record + fieldId2DbTableName: Record, + fieldId2TableId: Record ): Promise { const queries = fieldIds .map((fieldId) => @@ -473,8 +476,17 @@ export class GraphService { let total = 0; for (const { fieldId, fieldName, query } of queries) { try { - const [{ count }] = - await this.dataPrismaService.$queryRawUnsafe<{ count: bigint }[]>(query); + const tableId = fieldId2TableId[fieldId]; + if (!tableId) { + this.logger.warn( + `Skip affected cell count for field=${fieldId} name="${fieldName}" because table id is missing` + ); + continue; + } + const [{ count }] = await this.databaseRouter.queryDataPrismaForTable<{ count: bigint }[]>( + tableId, + query + ); total += Number(count); } catch (error) { if (this.shouldSkipAffectedCountError(error)) { @@ -615,7 +627,8 @@ export class GraphService { fieldId, allFieldIds, fieldMap, - fieldId2DbTableName + fieldId2DbTableName, + fieldId2TableId ); return { diff --git a/apps/nestjs-backend/src/features/health/health.controller.test.ts b/apps/nestjs-backend/src/features/health/health.controller.test.ts index 0025b31b50..9d3d70e386 100644 --- a/apps/nestjs-backend/src/features/health/health.controller.test.ts +++ b/apps/nestjs-backend/src/features/health/health.controller.test.ts @@ -1,7 +1,6 @@ import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; import { HealthCheckService, PrismaHealthIndicator } from '@nestjs/terminus'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { HealthController } from './health.controller'; @@ -15,7 +14,6 @@ describe('HealthController', () => { pingCheck: vi.fn(), }; const metaPrisma = {}; - const dataPrisma = {}; beforeEach(async () => { health.check.mockReset(); @@ -29,7 +27,6 @@ describe('HealthController', () => { { provide: HealthCheckService, useValue: health }, { provide: PrismaHealthIndicator, useValue: db }, { provide: PrismaService, useValue: metaPrisma }, - { provide: DataPrismaService, useValue: dataPrisma }, ], }).compile(); @@ -40,18 +37,16 @@ describe('HealthController', () => { expect(controller).toBeDefined(); }); - it('checks both meta and data databases', async () => { + it('checks the meta database without assuming a global data database', async () => { await controller.check(); expect(health.check).toHaveBeenCalledTimes(1); const indicators = health.check.mock.calls[0][0] as Array<() => Promise>; - expect(indicators).toHaveLength(2); + expect(indicators).toHaveLength(1); await indicators[0](); - await indicators[1](); expect(db.pingCheck).toHaveBeenNthCalledWith(1, 'metaDatabase', metaPrisma); - expect(db.pingCheck).toHaveBeenNthCalledWith(2, 'dataDatabase', dataPrisma); }); }); diff --git a/apps/nestjs-backend/src/features/health/health.controller.ts b/apps/nestjs-backend/src/features/health/health.controller.ts index 3063c86d33..5b23119ecc 100644 --- a/apps/nestjs-backend/src/features/health/health.controller.ts +++ b/apps/nestjs-backend/src/features/health/health.controller.ts @@ -1,6 +1,5 @@ import { Controller, Get, Logger } from '@nestjs/common'; import { HealthCheck, HealthCheckService, PrismaHealthIndicator } from '@nestjs/terminus'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { Public } from '../auth/decorators/public.decorator'; @@ -11,18 +10,14 @@ export class HealthController { constructor( private readonly health: HealthCheckService, private readonly db: PrismaHealthIndicator, - private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService + private readonly prismaService: PrismaService ) {} @Get() @HealthCheck() check() { try { - return this.health.check([ - () => this.db.pingCheck('metaDatabase', this.prismaService), - () => this.db.pingCheck('dataDatabase', this.dataPrismaService), - ]); + return this.health.check([() => this.db.pingCheck('metaDatabase', this.prismaService)]); } catch (error) { this.logger.error(error); throw error; diff --git a/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts b/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts index c48712d70b..1b8bf1abf6 100644 --- a/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts +++ b/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts @@ -120,10 +120,10 @@ export class ImportOpenApiV2Service { maxRowCount?: number, projection?: string[] ): Promise<{ totalImported: number }> { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const { attachmentUrl, fileType, insertConfig } = importOptions; const { sourceColumnMap, sourceWorkSheetKey, excludeFirstRow } = insertConfig; diff --git a/apps/nestjs-backend/src/features/integrity/foreign-key.service.ts b/apps/nestjs-backend/src/features/integrity/foreign-key.service.ts index 8e4064a618..77687fdee5 100644 --- a/apps/nestjs-backend/src/features/integrity/foreign-key.service.ts +++ b/apps/nestjs-backend/src/features/integrity/foreign-key.service.ts @@ -1,10 +1,10 @@ import { Injectable, Logger } from '@nestjs/common'; import { FieldType, type ILinkFieldOptions } from '@teable/core'; import { Prisma, PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { IntegrityIssueType, type IIntegrityIssue } from '@teable/openapi'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; @@ -14,7 +14,7 @@ export class ForeignKeyIntegrityService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @InjectModel(DATA_KNEX) private readonly knex: Knex ) {} @@ -43,6 +43,7 @@ export class ForeignKeyIntegrityService { field, referencedTableName: selfTableName, isSelfReference: true, + routingTableId: tableId, }); issues.push(...selfIssues); } @@ -56,6 +57,7 @@ export class ForeignKeyIntegrityService { field, referencedTableName: foreignTableName, isSelfReference: false, + routingTableId: tableId, }); issues.push(...foreignIssues); } @@ -70,6 +72,7 @@ export class ForeignKeyIntegrityService { field, referencedTableName, isSelfReference, + routingTableId, }: { fkHostTableName: string; targetTableName: string; @@ -77,6 +80,7 @@ export class ForeignKeyIntegrityService { field: { id: string; name: string }; referencedTableName: string; isSelfReference: boolean; + routingTableId: string; }): Promise { const issues: IIntegrityIssue[] = []; @@ -88,9 +92,11 @@ export class ForeignKeyIntegrityService { .toQuery(); try { - const invalidRefs = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ count: bigint }[]>(invalidQuery); + const invalidRefs = await this.databaseRouter.queryDataPrismaForTable<{ count: bigint }[]>( + routingTableId, + invalidQuery, + { useTransaction: true } + ); const refCount = Number(invalidRefs[0]?.count || 0); if (refCount > 0) { @@ -140,6 +146,7 @@ export class ForeignKeyIntegrityService { fkHostTableName, targetTableName: table.dbTableName, keyName: selfKeyName, + routingTableId: tableId, }); totalFixed += selfDeleted; } @@ -150,6 +157,7 @@ export class ForeignKeyIntegrityService { fkHostTableName, targetTableName: foreignTable.dbTableName, keyName: foreignKeyName, + routingTableId: tableId, }); totalFixed += foreignDeleted; } @@ -167,10 +175,12 @@ export class ForeignKeyIntegrityService { fkHostTableName, targetTableName, keyName, + routingTableId, }: { fkHostTableName: string; targetTableName: string; keyName: string; + routingTableId: string; }) { if (!fkHostTableName.split('.')[1].startsWith('junction_')) { throw new Error(`fkHostTableName: ${fkHostTableName} is not a junction table`); @@ -185,6 +195,8 @@ export class ForeignKeyIntegrityService { ) .delete() .toQuery(); - return await this.dataPrismaService.txClient().$executeRawUnsafe(deleteQuery); + return await this.databaseRouter.executeDataPrismaForTable(routingTableId, deleteQuery, { + useTransaction: true, + }); } } diff --git a/apps/nestjs-backend/src/features/integrity/integrity-v2.service.spec.ts b/apps/nestjs-backend/src/features/integrity/integrity-v2.service.spec.ts index abb97ed91e..97be4a32cd 100644 --- a/apps/nestjs-backend/src/features/integrity/integrity-v2.service.spec.ts +++ b/apps/nestjs-backend/src/features/integrity/integrity-v2.service.spec.ts @@ -378,6 +378,7 @@ describe('IntegrityV2Service repair telemetry', () => { ); expect(tableRepository.find).toHaveBeenCalledTimes(1); + expect(tableRepository.find.mock.calls[0]?.[2]).toEqual({ state: 'all' }); expect(tableRepository.find.mock.calls[0]?.[1]).toMatchObject({ left: { baseIdValue: foreignBaseId, diff --git a/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts b/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts index dd5225bc9b..8c187477e9 100644 --- a/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts +++ b/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts @@ -226,12 +226,13 @@ export class IntegrityV2Service { throw new HttpException(parsedTableId.error.message, HttpStatus.BAD_REQUEST); } - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const tableRepository = container.resolve(v2CoreTokens.tableRepository); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const tableResult = await tableRepository.findOne( context, - TableByIdSpec.create(parsedTableId.value) + TableByIdSpec.create(parsedTableId.value), + { state: 'all' } ); if (tableResult.isErr()) { @@ -245,7 +246,8 @@ export class IntegrityV2Service { if (options?.includeBaseTables) { const tablesResult = await tableRepository.find( context, - TableByBaseIdSpec.create(table.baseId()) + TableByBaseIdSpec.create(table.baseId()), + { state: 'all' } ); if (tablesResult.isErr()) { @@ -270,10 +272,10 @@ export class IntegrityV2Service { throw new HttpException(parsedBaseId.error.message, HttpStatus.BAD_REQUEST); } - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForBase(baseId); const tableRepository = container.resolve(v2CoreTokens.tableRepository); const baseRepository = container.resolve(v2CoreTokens.baseRepository); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const baseResult = await baseRepository.findOne(context, parsedBaseId.value); if (baseResult.isErr()) { @@ -286,7 +288,8 @@ export class IntegrityV2Service { const tablesResult = await tableRepository.find( context, - TableByBaseIdSpec.create(parsedBaseId.value) + TableByBaseIdSpec.create(parsedBaseId.value), + { state: 'all' } ); if (tablesResult.isErr()) { @@ -475,7 +478,7 @@ export class IntegrityV2Service { throw new HttpException(specResult.error.message, HttpStatus.INTERNAL_SERVER_ERROR); } - const tablesResult = await tableRepository.find(context, specResult.value); + const tablesResult = await tableRepository.find(context, specResult.value, { state: 'all' }); if (tablesResult.isErr()) { throw new HttpException(tablesResult.error.message, HttpStatus.INTERNAL_SERVER_ERROR); } diff --git a/apps/nestjs-backend/src/features/integrity/link-field.service.ts b/apps/nestjs-backend/src/features/integrity/link-field.service.ts index 9fab615a99..b7875178a6 100644 --- a/apps/nestjs-backend/src/features/integrity/link-field.service.ts +++ b/apps/nestjs-backend/src/features/integrity/link-field.service.ts @@ -1,10 +1,10 @@ import { Injectable, Logger } from '@nestjs/common'; import { FieldType, type ILinkFieldOptions } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { Prisma, PrismaService } from '@teable/db-main-prisma'; import { IntegrityIssueType, type IIntegrityIssue } from '@teable/openapi'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { DatabaseRouter } from '../../global/database-router.service'; import { createFieldInstanceByRaw } from '../field/model/factory'; import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; @@ -14,7 +14,7 @@ export class LinkFieldIntegrityService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @InjectDbProvider() private readonly dbProvider: IDbProvider ) {} @@ -31,6 +31,7 @@ export class LinkFieldIntegrityService { foreignKeyName, linkDbFieldName: field.dbFieldName, isMultiValue: Boolean(field.isMultipleCellValue), + routingTableId: tableId, }); if (inconsistentRecords.length > 0) { @@ -53,13 +54,15 @@ export class LinkFieldIntegrityService { foreignKeyName: string; linkDbFieldName: string; isMultiValue: boolean; + routingTableId: string; }) { + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForTable(params.routingTableId); // Some symmetric link fields may not persist a JSON column (depending on // creation path). If the link JSON column does not exist, skip comparison. const linkColumnExists = await this.dbProvider.checkColumnExist( params.dbTableName, params.linkDbFieldName, - this.dataPrismaService + dataPrisma ); if (!linkColumnExists) { @@ -68,7 +71,7 @@ export class LinkFieldIntegrityService { const query = this.dbProvider.integrityQuery().checkLinks(params); try { - return await this.dataPrismaService.$queryRawUnsafe<{ id: string }[]>(query); + return await dataPrisma.$queryRawUnsafe<{ id: string }[]>(query); } catch (error) { if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2010') { this.logger.warn( @@ -90,12 +93,14 @@ export class LinkFieldIntegrityService { foreignKeyName: string; linkDbFieldName: string; isMultiValue: boolean; + routingTableId: string; }) { + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForTable(params.routingTableId); // If display column does not exist (link fields are virtual by design), skip update const linkColumnExists = await this.dbProvider.checkColumnExist( params.dbTableName, params.linkDbFieldName, - this.dataPrismaService + dataPrisma ); if (!linkColumnExists) { @@ -103,7 +108,7 @@ export class LinkFieldIntegrityService { } const query = this.dbProvider.integrityQuery().fixLinks(params); - return await this.dataPrismaService.$executeRawUnsafe(query); + return await dataPrisma.$executeRawUnsafe(query); } private async checkAndFix(params: { @@ -115,6 +120,7 @@ export class LinkFieldIntegrityService { linkDbFieldName: string; isMultiValue: boolean; selfKeyName: string; + routingTableId: string; }) { try { const inconsistentRecords = await this.checkLinks(params); @@ -174,6 +180,7 @@ export class LinkFieldIntegrityService { linkDbFieldName: linkField.dbFieldName, isMultiValue: Boolean(linkField.isMultipleCellValue), selfKeyName, + routingTableId: tableId, }); totalFixed += linksFixed; diff --git a/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts b/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts index 8e2f5246d0..858d53754c 100644 --- a/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts +++ b/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts @@ -19,12 +19,12 @@ import type { } from '@teable/core'; import type { Field } from '@teable/db-main-prisma'; import { Prisma, PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { IntegrityIssueType, type IIntegrityCheckVo, type IIntegrityIssue } from '@teable/openapi'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import { LinkFieldQueryService } from '../field/field-calculate/link-field-query.service'; import { FieldService } from '../field/field.service'; @@ -42,7 +42,7 @@ export class LinkIntegrityService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly foreignKeyIntegrityService: ForeignKeyIntegrityService, private readonly linkFieldIntegrityService: LinkFieldIntegrityService, private readonly uniqueIndexService: UniqueIndexService, @@ -348,9 +348,10 @@ export class LinkIntegrityService { let canCheckLinks = false; const tableExistsSql = this.dbProvider.checkTableExist(options.fkHostTableName); - const tableExists = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ exists: boolean }[]>(tableExistsSql); + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForTable(table.id, { + useTransaction: true, + }); + const tableExists = await dataPrisma.$queryRawUnsafe<{ exists: boolean }[]>(tableExistsSql); const hostTableExists = tableExists[0].exists; if (!hostTableExists) { @@ -363,13 +364,13 @@ export class LinkIntegrityService { const selfKeyExists = await this.dbProvider.checkColumnExist( options.fkHostTableName, options.selfKeyName, - this.dataPrismaService.txClient() + dataPrisma ); const foreignKeyExists = await this.dbProvider.checkColumnExist( options.fkHostTableName, options.foreignKeyName, - this.dataPrismaService.txClient() + dataPrisma ); if (!selfKeyExists) { @@ -434,7 +435,9 @@ export class LinkIntegrityService { async checkEmptyString(tableId: string): Promise { const prisma = this.prismaService.txClient(); - const dataPrisma = this.dataPrismaService.txClient(); + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForTable(tableId, { + useTransaction: true, + }); const fields = await prisma.field.findMany({ where: { tableId, @@ -481,7 +484,6 @@ export class LinkIntegrityService { issueType?: IntegrityIssueType ): Promise { const prisma = this.prismaService.txClient(); - const dataPrisma = this.dataPrismaService.txClient(); const fieldRaw = await prisma.field.findFirst({ where: { id: fieldId, type: FieldType.Link, isLookup: null, deletedTime: null }, }); @@ -491,6 +493,9 @@ export class LinkIntegrityService { } const linkField = createFieldInstanceByRaw(fieldRaw) as LinkFieldDto; + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForTable(fieldRaw.tableId, { + useTransaction: true, + }); const options = linkField.options; const tableMeta = await prisma.tableMeta.findFirst({ where: { id: fieldRaw.tableId, deletedTime: null }, @@ -661,6 +666,7 @@ export class LinkIntegrityService { } await this.backfillForeignKeysFromLinkColumn({ + routingTableId: fieldRaw.tableId, dbTableName: tableMeta.dbTableName, linkDbFieldName: linkField.dbFieldName, fkHostTableName: options.fkHostTableName, @@ -685,6 +691,7 @@ export class LinkIntegrityService { foreignKeyName: string; relationship: Relationship; isOneWay?: boolean; + routingTableId: string; }) { const { dbTableName, @@ -694,8 +701,11 @@ export class LinkIntegrityService { foreignKeyName, relationship, isOneWay, + routingTableId, } = params; - const dataPrisma = this.dataPrismaService.txClient(); + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForTable(routingTableId, { + useTransaction: true, + }); const linkColumnExists = await this.dbProvider.checkColumnExist( dbTableName, @@ -1224,10 +1234,12 @@ export class LinkIntegrityService { async fixEmptyString(fieldId: string, tableId?: string): Promise { const prisma = this.prismaService.txClient(); - const dataPrisma = this.dataPrismaService.txClient(); if (!tableId) { return; } + const dataPrisma = await this.databaseRouter.dataPrismaExecutorForTable(tableId, { + useTransaction: true, + }); const { dbTableName } = await prisma.tableMeta.findFirstOrThrow({ where: { id: tableId, deletedTime: null }, diff --git a/apps/nestjs-backend/src/features/integrity/unique-index.service.ts b/apps/nestjs-backend/src/features/integrity/unique-index.service.ts index ebbcc5366c..8a8f3df2b5 100644 --- a/apps/nestjs-backend/src/features/integrity/unique-index.service.ts +++ b/apps/nestjs-backend/src/features/integrity/unique-index.service.ts @@ -1,10 +1,10 @@ import { Injectable } from '@nestjs/common'; import { IdPrefix } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { IntegrityIssueType, type IIntegrityIssue } from '@teable/openapi'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import { FieldService } from '../field/field.service'; @@ -12,7 +12,7 @@ import { FieldService } from '../field/field.service'; export class UniqueIndexService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @InjectModel(DATA_KNEX) private readonly knex: Knex, private readonly fieldService: FieldService ) {} @@ -26,7 +26,8 @@ export class UniqueIndexService { const colId = '__id'; const idUniqueIndexExists = - (await this.fieldService.findUniqueIndexesForField(table.dbTableName, colId)).length > 0; + (await this.fieldService.findUniqueIndexesForField(table.id, table.dbTableName, colId)) + .length > 0; if (!idUniqueIndexExists) { issues.push({ @@ -43,6 +44,7 @@ export class UniqueIndexService { for (const field of uniqueFields) { const indexNames = await this.fieldService.findUniqueIndexesForField( + table.id, table.dbTableName, field.dbFieldName ); @@ -98,7 +100,7 @@ export class UniqueIndexService { if (!sql) { return; } - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql, { useTransaction: true }); return { type: IntegrityIssueType.UniqueIndexNotFound, diff --git a/apps/nestjs-backend/src/features/notification/notification.service.ts b/apps/nestjs-backend/src/features/notification/notification.service.ts index 11f30c8314..715beec867 100644 --- a/apps/nestjs-backend/src/features/notification/notification.service.ts +++ b/apps/nestjs-backend/src/features/notification/notification.service.ts @@ -36,6 +36,11 @@ type INotifyEmailConfig = { buttonText?: string | ILocalization; }; +function toArray(value?: T | T[]): T[] { + if (!value) return []; + return Array.isArray(value) ? value : [value]; +} + const notificationListLimit = 10; const notificationListSelect = { @@ -63,6 +68,7 @@ export class NotificationService { [NotificationTypeEnum.CollaboratorMultiRowTag]: MailType.CollaboratorMultiRowTag, [NotificationTypeEnum.Comment]: MailType.Common, [NotificationTypeEnum.ExportBase]: MailType.ExportBase, + [NotificationTypeEnum.AdminNotice]: MailType.System, }; constructor( private readonly prismaService: PrismaService, @@ -304,91 +310,118 @@ export class NotificationService { async sendCommonNotify( params: { - path: string; + path?: string; fromUserId?: string; - toUserId: string; + toUserId?: string | string[]; + toEmail?: string | string[]; message: string | ILocalization; severity?: NotificationSeverityEnum; emailConfig?: INotifyEmailConfig; }, type = NotificationTypeEnum.System - ) { - const { toUserId, emailConfig, path, fromUserId = SYSTEM_USER_ID } = params; - const notifyId = generateNotificationId(); - const toUser = await this.userService.getUserById(toUserId); - if (!toUser) { - return; + ): Promise<{ + sentCount: number; + invalidUserIds?: string[]; + invalidEmails?: string[]; + }> { + const { emailConfig, path = '', fromUserId = SYSTEM_USER_ID } = params; + const ids = toArray(params.toUserId); + const emails = toArray(params.toEmail); + + const toUsers = await this.userService.getUsersByIdsOrEmails({ ids, emails }); + + const invalidUserIds = ids.length + ? ids.filter((id) => !toUsers.some((u) => u.id === id)) + : undefined; + const invalidEmails = emails.length + ? emails.filter((e) => !toUsers.some((u) => u.email.toLowerCase() === e.toLowerCase())) + : undefined; + + if (toUsers.length === 0) { + return { sentCount: 0, invalidUserIds, invalidEmails }; } const severity = params.severity ?? this.getNotificationSeverity(type); const messageI18n = this.getMessageI18n(params.message); - const data: Prisma.NotificationCreateInput = { - id: notifyId, - fromUserId: fromUserId, - toUserId, - type, - urlPath: path, - createdBy: fromUserId, - message: this.getMessage(params.message, 'en'), - messageI18n, - severity, - }; - const notifyData = await this.createNotify(data); - - const unreadCount = (await this.unreadCount(toUser.id)).unreadCount; + const messageEn = this.getMessage(params.message, 'en'); const rawUsers = await this.prismaService.user.findMany({ select: { id: true, name: true, avatar: true }, where: { id: fromUserId }, }); const fromUserSets = keyBy(rawUsers, 'id'); + const notifyIcon = this.generateNotifyIcon(type, fromUserId, fromUserSets); - const systemNotifyIcon = this.generateNotifyIcon( - notifyData.type as NotificationTypeEnum, + const createdTime = new Date(); + const notifyRecords = toUsers.map((toUser) => ({ + id: generateNotificationId(), fromUserId, - fromUserSets - ); - - const socketNotification = { - notification: { - id: notifyData.id, - message: notifyData.message, - messageI18n: notifyData.messageI18n, - notifyType: type, - url: path, - notifyIcon: systemNotifyIcon, - severity, - isRead: false, - createdTime: notifyData.createdTime.toISOString(), - }, - unreadCount: unreadCount, - }; - - this.sendNotifyBySocket(toUser.id, socketNotification); + toUserId: toUser.id, + type, + urlPath: path, + createdBy: fromUserId, + message: messageEn, + messageI18n, + severity, + createdTime, + })); - if (emailConfig && toUser.notifyMeta && toUser.notifyMeta.email) { - const lang = this.getUserLang(toUser.lang); - const emailOptions = await this.mailSenderService.commonEmailOptions({ - ...emailConfig, - title: this.getMessage(emailConfig.title, lang), - message: this.getMessage(emailConfig.message, lang), - to: toUserId, - buttonUrl: emailConfig.buttonUrl || this.mailConfig.origin + path, - buttonText: emailConfig.buttonText - ? this.getMessage(emailConfig.buttonText, lang) - : this.i18n.t('common.email.templates.notify.buttonText'), - }); - this.mailSenderService.sendMail( - { - to: toUser.email, - ...emailOptions, + const toUserIdList = toUsers.map((u) => u.id); + const unreadCounts = await this.prismaService.notification.groupBy({ + by: ['toUserId'], + where: { toUserId: { in: toUserIdList }, isRead: false }, + _count: { _all: true }, + }); + const unreadCountMap = new Map(unreadCounts.map((r) => [r.toUserId, r._count._all])); + + await this.prismaService.notification.createMany({ data: notifyRecords }); + + const notifyById = keyBy(notifyRecords, 'toUserId'); + for (const toUser of toUsers) { + const record = notifyById[toUser.id]; + const unreadCount = (unreadCountMap.get(toUser.id) ?? 0) + 1; + + this.sendNotifyBySocket(toUser.id, { + notification: { + id: record.id, + message: messageEn, + messageI18n, + notifyType: type, + url: path, + notifyIcon: notifyIcon, + severity, + isRead: false, + createdTime: createdTime.toISOString(), }, - { - type: this.mailTypeMap[type], - transporterName: MailTransporterType.Notify, - } - ); + unreadCount, + }); + + if (emailConfig && toUser.notifyMeta && toUser.notifyMeta.email) { + const lang = this.getUserLang(toUser.lang); + const emailOptions = await this.mailSenderService.commonEmailOptions({ + ...emailConfig, + title: this.getMessage(emailConfig.title, lang), + message: this.getMessage(emailConfig.message, lang), + to: toUser.id, + buttonUrl: emailConfig.buttonUrl || this.mailConfig.origin + path, + buttonText: emailConfig.buttonText + ? this.getMessage(emailConfig.buttonText, lang) + : this.i18n.t('common.email.templates.notify.buttonText'), + }); + this.mailSenderService.sendMail( + { + to: toUser.email, + ...emailOptions, + }, + { + type: this.mailTypeMap[type], + transporterName: MailTransporterType.Notify, + } + ); + } } + + return { sentCount: toUsers.length, invalidUserIds, invalidEmails }; } async sendImportResultNotify(params: { @@ -574,7 +607,7 @@ export class NotificationService { id: v.id, notifyIcon: notifyIcon, notifyType: v.type as NotificationTypeEnum, - url: this.mailConfig.origin + v.urlPath, + url: v.urlPath ? this.mailConfig.origin + v.urlPath : '', message: v.message, messageI18n: v.messageI18n, severity: this.getNotificationSeverity(v.type as NotificationTypeEnum, v.severity), @@ -594,6 +627,7 @@ export class NotificationService { switch (notifyType) { case NotificationTypeEnum.System: case NotificationTypeEnum.ExportBase: + case NotificationTypeEnum.AdminNotice: return { iconUrl: `${origin}/images/favicon/favicon.svg` }; case NotificationTypeEnum.Comment: case NotificationTypeEnum.CollaboratorCellTag: @@ -628,6 +662,7 @@ export class NotificationService { case NotificationTypeEnum.CollaboratorMultiRowTag: case NotificationTypeEnum.ExportBase: case NotificationTypeEnum.System: + case NotificationTypeEnum.AdminNotice: return NotificationSeverityEnum.Info; default: throw assertNever(notifyType); @@ -655,6 +690,8 @@ export class NotificationService { const { downloadUrl } = urlMeta || {}; return downloadUrl as string; } + case NotificationTypeEnum.AdminNotice: + return ''; default: throw assertNever(notifyType); } diff --git a/apps/nestjs-backend/src/features/oauth/oauth-app-init.service.spec.ts b/apps/nestjs-backend/src/features/oauth/oauth-app-init.service.spec.ts new file mode 100644 index 0000000000..67a363604f --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth-app-init.service.spec.ts @@ -0,0 +1,63 @@ +import { cliOAuthApp } from '@teable/core'; +import type { PrismaService } from '@teable/db-main-prisma'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { DistributedLockService } from '../../distributed-lock'; +import { OAuthAppInitService } from './oauth-app-init.service'; + +describe('OAuthAppInitService', () => { + const oAuthApp = { upsert: vi.fn() }; + // Lock stub: run the guarded task immediately, as if the lock were acquired. + const distributedLock = { + runExclusive: vi.fn(async (_name: string, _ttl: number, task: () => Promise) => { + await task(); + return true; + }), + }; + + const prismaService = { oAuthApp } as unknown as PrismaService; + const lockService = distributedLock as unknown as DistributedLockService; + const newService = () => new OAuthAppInitService(prismaService, lockService); + + /** The serialized `oauth_app` payload derived from `cliOAuthApp`. */ + const data = { + name: cliOAuthApp.name, + homepage: cliOAuthApp.homepage, + description: cliOAuthApp.description, + logo: cliOAuthApp.logo, + redirectUris: JSON.stringify(cliOAuthApp.redirectUris), + scopes: JSON.stringify(cliOAuthApp.scopes), + }; + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('seeds the CLI OAuth app under a distributed lock', async () => { + oAuthApp.upsert.mockResolvedValue({}); + + await newService().onModuleInit(); + + expect(distributedLock.runExclusive).toHaveBeenCalledWith( + 'oauth-app-init', + 60, + expect.any(Function) + ); + expect(oAuthApp.upsert).toHaveBeenCalledWith({ + where: { clientId: cliOAuthApp.clientId }, + create: { clientId: cliOAuthApp.clientId, createdBy: 'system', ...data }, + update: data, + }); + }); + + it('ignores a concurrent-create unique conflict (P2002)', async () => { + oAuthApp.upsert.mockRejectedValue({ code: 'P2002' }); + + await expect(newService().onModuleInit()).resolves.toBeUndefined(); + }); + + it('rethrows unexpected errors', async () => { + oAuthApp.upsert.mockRejectedValue(new Error('database unavailable')); + + await expect(newService().onModuleInit()).rejects.toThrow('database unavailable'); + }); +}); diff --git a/apps/nestjs-backend/src/features/oauth/oauth-app-init.service.ts b/apps/nestjs-backend/src/features/oauth/oauth-app-init.service.ts new file mode 100644 index 0000000000..e5c4e61730 --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth-app-init.service.ts @@ -0,0 +1,57 @@ +import { Injectable, Logger, type OnModuleInit } from '@nestjs/common'; +import { cliOAuthApp } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { DistributedLockService } from '../../distributed-lock'; + +/** + * Seeds the first-party CLI OAuth app on startup so `@teable/cli`'s + * Authorization Code + PKCE login works against any Teable deployment without + * manually registering an OAuth app. + * + * Idempotent: upserts the `oauth_app` row to match `cliOAuthApp`. A distributed + * lock keeps only one instance seeding in a multi-pod deployment. + */ +@Injectable() +export class OAuthAppInitService implements OnModuleInit { + private readonly logger = new Logger(OAuthAppInitService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly distributedLock: DistributedLockService + ) {} + + async onModuleInit() { + // 60s lock — ample for a single upsert; auto-expires if this instance dies. + await this.distributedLock.runExclusive('oauth-app-init', 60, () => this.seedCliOAuthApp()); + } + + /** Seed/reconcile the first-party CLI OAuth app row. Idempotent. */ + private async seedCliOAuthApp() { + const { clientId, name, homepage, description, logo, redirectUris, scopes } = cliOAuthApp; + const data = { + name, + homepage, + description, + logo, + redirectUris: JSON.stringify(redirectUris), + scopes: JSON.stringify(scopes), + }; + + try { + await this.prismaService.oAuthApp.upsert({ + where: { clientId }, + // `createdBy: 'system'` — system-seeded row, no owning user. + create: { clientId, createdBy: 'system', ...data }, + update: data, + }); + this.logger.log(`Initialized CLI OAuth app: ${clientId}`); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + // Without Redis the lock is a no-op, so a concurrent upsert from another + // instance can still race on the unique clientId — ignore that conflict. + if (error.code !== 'P2002') { + throw error; + } + } + } +} diff --git a/apps/nestjs-backend/src/features/oauth/oauth.module.ts b/apps/nestjs-backend/src/features/oauth/oauth.module.ts index ac62c2396b..e11e78fcb0 100644 --- a/apps/nestjs-backend/src/features/oauth/oauth.module.ts +++ b/apps/nestjs-backend/src/features/oauth/oauth.module.ts @@ -2,7 +2,9 @@ import { Module } from '@nestjs/common'; import { JwtModule } from '@nestjs/jwt'; import { PassportModule } from '@nestjs/passport'; import { authConfig, type IAuthConfig } from '../../configs/auth.config'; +import { DistributedLockModule } from '../../distributed-lock'; import { AccessTokenModule } from '../access-token/access-token.module'; +import { OAuthAppInitService } from './oauth-app-init.service'; import { OAuthServerController } from './oauth-server.controller'; import { OAuthServerService } from './oauth-server.service'; import { OAuthTxStore } from './oauth-tx-store'; @@ -15,6 +17,7 @@ import { OAuthPkceClientStrategy } from './strategies/oauth2-pkce-client.strateg @Module({ imports: [ AccessTokenModule, + DistributedLockModule, PassportModule.register({ session: true }), JwtModule.registerAsync({ useFactory: (config: IAuthConfig) => ({ @@ -30,6 +33,7 @@ import { OAuthPkceClientStrategy } from './strategies/oauth2-pkce-client.strateg providers: [ OAuthServerService, OAuthService, + OAuthAppInitService, OAuthClientStrategy, OAuthPkceClientStrategy, OAuthTxStore, diff --git a/apps/nestjs-backend/src/features/oauth/oauth.service.ts b/apps/nestjs-backend/src/features/oauth/oauth.service.ts index b58d9feb72..030ae1f524 100644 --- a/apps/nestjs-backend/src/features/oauth/oauth.service.ts +++ b/apps/nestjs-backend/src/features/oauth/oauth.service.ts @@ -1,5 +1,5 @@ import { ForbiddenException, Injectable, NotFoundException } from '@nestjs/common'; -import { generateClientId, getRandomString, nullsToUndefined } from '@teable/core'; +import { SYSTEM_USER_ID, generateClientId, getRandomString, nullsToUndefined } from '@teable/core'; import { Prisma, PrismaService } from '@teable/db-main-prisma'; import type { AuthorizedVo, @@ -79,6 +79,7 @@ export class OAuthService { }; async getOAuth(clientId: string): Promise { + await this.validateOwnership(clientId); const res = await this.prismaService.oAuthApp.findUnique({ where: { clientId, @@ -110,6 +111,7 @@ export class OAuthService { } async updateOAuth(clientId: string, ro: OAuthCreateRo): Promise { + await this.validateOwnership(clientId); const { redirectUris, name, description, scopes, homepage, logo } = ro; const res = await this.prismaService.oAuthApp.update({ where: { @@ -141,7 +143,27 @@ export class OAuthService { ); } + private validateOwnership = async (clientId: string) => { + const app = await this.prismaService.oAuthApp.findUnique({ + where: { + clientId, + }, + select: { createdBy: true }, + }); + if (!app) { + throw new NotFoundException('OAuth client not found'); + } + const user = this.cls.get('user'); + if (user.isAdmin && SYSTEM_USER_ID === app.createdBy) { + return; + } + if (app.createdBy !== user.id) { + throw new ForbiddenException('No permission to operate on this OAuth client'); + } + }; + async deleteOAuth(clientId: string): Promise { + await this.validateOwnership(clientId); await this.prismaService.$tx(async (prisma) => { await prisma.oAuthApp.delete({ where: { @@ -158,10 +180,16 @@ export class OAuthService { async getOAuthList(): Promise { const userId = this.cls.get('user.id'); + const isAdmin = this.cls.get('user.isAdmin'); + const where: Prisma.OAuthAppWhereInput = isAdmin + ? { + OR: [{ createdBy: userId }, { createdBy: SYSTEM_USER_ID }], + } + : { + createdBy: userId, + }; const res = await this.prismaService.oAuthApp.findMany({ - where: { - createdBy: userId, - }, + where, select: { clientId: true, name: true, @@ -174,6 +202,7 @@ export class OAuthService { } async generateSecret(clientId: string): Promise { + await this.validateOwnership(clientId); const secret = getRandomString(40).toLocaleLowerCase(); const hashedSecret = await bcrypt.hash(secret, 10); @@ -198,6 +227,7 @@ export class OAuthService { } async deleteSecret(clientId: string, secretId: string): Promise { + await this.validateOwnership(clientId); await this.prismaService.oAuthAppSecret.delete({ where: { id: secretId, @@ -207,14 +237,7 @@ export class OAuthService { } async revokeAccess(clientId: string) { - // validate clientId is match with current user - const currentUserId = this.cls.get('user.id'); - const app = await this.prismaService.oAuthApp.findFirst({ - where: { clientId, createdBy: currentUserId }, - }); - if (!app) { - throw new ForbiddenException('No permission to revoke access: ' + clientId); - } + await this.validateOwnership(clientId); await this.prismaService.$tx(async (prisma) => { await prisma.oAuthAppAuthorized.deleteMany({ where: { clientId }, diff --git a/apps/nestjs-backend/src/features/record/computed/services/computed-dependency-collector.service.ts b/apps/nestjs-backend/src/features/record/computed/services/computed-dependency-collector.service.ts index 66754071fe..172a0c4f3a 100644 --- a/apps/nestjs-backend/src/features/record/computed/services/computed-dependency-collector.service.ts +++ b/apps/nestjs-backend/src/features/record/computed/services/computed-dependency-collector.service.ts @@ -16,11 +16,11 @@ import type { } from '@teable/core'; import { DbFieldType, DriverClient, FieldType, isFieldReferenceValue } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; import { InjectDbProvider } from '../../../../db-provider/db.provider'; import { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import { DatabaseRouter } from '../../../../global/database-router.service'; import { CUSTOM_KNEX, DATA_KNEX } from '../../../../global/knex/knex.module'; import { Timing } from '../../../../utils/timing'; import type { ICellContext } from '../../../calculation/utils/changes'; @@ -76,7 +76,7 @@ export class ComputedDependencyCollectorService { private logger = new Logger(ComputedDependencyCollectorService.name); constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly tableDomainQueryService: TableDomainQueryService, @InjectModel(CUSTOM_KNEX) private readonly knex: Knex, @InjectModel(DATA_KNEX) private readonly dataKnex: Knex, @@ -155,9 +155,11 @@ export class ComputedDependencyCollectorService { const qb = (schema ? this.dataKnex.withSchema(schema) : this.dataKnex) .select('__id') .from(table); - const rows = await this.dataPrismaService - .txClient() - .$queryRawUnsafe>(qb.toQuery()); + const rows = await this.databaseRouter.queryDataPrismaForTable>( + tableId, + qb.toQuery(), + { useTransaction: true } + ); return rows.map((r) => r.__id).filter(Boolean); } @@ -870,9 +872,9 @@ export class ComputedDependencyCollectorService { const sql = queryBuilder.toQuery(); this.logger.debug(`Conditional Rollup Impacted Records SQL: ${sql}`); - const rows = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id?: string; __id?: string }[]>(sql); + const rows = await this.databaseRouter.queryDataPrismaForTable< + { id?: string; __id?: string }[] + >(edge.tableId, sql, { useTransaction: true }); const ids = new Set(); for (const row of rows) { @@ -915,9 +917,11 @@ export class ComputedDependencyCollectorService { `${baseIdAlias}.__id` ); - const baseRows = await this.dataPrismaService - .txClient() - .$queryRawUnsafe[]>(baseRowsQuery.toQuery()); + const baseRows = await this.databaseRouter.queryDataPrismaForTable[]>( + edge.foreignTableId, + baseRowsQuery.toQuery(), + { useTransaction: true } + ); const baseRowById = new Map>(); for (const row of baseRows) { const id = row['__id']; @@ -1043,9 +1047,9 @@ export class ComputedDependencyCollectorService { const postQuery = postQueryBuilder.toQuery(); this.logger.debug('postQuery %s', postQuery); - const postRows = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id?: string; __id?: string }[]>(postQuery); + const postRows = await this.databaseRouter.queryDataPrismaForTable< + { id?: string; __id?: string }[] + >(edge.tableId, postQuery, { useTransaction: true }); for (const row of postRows) { const id = row.id || row.__id; diff --git a/apps/nestjs-backend/src/features/record/computed/services/computed-orchestrator.service.ts b/apps/nestjs-backend/src/features/record/computed/services/computed-orchestrator.service.ts index ab50385c0d..8682f8edbc 100644 --- a/apps/nestjs-backend/src/features/record/computed/services/computed-orchestrator.service.ts +++ b/apps/nestjs-backend/src/features/record/computed/services/computed-orchestrator.service.ts @@ -2,11 +2,11 @@ import { Injectable, NotFoundException } from '@nestjs/common'; import { FieldType } from '@teable/core'; import type { TableDomain, LastModifiedByFieldCore, LastModifiedTimeFieldCore } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { ClsService } from 'nestjs-cls'; import { InjectDbProvider } from '../../../../db-provider/db.provider'; import { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import { DatabaseRouter } from '../../../../global/database-router.service'; import type { IClsStore } from '../../../../types/cls'; import { Timing } from '../../../../utils/timing'; import type { ICellContext } from '../../../calculation/utils/changes'; @@ -25,7 +25,7 @@ export class ComputedOrchestratorService { private readonly collector: ComputedDependencyCollectorService, private readonly evaluator: ComputedEvaluatorService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly tableDomainQueryService: TableDomainQueryService, private readonly cls: ClsService, @InjectDbProvider() private readonly dbProvider: IDbProvider @@ -441,7 +441,9 @@ export class ComputedOrchestratorService { recordIds: target.recordIds, }); if (sql) { - await this.dataPrismaService.txClient().$queryRawUnsafe(sql); + await this.databaseRouter.queryDataPrismaForTable(target.tableId, sql, { + useTransaction: true, + }); } } } diff --git a/apps/nestjs-backend/src/features/record/computed/services/link-cascade-resolver.ts b/apps/nestjs-backend/src/features/record/computed/services/link-cascade-resolver.ts index 84c19663bf..8c3fa0bbb2 100644 --- a/apps/nestjs-backend/src/features/record/computed/services/link-cascade-resolver.ts +++ b/apps/nestjs-backend/src/features/record/computed/services/link-cascade-resolver.ts @@ -1,8 +1,8 @@ /* eslint-disable sonarjs/cognitive-complexity */ /* eslint-disable @typescript-eslint/naming-convention */ import { Injectable } from '@nestjs/common'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { chunk } from 'lodash'; +import { DatabaseRouter } from '../../../../global/database-router.service'; import { Timing } from '../../../../utils/timing'; export interface ILinkEdge { @@ -36,7 +36,7 @@ const IN_CHUNK = 500; @Injectable() export class LinkCascadeResolver { - constructor(private readonly dataPrismaService: DataPrismaService) {} + constructor(private readonly databaseRouter: DatabaseRouter) {} /** * Iterative BFS over link edges using only frontier ids; avoids full edge table scans and keeps @@ -186,9 +186,12 @@ from ${fkTableRef} where ${srcCol} in (${placeholders}) and ${srcCol} is not null and ${dstCol} is not null`; - return await this.dataPrismaService - .txClient() - .$queryRawUnsafe>(sql, ...srcIds); + return await this.databaseRouter.queryDataPrismaForTable>( + edge.foreignTableId, + sql, + { useTransaction: true }, + ...srcIds + ); } private async fetchEdgeTargetsBatched( @@ -211,7 +214,11 @@ where ${srcCol} in (${placeholders}) from ${fkTableRef} where ${srcCol} is not null and ${dstCol} is not null`; - return this.dataPrismaService.txClient().$queryRawUnsafe>(sql); + return this.databaseRouter.queryDataPrismaForTable>( + edge.foreignTableId, + sql, + { useTransaction: true } + ); } private quoteIdentifier(identifier: string): string { diff --git a/apps/nestjs-backend/src/features/record/computed/services/record-computed-update.service.ts b/apps/nestjs-backend/src/features/record/computed/services/record-computed-update.service.ts index 7cee267569..b87500440c 100644 --- a/apps/nestjs-backend/src/features/record/computed/services/record-computed-update.service.ts +++ b/apps/nestjs-backend/src/features/record/computed/services/record-computed-update.service.ts @@ -2,12 +2,12 @@ import { Injectable, Logger } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { FieldType } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; import { match } from 'ts-pattern'; import { InjectDbProvider } from '../../../../db-provider/db.provider'; import { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import { DatabaseRouter } from '../../../../global/database-router.service'; import { retryOnDeadlock } from '../../../../utils/retry-decorator'; import { Timing } from '../../../../utils/timing'; import { AUTO_NUMBER_FIELD_NAME } from '../../../field/constant'; @@ -20,7 +20,7 @@ export class RecordComputedUpdateService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @InjectDbProvider() private readonly dbProvider: IDbProvider ) {} @@ -115,7 +115,7 @@ export class RecordComputedUpdateService { } @Timing() - private async lockRestrictRecords(dbTableName: string, recordIds?: string[]) { + private async lockRestrictRecords(tableId: string, dbTableName: string, recordIds?: string[]) { if (!recordIds?.length) { return; } @@ -130,7 +130,7 @@ export class RecordComputedUpdateService { if (!sql) { return; } - await this.dataPrismaService.txClient().$queryRawUnsafe(sql); + await this.databaseRouter.queryDataPrismaForTable(tableId, sql, { useTransaction: true }); } @retryOnDeadlock() @@ -159,7 +159,7 @@ export class RecordComputedUpdateService { // Acquire row-level locks in a deterministic order to avoid deadlocks when multiple // computed updates touch the same set of records concurrently. - await this.lockRestrictRecords(dbTableName, restrictRecordIds); + await this.lockRestrictRecords(tableId, dbTableName, restrictRecordIds); const sql = this.dbProvider.updateFromSelectSql({ dbTableName, @@ -171,9 +171,9 @@ export class RecordComputedUpdateService { }); this.logger.debug('updateFromSelect SQL:', sql); try { - return await this.dataPrismaService - .txClient() - .$queryRawUnsafe>>(sql); + return await this.databaseRouter.queryDataPrismaForTable< + Array<{ __id: string; __version: number } & Record> + >(tableId, sql, { useTransaction: true }); } catch (error) { this.handleRawQueryError(error, sql, tableId, fields); } diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts index 3bc7978e4f..89db78a19c 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts @@ -30,8 +30,12 @@ describe('RecordOpenApiV2Service', () => { const resolve = vi.fn(); const getContainer = vi.fn(); const clsGet = vi.fn(); + const clsSet = vi.fn(); + const clsRunWith = vi.fn(); const cacheDel = vi.fn(); const cacheSetDetail = vi.fn(); + const getDataDatabaseForTable = vi.fn(); + const dataPrismaForTable = vi.fn(); let service: RecordOpenApiV2Service; @@ -138,6 +142,9 @@ describe('RecordOpenApiV2Service', () => { getContainer.mockResolvedValue({ resolve }); createContext.mockResolvedValue({}); clsGet.mockImplementation((key: string) => { + if (key == null) { + return {}; + } if (key === 'user.id') { return `usr${'h'.repeat(16)}`; } @@ -146,8 +153,14 @@ describe('RecordOpenApiV2Service', () => { } return undefined; }); + clsRunWith.mockImplementation((_store, fn: () => unknown) => fn()); getReadQuerySource.mockResolvedValue(undefined); getFieldsByQuery.mockResolvedValue([]); + getDataDatabaseForTable.mockResolvedValue({ + cacheKey: 'meta-fallback', + url: 'postgresql://meta', + isMetaFallback: true, + }); commandExecute.mockResolvedValue({ isErr: () => false, value: UpdateRecordsResult.create(2, []), @@ -169,15 +182,16 @@ describe('RecordOpenApiV2Service', () => { { data: { id: 'rec2222222222222222', fields: {} } }, ]); service = new RecordOpenApiV2Service( - { getContainer } as never, + { getContainerForTable: getContainer } as never, { createContext } as never, { getDocIdsByQuery, getSnapshotBulkWithPermission } as never, {} as never, - { get: clsGet } as never, + { get: clsGet, set: clsSet, runWith: clsRunWith } as never, { del: cacheDel, setDetail: cacheSetDetail } as never, { getFieldsByQuery } as never, { getReadQuerySource } as never, - {} as never + {} as never, + { getDataDatabaseForTable, dataPrismaForTable } as never ); }); @@ -266,6 +280,33 @@ describe('RecordOpenApiV2Service', () => { ]); }); + it('runs legacy snapshot compatibility reads against the table data client for BYODB tables', async () => { + const tableId = `tbl${'c'.repeat(16)}`; + const dataPrisma = { $queryRawUnsafe: vi.fn() }; + getDataDatabaseForTable.mockResolvedValue({ + cacheKey: 'ddc-byodb', + url: 'postgresql://byodb', + isMetaFallback: false, + }); + dataPrismaForTable.mockResolvedValue(dataPrisma); + + const result = await service.getRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 2, + }); + + expect(result.records).toEqual([ + { id: 'rec1111111111111111', fields: {} }, + { id: 'rec2222222222222222', fields: {} }, + ]); + expect(dataPrismaForTable).toHaveBeenCalledWith(tableId); + expect(clsRunWith).toHaveBeenCalled(); + expect(clsSet).toHaveBeenCalledWith('dataTx.client', dataPrisma); + expect(clsSet).toHaveBeenLastCalledWith('dataTx.client', undefined); + expect(getSnapshotBulkWithPermission).toHaveBeenCalledTimes(1); + }); + it('formats sorted top-level system datetime fields in the final OpenAPI response', async () => { execute.mockResolvedValue({ isErr: () => false, diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts index 360a5b5950..10e284553d 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts @@ -48,6 +48,7 @@ import { executeListTableRecordsEndpoint, } from '@teable/v2-contract-http-implementation/handlers'; import { v2CoreTokens } from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; import { ClearStreamCommand, DeleteByRangeStreamCommand, @@ -73,6 +74,7 @@ import { ClsService } from 'nestjs-cls'; import { CacheService } from '../../../cache/cache.service'; import type { ICacheStore } from '../../../cache/types'; import { CustomHttpException, getDefaultCodeByStatus } from '../../../custom.exception'; +import { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import type { IClsStore } from '../../../types/cls'; import { AggregationService } from '../../aggregation/aggregation.service'; import { FieldService } from '../../field/field.service'; @@ -114,7 +116,8 @@ export class RecordOpenApiV2Service { private readonly cacheService: CacheService, private readonly fieldService: FieldService, private readonly recordPermissionService: RecordPermissionService, - private readonly aggregationService: AggregationService + private readonly aggregationService: AggregationService, + private readonly dataDbClientManager: DataDbClientManager ) {} private throwV2Error( @@ -218,7 +221,8 @@ export class RecordOpenApiV2Service { ); } - const context = await this.createV2ReadContext(tableId, query); + const container = await this.v2ContainerService.getContainerForTable(tableId); + const context = await this.createV2ReadContext(tableId, query, container); const enabledFieldIds = ( context as IExecutionContext & { recordReadQuerySource?: { enabledFieldIds?: string[] }; @@ -247,10 +251,9 @@ export class RecordOpenApiV2Service { })); const normalizedGroupBy = effectiveQuery.groupBy?.map((item) => item.fieldId); const queryExtra = this.shouldLoadQueryExtra(effectiveQuery) - ? await this.getQueryExtra(tableId, effectiveQuery) + ? await this.withTableDataClient(tableId, () => this.getQueryExtra(tableId, effectiveQuery)) : undefined; - const container = await this.v2ContainerService.getContainer(); const queryBus = container.resolve(v2CoreTokens.queryBus); const pageResult = await this.executeListRecordsEndpoint( { @@ -287,13 +290,15 @@ export class RecordOpenApiV2Service { } const recordIds = orderedRecords.map((record) => record.id); - const snapshots = await this.recordService.getSnapshotBulkWithPermission( - tableId, - recordIds, - snapshotProjection, - requestedFieldKeyType, - query.cellFormat, - true + const snapshots = await this.withTableDataClient(tableId, () => + this.recordService.getSnapshotBulkWithPermission( + tableId, + recordIds, + snapshotProjection, + requestedFieldKeyType, + query.cellFormat, + true + ) ); if (snapshots.length !== recordIds.length) { @@ -323,6 +328,27 @@ export class RecordOpenApiV2Service { : { records: normalizedRecords }; } + private async withTableDataClient(tableId: string, fn: () => Promise): Promise { + const resolvedDataDb = await this.dataDbClientManager.getDataDatabaseForTable(tableId); + if (resolvedDataDb.isMetaFallback) { + return fn(); + } + + const dataPrisma = await this.dataDbClientManager.dataPrismaForTable(tableId); + const cls = this.cls as unknown as ClsService<{ dataTx: { client?: unknown } }>; + const store = cls.get(); + const previousClient = cls.get('dataTx.client'); + + return cls.runWith(store, async () => { + cls.set('dataTx.client', dataPrisma); + try { + return await fn(); + } finally { + cls.set('dataTx.client', previousClient); + } + }); + } + private async formatSystemDatetimeFields( tableId: string, records: IRecord[], @@ -496,9 +522,10 @@ export class RecordOpenApiV2Service { private async createV2ReadContext( tableId: string, - query: Pick + query: Pick, + container: DependencyContainer ): Promise { - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const readSource = await this.recordPermissionService.getReadQuerySource(tableId, { viewId: query.viewId, keepPrimaryKey: Boolean(query.filterLinkCellSelected), @@ -576,9 +603,9 @@ export class RecordOpenApiV2Service { const fields = updateRecordRo.record.fields ?? {}; const hasFields = Object.keys(fields).length > 0; - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); if (hasFields || (hasOrder && order)) { // Convert v1 input format to v2 format @@ -645,9 +672,9 @@ export class RecordOpenApiV2Service { 'record.update.request.typecast': updateRecordsRo.typecast ?? false, }); - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const updateResult = await executeUpdateRecordsEndpoint( context, { @@ -684,9 +711,9 @@ export class RecordOpenApiV2Service { createRecordsRo: ICreateRecordsRo, _isAiInternal?: string ): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); // Preserve v1's default typecast behavior (false) to ensure proper validation const records = createRecordsRo.records; @@ -718,9 +745,9 @@ export class RecordOpenApiV2Service { } async formSubmit(tableId: string, formSubmitRo: IFormSubmitRo): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const result = await executeSubmitRecordEndpoint( context, @@ -755,9 +782,9 @@ export class RecordOpenApiV2Service { allowRecordExpansion?: boolean; } ): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); ( context as IExecutionContext & { [V2_RECORD_PASTE_AUDIT_CONTEXT_KEY]?: boolean; @@ -829,9 +856,9 @@ export class RecordOpenApiV2Service { allowRecordExpansion?: boolean; } ): Promise> { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); ( context as IExecutionContext & { [V2_RECORD_PASTE_AUDIT_CONTEXT_KEY]?: boolean; @@ -1097,9 +1124,9 @@ export class RecordOpenApiV2Service { } async clear(tableId: string, rangesRo: IRangesRo): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const rangeQuery = await this.normalizeRangeQuery(tableId, rangesRo); const normalizedFilter = await this.normalizeFilterForV2(tableId, rangeQuery.filter); @@ -1140,9 +1167,9 @@ export class RecordOpenApiV2Service { tableId: string, rangesRo: IRangesRo ): Promise> { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const rangeQuery = await this.normalizeRangeQuery(tableId, rangesRo); const normalizedFilter = await this.normalizeFilterForV2(tableId, rangeQuery.filter); @@ -1268,9 +1295,9 @@ export class RecordOpenApiV2Service { rangesRo: IRangesRo, _windowId?: string ): Promise<{ ids: string[] }> { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const rangeQuery = await this.normalizeRangeQuery(tableId, rangesRo); const sortWithGroupFallback = this.mergeGroupByIntoSort(rangeQuery.groupBy, rangeQuery.orderBy); @@ -1315,9 +1342,9 @@ export class RecordOpenApiV2Service { tableId: string, rangesRo: IRangesRo ): Promise> { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const rangeQuery = await this.normalizeRangeQuery(tableId, rangesRo); const sortWithGroupFallback = this.mergeGroupByIntoSort(rangeQuery.groupBy, rangeQuery.orderBy); @@ -1364,9 +1391,9 @@ export class RecordOpenApiV2Service { tableId: string, rangesRo: IRangesRo ): Promise> { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const rangeQuery = await this.normalizeRangeQuery(tableId, rangesRo); const sortWithGroupFallback = this.mergeGroupByIntoSort(rangeQuery.groupBy, rangeQuery.orderBy); @@ -1414,19 +1441,27 @@ export class RecordOpenApiV2Service { recordIds: string[], _windowId?: string ): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const queryBus = container.resolve(v2CoreTokens.queryBus); + const context = await this.v2ContextFactory.createContext(container); - // Query records before deletion to return them in V1 format - const recordSnapshots = await this.recordService.getSnapshotBulkWithPermission( - tableId, - recordIds, - undefined, - FieldKeyType.Id, - undefined, - true - ); + const recordsBeforeDelete: IRecord[] = []; + for (let index = 0; index < recordIds.length; index += 1000) { + const selectedRecordIds = recordIds.slice(index, index + 1000); + const page = await this.executeListRecordsEndpoint( + { + tableId, + fieldKeyType: FieldKeyType.Id, + selectedRecordIds, + limit: selectedRecordIds.length, + ignoreViewQuery: true, + }, + context, + queryBus + ); + recordsBeforeDelete.push(...(page.records as IRecord[])); + } const v2Input = { tableId, @@ -1440,7 +1475,7 @@ export class RecordOpenApiV2Service { // Return records that were deleted (V1 format) return { - records: recordSnapshots.map((snapshot) => snapshot.data as IRecord), + records: recordsBeforeDelete, }; } @@ -1961,9 +1996,9 @@ export class RecordOpenApiV2Service { recordId: string, order?: IRecordInsertOrderRo ): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const result = await executeDuplicateRecordEndpoint( context, diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.spec.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.spec.ts index 1e5375e466..41a934ce7d 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.spec.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.spec.ts @@ -5,10 +5,12 @@ const createService = ({ prismaService = {}, dataPrismaService = {}, recordService = {}, + dataDbClientManager, }: { prismaService?: unknown; dataPrismaService?: unknown; recordService?: unknown; + dataDbClientManager?: unknown; } = {}) => new RecordOpenApiService( prismaService as never, @@ -21,7 +23,10 @@ const createService = ({ {} as never, {} as never, {} as never, - {} as never + {} as never, + (dataDbClientManager ?? { + dataPrismaForTable: vi.fn().mockResolvedValue(dataPrismaService), + }) as never ); describe('RecordOpenApiService', () => { @@ -50,6 +55,9 @@ describe('RecordOpenApiService', () => { avatar: null, }, ]); + const dataPrismaForTable = vi.fn().mockResolvedValue({ + recordHistory: { findMany: dataRecordHistoryFindMany }, + }); const service = createService({ prismaService: { @@ -59,6 +67,9 @@ describe('RecordOpenApiService', () => { dataPrismaService: { recordHistory: { findMany: dataRecordHistoryFindMany }, }, + dataDbClientManager: { + dataPrismaForTable, + }, }); const result = await service.getRecordHistory( @@ -82,6 +93,7 @@ describe('RecordOpenApiService', () => { orderBy: { createdTime: 'desc' }, }) ); + expect(dataPrismaForTable).toHaveBeenCalledWith('tbl1'); expect(metaRecordHistoryFindMany).not.toHaveBeenCalled(); expect(userFindMany).toHaveBeenCalledWith({ where: { id: { in: ['usr1'] } }, diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.ts index d49fc3c3f8..242f060766 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.ts @@ -8,7 +8,6 @@ import type { IMakeOptional, } from '@teable/core'; import { FieldKeyType, FieldType, HttpErrorCode, ViewType } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { CreateRecordAction, @@ -32,6 +31,7 @@ import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.co import { CustomHttpException } from '../../../custom.exception'; import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; import { Events } from '../../../event-emitter/events'; +import { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import type { IClsStore } from '../../../types/cls'; import { retryOnDeadlock } from '../../../utils/retry-decorator'; import { AttachmentsService } from '../../attachments/attachments.service'; @@ -49,7 +49,6 @@ import type { IUpdateRecordsInternalRo } from '../type'; export class RecordOpenApiService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, private readonly recordService: RecordService, private readonly attachmentsService: AttachmentsService, private readonly recordModifyService: RecordModifyService, @@ -58,7 +57,8 @@ export class RecordOpenApiService { private readonly tableDomainQueryService: TableDomainQueryService, private readonly fieldService: FieldService, private readonly cls: ClsService, - private readonly eventEmitterService: EventEmitterService + private readonly eventEmitterService: EventEmitterService, + private readonly dataDbClientManager: DataDbClientManager ) {} @retryOnDeadlock() @@ -231,7 +231,8 @@ export class RecordOpenApiService { dateFilter['lte'] = new Date(endDate); } - const list = await this.dataPrismaService.recordHistory.findMany({ + const dataPrisma = await this.dataDbClientManager.dataPrismaForTable(tableId); + const list = await dataPrisma.recordHistory.findMany({ where: { tableId, ...(recordId ? { recordId } : {}), diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-create.service.ts b/apps/nestjs-backend/src/features/record/record-modify/record-create.service.ts index a9207210af..84a62f2da1 100644 --- a/apps/nestjs-backend/src/features/record/record-modify/record-create.service.ts +++ b/apps/nestjs-backend/src/features/record/record-modify/record-create.service.ts @@ -112,7 +112,7 @@ export class RecordCreateService { const typecastRecords = await this.shared.validateFieldsAndTypecast< IMakeOptional >(table, records, fieldKeyType, typecast); - await this.recordService.createRecordsOnlySql(table, typecastRecords); + await this.recordService.createRecordsOnlySql(table, typecastRecords, fieldKeyType); } private buildProjectionByTable( diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-modify.shared.service.ts b/apps/nestjs-backend/src/features/record/record-modify/record-modify.shared.service.ts index 1793a112e0..4d917edfdf 100644 --- a/apps/nestjs-backend/src/features/record/record-modify/record-modify.shared.service.ts +++ b/apps/nestjs-backend/src/features/record/record-modify/record-modify.shared.service.ts @@ -273,7 +273,11 @@ export class RecordModifySharedService { tableId: table.id, dbTableName, itemLength: recordCount, - indexField: await this.viewService.getOrCreateViewIndexField(dbTableName, orderRo.viewId), + indexField: await this.viewService.getOrCreateViewIndexFieldForTable( + table.id, + dbTableName, + orderRo.viewId + ), orderRo, update: async (result) => { indexes = result; diff --git a/apps/nestjs-backend/src/features/record/record-query.service.ts b/apps/nestjs-backend/src/features/record/record-query.service.ts index 4ec81bc200..d3ebb6a012 100644 --- a/apps/nestjs-backend/src/features/record/record-query.service.ts +++ b/apps/nestjs-backend/src/features/record/record-query.service.ts @@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { TableDomain, type IRecord } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; +import { DatabaseRouter } from '../../global/database-router.service'; import { Timing } from '../../utils/timing'; import type { IFieldInstance } from '../field/model/factory'; import { fieldCore2FieldInstance } from '../field/model/factory'; @@ -17,7 +17,7 @@ export class RecordQueryService { private readonly logger = new Logger(RecordQueryService.name); constructor( - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder ) {} @@ -59,9 +59,9 @@ export class RecordQueryService { this.logger.debug(`Querying records: ${sql}`); - const rawRecords = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ [key: string]: unknown }[]>(sql); + const rawRecords = await this.databaseRouter.queryDataPrismaForTable< + { [key: string]: unknown }[] + >(table.id, sql); const fields = table.fieldList.map((f) => fieldCore2FieldInstance(f)); diff --git a/apps/nestjs-backend/src/features/record/record.service.spec.ts b/apps/nestjs-backend/src/features/record/record.service.spec.ts index a5a3a1ee4d..16fe4e29d1 100644 --- a/apps/nestjs-backend/src/features/record/record.service.spec.ts +++ b/apps/nestjs-backend/src/features/record/record.service.spec.ts @@ -1,21 +1,67 @@ -import type { TestingModule } from '@nestjs/testing'; -import { Test } from '@nestjs/testing'; -import { GlobalModule } from '../../global/global.module'; -import { RecordModule } from './record.module'; +import { FieldKeyType, FieldType } from '@teable/core'; +import Knex from 'knex'; +import { vi } from 'vitest'; import { RecordService } from './record.service'; describe('RecordService', () => { - let service: RecordService; + it('writes SQL-only created record history into the routed data DB internal schema', async () => { + const dataKnex = Knex({ client: 'pg' }); + const executedSql: string[] = []; + const service = Object.create(RecordService.prototype) as RecordService & { + creditCheck: ReturnType; + getFieldsByProjection: ReturnType; + getWritableCreatedTimeFieldNames: ReturnType; + cls: { get: ReturnType }; + dbProvider: { batchInsertSql: ReturnType }; + databaseRouter: { + executeDataPrismaForTable: ReturnType; + dataKnexForTable: ReturnType; + getDataDatabaseUrlForTable: ReturnType; + }; + }; - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [GlobalModule, RecordModule], - }).compile(); + service.cls = { + get: vi.fn((key: string) => + key === 'user' ? { id: 'usrImport', name: 'User', email: 'user@example.com' } : undefined + ), + }; + service.creditCheck = vi.fn().mockResolvedValue(undefined); + service.getFieldsByProjection = vi.fn().mockResolvedValue([ + { + id: 'fldText', + name: 'Text', + type: FieldType.SingleLineText, + dbFieldName: 'fld_text', + convertCellValue2DBValue: vi.fn((value) => value), + }, + ]); + service.getWritableCreatedTimeFieldNames = vi.fn().mockResolvedValue(new Set()); + service.dbProvider = { + batchInsertSql: vi.fn().mockReturnValue('insert into "bse_data"."tbl_imported" values (...)'), + }; + service.databaseRouter = { + executeDataPrismaForTable: vi.fn(async (_tableId: string, sql: string) => { + executedSql.push(sql); + return 1; + }), + dataKnexForTable: vi.fn().mockResolvedValue(dataKnex), + getDataDatabaseUrlForTable: vi + .fn() + .mockResolvedValue('postgresql://user:pass@example.test:5432/data?schema=teable_internal'), + }; - service = module.get(RecordService); - }); + await service.createRecordsOnlySql( + { id: 'tblImport', dbTableName: 'bse_data.tbl_imported' } as never, + [{ fields: { fldText: 'Imported value' } }], + FieldKeyType.Id + ); + + expect(executedSql[0]).toContain('"bse_data"."tbl_imported"'); + expect(executedSql.some((sql) => sql.includes('"teable_internal"."record_history"'))).toBe( + true + ); + expect(executedSql.some((sql) => sql.includes('insert into "record_history"'))).toBe(false); - it('should be defined', () => { - expect(service).toBeDefined(); + await dataKnex.destroy(); }); }); diff --git a/apps/nestjs-backend/src/features/record/record.service.ts b/apps/nestjs-backend/src/features/record/record.service.ts index 29940997cb..ae05a2d969 100644 --- a/apps/nestjs-backend/src/features/record/record.service.ts +++ b/apps/nestjs-backend/src/features/record/record.service.ts @@ -28,6 +28,7 @@ import { DriverClient, FieldKeyType, FieldType, + generateRecordHistoryId, generateRecordId, HttpErrorCode, identify, @@ -44,7 +45,6 @@ import { TableDomain, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { CreateRecordAction, ICreateRecordsRo, @@ -70,6 +70,7 @@ import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; import { Events } from '../../event-emitter/events'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import { RawOpType } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; @@ -124,7 +125,7 @@ export class RecordService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly batchService: BatchService, private readonly cls: ClsService, private readonly cacheService: CacheService, @@ -147,7 +148,24 @@ export class RecordService { return field.dbFieldName; } + private getBaseIdFromDbTableName(dbTableName: string) { + return this.dbProvider.splitTableName(dbTableName)[0]; + } + + private async queryDataTableByPhysicalName( + dbTableName: string, + query: string, + ...values: unknown[] + ) { + return this.databaseRouter.queryDataPrismaForBase( + this.getBaseIdFromDbTableName(dbTableName), + query, + ...values + ); + } + private async getWritableCreatedTimeFieldNames( + tableId: string, dbTableName: string, fields: readonly FieldCore[] ): Promise> { @@ -184,9 +202,11 @@ export class RecordService { .toSQL() .toNative(); - const rows = await this.dataPrismaService - .txClient() - .$queryRawUnsafe(sqlNative.sql, ...sqlNative.bindings); + const rows = await this.databaseRouter.queryDataPrismaForTable( + tableId, + sqlNative.sql, + ...sqlNative.bindings + ); const columnStateMap = new Map(rows.map((row) => [row.column_name, row.is_generated])); return new Set( @@ -221,12 +241,20 @@ export class RecordService { }, {}); } - async getAllRecordCount(dbTableName: string) { + async getAllRecordCount(dbTableName: string, tableId?: string) { const sqlNative = this.knex(dbTableName).count({ count: '*' }).toSQL().toNative(); - const queryResult = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ count?: number }[]>(sqlNative.sql, ...sqlNative.bindings); + const queryResult = tableId + ? await this.databaseRouter.queryDataPrismaForTable<{ count?: number }[]>( + tableId, + sqlNative.sql, + ...sqlNative.bindings + ) + : await this.queryDataTableByPhysicalName<{ count?: number }[]>( + dbTableName, + sqlNative.sql, + ...sqlNative.bindings + ); return Number(queryResult[0]?.count ?? 0); } @@ -279,7 +307,6 @@ export class RecordService { private async getLinkCellIds(tableId: string, field: IFieldInstance, recordId: string) { const prisma = this.prismaService.txClient(); - const dataPrisma = this.dataPrismaService.txClient(); const { dbTableName } = await prisma.tableMeta.findFirstOrThrow({ where: { id: tableId }, select: { dbTableName: true }, @@ -296,7 +323,9 @@ export class RecordService { ); const sql = queryBuilder.where('__id', recordId).toQuery(); - const result = await dataPrisma.$queryRawUnsafe<{ id: string; [key: string]: unknown }[]>(sql); + const result = await this.databaseRouter.queryDataPrismaForTable< + { id: string; [key: string]: unknown }[] + >(tableId, sql); return result .map((item) => { return field.convertDBValue2CellValue(item[field.dbFieldName]) as @@ -765,7 +794,7 @@ export class RecordService { }; } - async getBasicOrderIndexField(dbTableName: string, viewId: string | undefined) { + async getBasicOrderIndexField(tableId: string, dbTableName: string, viewId: string | undefined) { if (!viewId) { return '__auto_number'; } @@ -773,7 +802,7 @@ export class RecordService { const exists = await this.dbProvider.checkColumnExist( dbTableName, columnName, - this.dataPrismaService.txClient() + await this.databaseRouter.dataPrismaExecutorForTable(tableId) ); if (exists) { @@ -825,7 +854,7 @@ export class RecordService { enabledFieldIds, } = await this.prepareQuery(tableId, query); - const basicSortIndex = await this.getBasicOrderIndexField(dbTableName, query.viewId); + const basicSortIndex = await this.getBasicOrderIndexField(tableId, dbTableName, query.viewId); const restrictRecordIds = query.selectedRecordIds && !query.filterLinkCellCandidate @@ -1110,12 +1139,14 @@ export class RecordService { return record.fields[fieldId]; } - async getMaxRecordOrder(dbTableName: string) { + async getMaxRecordOrder(tableId: string, dbTableName: string) { const sqlNative = this.knex(dbTableName).max('__auto_number', { as: 'max' }).toSQL().toNative(); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ max?: number }[]>(sqlNative.sql, ...sqlNative.bindings); + const result = await this.databaseRouter.queryDataPrismaForTable<{ max?: number }[]>( + tableId, + sqlNative.sql, + ...sqlNative.bindings + ); return Number(result[0]?.max ?? 0) + 1; } @@ -1127,9 +1158,9 @@ export class RecordService { .select('__id as id', '__version as version') .whereIn('__id', recordIds) .toQuery(); - const recordRaw = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id: string; version: number }[]>(nativeQuery); + const recordRaw = await this.databaseRouter.queryDataPrismaForTable< + { id: string; version: number }[] + >(tableId, nativeQuery); if (recordIds.length !== recordRaw.length) { throw new CustomHttpException( @@ -1158,11 +1189,12 @@ export class RecordService { await this.batchDel(tableId, recordIds); } - private async getViewIndexColumns(dbTableName: string) { + private async getViewIndexColumns(tableId: string, dbTableName: string) { const columnInfoQuery = this.dbProvider.columnInfo(dbTableName); - const columns = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ name: string }[]>(columnInfoQuery); + const columns = await this.databaseRouter.queryDataPrismaForTable<{ name: string }[]>( + tableId, + columnInfoQuery + ); return columns .filter((column) => column.name.startsWith(ROW_ORDER_FIELD_PREFIX)) .map((column) => column.name); @@ -1175,7 +1207,7 @@ export class RecordService { viewId?: string ): Promise[] | undefined> { const dbTableName = table.dbTableName; - const allViewIndexColumns = await this.getViewIndexColumns(dbTableName); + const allViewIndexColumns = await this.getViewIndexColumns(table.id, dbTableName); const viewIndexColumns = viewId ? (() => { const viewIndexColumns = allViewIndexColumns.filter((column) => column.endsWith(viewId)); @@ -1203,9 +1235,10 @@ export class RecordService { .select('__id') .whereIn('__id', recordIds) .toQuery(); - const indexValues = await this.dataPrismaService - .txClient() - .$queryRawUnsafe[]>(indexQuery); + const indexValues = await this.databaseRouter.queryDataPrismaForTable[]>( + table.id, + indexQuery + ); const indexMap = indexValues.reduce>>((map, cur) => { const id = cur.__id; @@ -1225,7 +1258,7 @@ export class RecordService { }[] ) { const dbTableName = await this.getDbTableName(tableId); - const viewIndexColumns = await this.getViewIndexColumns(dbTableName); + const viewIndexColumns = await this.getViewIndexColumns(tableId, dbTableName); if (!viewIndexColumns.length) { return; } @@ -1251,7 +1284,7 @@ export class RecordService { .filter(Boolean) as string[]; for (const sql of updateRecordSqls) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } } @@ -1277,7 +1310,8 @@ export class RecordService { table: TableDomain, records: { fields: Record; - }[] + }[], + fieldKeyType: FieldKeyType = FieldKeyType.Id ) { const user = this.cls.get('user'); const userId = user.id; @@ -1285,6 +1319,7 @@ export class RecordService { const dbTableName = table.dbTableName; const fields = await this.getFieldsByProjection(table.id); const writableCreatedTimeFieldNames = await this.getWritableCreatedTimeFieldNames( + table.id, dbTableName, fields ); @@ -1300,19 +1335,40 @@ export class RecordService { ) as IFieldInstance[]; const fieldInstanceMap = fields.reduce( (map, curField) => { - map[curField.id] = curField; + map[curField[fieldKeyType]] = curField; return map; }, {} as Record ); + const recordHistoryList: { + id: string; + table_id: string; + record_id: string; + field_id: string; + before: string; + after: string; + created_by: string; + }[] = []; const newRecords = records.map((record) => { const createdTime = writableCreatedTimeFieldNames.size > 0 ? new Date().toISOString() : undefined; const fieldsValues: Record = {}; + const recordId = generateRecordId(); Object.entries(record.fields).forEach(([fieldId, value]) => { const fieldInstance = fieldInstanceMap[fieldId]; fieldsValues[fieldInstance.dbFieldName] = fieldInstance.convertCellValue2DBValue(value); + if (value !== '' && value != null) { + recordHistoryList.push({ + id: generateRecordHistoryId(), + table_id: table.id, + record_id: recordId, + field_id: fieldInstance.id, + before: JSON.stringify({ data: null }), + after: JSON.stringify({ data: value }), + created_by: userId, + }); + } }); if (auditUserValue && createdByFields.length) { createdByFields.forEach((field) => { @@ -1327,7 +1383,7 @@ export class RecordService { } }); return removeUndefined({ - __id: generateRecordId(), + __id: recordId, __created_by: userId, __created_time: createdTime, __version: 1, @@ -1335,7 +1391,18 @@ export class RecordService { }); }); const sql = this.dbProvider.batchInsertSql(dbTableName, newRecords); - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(table.id, sql); + if (recordHistoryList.length) { + const dataKnex = await this.databaseRouter.dataKnexForTable(table.id); + const dataDbUrl = await this.databaseRouter.getDataDatabaseUrlForTable(table.id); + const dataDbInternalSchema = new URL(dataDbUrl).searchParams.get('schema') || 'public'; + const historySql = dataKnex + .withSchema(dataDbInternalSchema) + .insert(recordHistoryList) + .into('record_history') + .toQuery(); + await this.databaseRouter.executeDataPrismaForTable(table.id, historySql); + } } async creditCheck(tableId: string) { @@ -1348,7 +1415,7 @@ export class RecordService { select: { dbTableName: true, base: { select: { space: { select: { credit: true } } } } }, }); - const rowCount = await this.getAllRecordCount(table.dbTableName); + const rowCount = await this.getAllRecordCount(table.dbTableName, tableId); const maxRowCount = table.base.space.credit == null @@ -1372,11 +1439,12 @@ export class RecordService { } } - private async getAllViewIndexesField(dbTableName: string) { + private async getAllViewIndexesField(tableId: string, dbTableName: string) { const query = this.dbProvider.columnInfo(dbTableName); - const columns = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ name: string }[]>(query); + const columns = await this.databaseRouter.queryDataPrismaForTable<{ name: string }[]>( + tableId, + query + ); return columns .filter((column) => column.name.startsWith(ROW_ORDER_FIELD_PREFIX)) .map((column) => column.name) @@ -1423,8 +1491,9 @@ export class RecordService { await this.creditCheck(table.id); const { dbTableName, name: tableName } = table; - const maxRecordOrder = await this.getMaxRecordOrder(dbTableName); + const maxRecordOrder = await this.getMaxRecordOrder(table.id, dbTableName); const writableCreatedTimeFieldNames = await this.getWritableCreatedTimeFieldNames( + table.id, dbTableName, fields ); @@ -1434,7 +1503,7 @@ export class RecordService { select: { id: true }, }); - const allViewIndexes = await this.getAllViewIndexesField(dbTableName); + const allViewIndexes = await this.getAllViewIndexesField(table.id, dbTableName); const validationFields = fields .filter((f) => !f.isComputed) @@ -1554,7 +1623,7 @@ export class RecordService { ); await handleDBValidationErrors({ - fn: () => this.dataPrismaService.txClient().$executeRawUnsafe(sql), + fn: () => this.databaseRouter.executeDataPrismaForTable(table.id, sql), handleUniqueError: () => { throw new CustomHttpException( `Fields ${validationFields.map((f) => f.id).join(', ')} unique validation failed`, @@ -1594,7 +1663,7 @@ export class RecordService { const dbTableName = await this.getDbTableName(tableId); const nativeQuery = this.knex(dbTableName).whereIn('__id', recordIds).del().toQuery(); - await this.dataPrismaService.txClient().$executeRawUnsafe(nativeQuery); + await this.databaseRouter.executeDataPrismaForTable(tableId, nativeQuery); } public async getFieldsByProjection( @@ -1825,11 +1894,9 @@ export class RecordService { let result: ({ [fieldName: string]: unknown } & IVisualTableDefaultField)[]; try { - result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe< - ({ [fieldName: string]: unknown } & IVisualTableDefaultField)[] - >(nativeQuery); + result = await this.databaseRouter.queryDataPrismaForTable< + ({ [fieldName: string]: unknown } & IVisualTableDefaultField)[] + >(tableId, nativeQuery); } catch (error) { this.handleRawQueryError(error, nativeQuery, { tableId, @@ -2008,9 +2075,11 @@ export class RecordService { this.logger.debug('getRecordsQuery: %s', sqlDebug); let result: { __id: string }[]; try { - result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ __id: string }[]>(sqlNative.sql, ...sqlNative.bindings); + result = await this.databaseRouter.queryDataPrismaForTable<{ __id: string }[]>( + tableId, + sqlNative.sql, + ...sqlNative.bindings + ); } catch (error) { this.handleRawQueryError(error, sqlNative.sql, { tableId, @@ -2226,9 +2295,9 @@ export class RecordService { this.logger.debug('getSearchHitIndex query: %s', searchQuery); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ __id: string; fieldId: string }[]>(searchQuery); + const result = await this.databaseRouter.queryDataPrismaForTable< + { __id: string; fieldId: string }[] + >(tableId, searchQuery); if (!result.length) { return null; @@ -2304,9 +2373,9 @@ export class RecordService { this.logger.debug('getRecordsFields query: %s', sql); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<(Pick & Pick)[]>(sql); + const result = await this.databaseRouter.queryDataPrismaForTable< + (Pick & Pick)[] + >(tableId, sql); return result.map((record) => { return { @@ -2349,9 +2418,10 @@ export class RecordService { const querySql = queryBuilder.toQuery(); - return this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id: string; title: string }[]>(querySql); + return this.databaseRouter.queryDataPrismaForTable<{ id: string; title: string }[]>( + tableId, + querySql + ); } async getRecordsHeadWithIds(tableId: string, recordIds: string[]) { @@ -2364,9 +2434,9 @@ export class RecordService { const querySql = queryBuilder.toQuery(); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id: string; title: unknown }[]>(querySql); + const result = await this.databaseRouter.queryDataPrismaForTable< + { id: string; title: unknown }[] + >(tableId, querySql); return result.map((r) => ({ id: r.id, @@ -2387,9 +2457,10 @@ export class RecordService { true ); queryBuilder.whereIn(`${alias}.__id`, recordIds); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ __id: string }[]>(queryBuilder.toQuery()); + const result = await this.databaseRouter.queryDataPrismaForTable<{ __id: string }[]>( + tableId, + queryBuilder.toQuery() + ); return result.map((r) => r.__id); } @@ -2599,9 +2670,10 @@ export class RecordService { const rowCountSql = qb.count({ count: '*' }); const sql = rowCountSql.toQuery(); this.logger.debug('getRowCountSql: %s', sql); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ count?: number }[]>(sql); + const result = await this.databaseRouter.queryDataPrismaForTable<{ count?: number }[]>( + tableId, + sql + ); return Number(result[0].count); } @@ -2711,9 +2783,9 @@ export class RecordService { ); try { - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ [key: string]: unknown; __c: number }[]>(groupSql); + const result = await this.databaseRouter.queryDataPrismaForTable< + { [key: string]: unknown; __c: number }[] + >(tableId, groupSql); const pointsResult = await this.groupDbCollection2GroupPoints( result, groupFields, @@ -2750,9 +2822,10 @@ export class RecordService { const dbTableName = await this.getDbTableName(tableId); const queryBuilder = this.knex(dbTableName).select('__id').where('__id', recordId).limit(1); - const result = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ __id: string }[]>(queryBuilder.toQuery()); + const result = await this.databaseRouter.queryDataPrismaForTable<{ __id: string }[]>( + tableId, + queryBuilder.toQuery() + ); const isDeleted = result.length === 0; @@ -2840,9 +2913,9 @@ export class RecordService { ); const collaboratorIdsQuery = collaboratorsQueryBuilder.distinct('user_id').toQuery(); - const collaboratorIds = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ user_id: string | null }[]>(collaboratorIdsQuery); + const collaboratorIds = await this.databaseRouter.queryDataPrismaForTable< + { user_id: string | null }[] + >(tableId, collaboratorIdsQuery); const userIds = Array.from( new Set( collaboratorIds diff --git a/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.controller.ts b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.controller.ts index 0a324a717b..9c91ecdf46 100644 --- a/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.controller.ts @@ -1,5 +1,11 @@ -import { Controller, Delete, Get, Param, Patch, Post, Query, Res } from '@nestjs/common'; +import { Body, Controller, Delete, Get, Param, Patch, Post, Query, Res } from '@nestjs/common'; +import { + adminSendNotificationRoSchema, + type IAdminSendNotificationRo, + type IAdminSendNotificationVo, +} from '@teable/openapi'; import { Response } from 'express'; +import { ZodValidationPipe } from '../../../zod.validation.pipe'; import { Permissions } from '../../auth/decorators/permissions.decorator'; import { AdminOpenApiService } from './admin-open-api.service'; @@ -37,4 +43,11 @@ export class AdminOpenApiController { async deletePerformanceCache(@Query('key') key?: string) { return await this.adminService.deletePerformanceCache(key); } + + @Post('notification') + async sendNotification( + @Body(new ZodValidationPipe(adminSendNotificationRoSchema)) ro: IAdminSendNotificationRo + ): Promise { + return this.adminService.sendAdminNotification(ro); + } } diff --git a/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.module.ts b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.module.ts index be3569725e..0b12050c72 100644 --- a/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.module.ts +++ b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.module.ts @@ -3,6 +3,7 @@ import { MulterModule } from '@nestjs/platform-express'; import multer from 'multer'; import { AttachmentsCropModule } from '../../attachments/attachments-crop.module'; import { StorageModule } from '../../attachments/plugins/storage.module'; +import { NotificationModule } from '../../notification/notification.module'; import { AdminOpenApiController } from './admin-open-api.controller'; import { AdminOpenApiService } from './admin-open-api.service'; @@ -13,6 +14,7 @@ import { AdminOpenApiService } from './admin-open-api.service'; storage: multer.diskStorage({}), }), StorageModule, + NotificationModule, ], controllers: [AdminOpenApiController], exports: [AdminOpenApiService], diff --git a/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.service.ts b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.service.ts index 6c0f57752f..8d62291c4e 100644 --- a/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.service.ts +++ b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.service.ts @@ -7,15 +7,20 @@ import { InternalServerErrorException, Logger, } from '@nestjs/common'; +import { NotificationSeverityEnum, NotificationTypeEnum } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import type { IAdminSendNotificationRo } from '@teable/openapi'; import { PluginStatus, UploadType } from '@teable/openapi'; import { Response } from 'express'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; import { PerformanceCacheService } from '../../../performance-cache'; +import type { IClsStore } from '../../../types/cls'; import { Timing } from '../../../utils/timing'; import { AttachmentsCropQueueProcessor } from '../../attachments/attachments-crop.processor'; import StorageAdapter from '../../attachments/plugins/adapter'; +import { NotificationService } from '../../notification/notification.service'; @Injectable() export class AdminOpenApiService { @@ -24,7 +29,9 @@ export class AdminOpenApiService { private readonly prismaService: PrismaService, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, private readonly attachmentsCropQueueProcessor: AttachmentsCropQueueProcessor, - private readonly performanceCacheService: PerformanceCacheService + private readonly performanceCacheService: PerformanceCacheService, + private readonly notificationService: NotificationService, + private readonly cls: ClsService ) {} async publishPlugin(pluginId: string) { @@ -178,4 +185,20 @@ export class AdminOpenApiService { // eslint-disable-next-line @typescript-eslint/no-explicit-any await this.performanceCacheService.del(key as any); } + + async sendAdminNotification(ro: IAdminSendNotificationRo) { + const fromUserId = this.cls.get('user.id'); + const { message, severity, userIds, emails } = ro; + + return this.notificationService.sendCommonNotify( + { + fromUserId, + toUserId: userIds, + toEmail: emails, + message, + severity, + }, + NotificationTypeEnum.AdminNotice + ); + } } diff --git a/apps/nestjs-backend/src/features/share/guard/auth.guard.ts b/apps/nestjs-backend/src/features/share/guard/auth.guard.ts index b23d7d9353..01d2671dc0 100644 --- a/apps/nestjs-backend/src/features/share/guard/auth.guard.ts +++ b/apps/nestjs-backend/src/features/share/guard/auth.guard.ts @@ -2,7 +2,7 @@ import type { ExecutionContext } from '@nestjs/common'; import { Injectable } from '@nestjs/common'; import { Reflector } from '@nestjs/core'; import { AuthGuard as PassportAuthGuard } from '@nestjs/passport'; -import { ANONYMOUS_USER_ID, HttpErrorCode, IdPrefix } from '@teable/core'; +import { ANONYMOUS_USER_ID, HttpErrorCode, IdPrefix, ViewType } from '@teable/core'; import { ClsService } from 'nestjs-cls'; import { CustomHttpException } from '../../../custom.exception'; import type { IClsStore } from '../../../types/cls'; @@ -50,7 +50,8 @@ export class ShareAuthGuard extends PassportAuthGuard([SHARE_JWT_STRATEGY]) { context.getClass(), ]); const submit = shareInfo.shareMeta?.submit; - if (isShareSubmit && submit?.allow && submit?.requireLogin) { + const isFormView = shareInfo.view?.type === ViewType.Form; + if (isShareSubmit && isFormView && submit?.requireLogin) { return this.authGuard.validate(context); } diff --git a/apps/nestjs-backend/src/features/share/share.service.ts b/apps/nestjs-backend/src/features/share/share.service.ts index e532d50090..bb8ee919da 100644 --- a/apps/nestjs-backend/src/features/share/share.service.ts +++ b/apps/nestjs-backend/src/features/share/share.service.ts @@ -3,7 +3,6 @@ import { Injectable } from '@nestjs/common'; import type { IFilter, IFieldVo, IViewVo, ILinkFieldOptions, StatisticsFunc } from '@teable/core'; import { CellFormat, FieldKeyType, FieldType, HttpErrorCode, ViewType } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { ShareViewLinkRecordsType, PluginPosition } from '@teable/openapi'; import type { IShareViewCalendarDailyCollectionRo, @@ -29,6 +28,7 @@ import { ClsService } from 'nestjs-cls'; import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import type { IClsStore } from '../../types/cls'; import { convertViewVoAttachmentUrl } from '../../utils/convert-view-vo-attachment-url'; @@ -55,7 +55,7 @@ export interface IJwtShareInfo { export class ShareService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly fieldService: FieldService, private readonly recordService: RecordService, @InjectAggregationService() private readonly aggregationService: IAggregationService, @@ -253,19 +253,19 @@ export class ShareService { } async formSubmit(shareInfo: IShareViewInfo, shareViewFormSubmitRo: ShareViewFormSubmitRo) { - const { tableId, view, shareMeta } = shareInfo; + const { tableId, view } = shareInfo; const { fields, typecast } = shareViewFormSubmitRo; - if (!shareMeta?.submit?.allow) { - throw new CustomHttpException('not allowed to submit', HttpErrorCode.RESTRICTED_RESOURCE, { + if (!view) { + throw new CustomHttpException('view is required', HttpErrorCode.RESTRICTED_RESOURCE, { localization: { - i18nKey: 'httpErrors.share.notAllowedToSubmit', + i18nKey: 'httpErrors.share.viewRequired', }, }); } - if (!view) { - throw new CustomHttpException('view is required', HttpErrorCode.RESTRICTED_RESOURCE, { + if (view.type !== ViewType.Form) { + throw new CustomHttpException('not allowed to submit', HttpErrorCode.RESTRICTED_RESOURCE, { localization: { - i18nKey: 'httpErrors.share.viewRequired', + i18nKey: 'httpErrors.share.notAllowedToSubmit', }, }); } @@ -470,9 +470,10 @@ export class ShareService { queryBuilder.whereNotNull(dbFieldName); this.dbProvider.filterQuery(queryBuilder, fieldMap, filter).appendQueryBuilder(); const nativeQuery = queryBuilder.toQuery(); - const rows = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ user_id: string | null }[]>(nativeQuery); + const rows = await this.databaseRouter.queryDataPrismaForTable<{ user_id: string | null }[]>( + tableId, + nativeQuery + ); return Array.from( new Set( diff --git a/apps/nestjs-backend/src/features/space/data-db-baseline.service.ts b/apps/nestjs-backend/src/features/space/data-db-baseline.service.ts index 4d193c1ac0..6777ea8ac4 100644 --- a/apps/nestjs-backend/src/features/space/data-db-baseline.service.ts +++ b/apps/nestjs-backend/src/features/space/data-db-baseline.service.ts @@ -1,60 +1,12 @@ -import { existsSync, readFileSync } from 'fs'; -import { join } from 'path'; -import { Inject, Injectable, Optional } from '@nestjs/common'; -import { - IDataDbPreflightClientFactory, - DATA_DB_PREFLIGHT_CLIENT_FACTORY, - dataDbKnexClientFactory, -} from './data-db-preflight.service'; - -export const dataDbBaselineSqlToken = Symbol('DATA_DB_BASELINE_SQL'); - -const getBaselineSqlPath = () => { - const candidates = [ - join( - process.cwd(), - 'community/packages/db-data-prisma/prisma/migrations/20260421000000_init_data_db_baseline/migration.sql' - ), - join( - process.cwd(), - '../../community/packages/db-data-prisma/prisma/migrations/20260421000000_init_data_db_baseline/migration.sql' - ), - ]; - const found = candidates.find((path) => existsSync(path)); - if (!found) { - throw new Error('Data DB baseline SQL migration not found'); - } - return found; -}; - -const readBaselineSql = () => readFileSync(getBaselineSqlPath(), 'utf8'); +import { Injectable } from '@nestjs/common'; +import { DataDbMigrationService } from './data-db-migration.service'; @Injectable() export class DataDbBaselineService { - private readonly clientFactory: IDataDbPreflightClientFactory; - - constructor( - @Optional() - @Inject(dataDbBaselineSqlToken) - private readonly baselineSql?: string, - @Optional() - @Inject(DATA_DB_PREFLIGHT_CLIENT_FACTORY) - clientFactory?: IDataDbPreflightClientFactory - ) { - this.clientFactory = clientFactory ?? dataDbKnexClientFactory; - } - - async initialize(url: string) { - const sql = this.baselineSql ?? readBaselineSql(); - if (!sql.trim()) { - return; - } + constructor(private readonly migrationService: DataDbMigrationService) {} - const client = this.clientFactory(url); - try { - await client.raw(sql); - } finally { - await client.destroy().catch(() => undefined); - } + async initialize(url: string, internalSchema?: string) { + await this.migrationService.migrate(url, internalSchema); + return this.migrationService.getLatestSchemaVersion(); } } diff --git a/apps/nestjs-backend/src/features/space/data-db-binding.service.spec.ts b/apps/nestjs-backend/src/features/space/data-db-binding.service.spec.ts index f20f9e4b9c..578a108da7 100644 --- a/apps/nestjs-backend/src/features/space/data-db-binding.service.spec.ts +++ b/apps/nestjs-backend/src/features/space/data-db-binding.service.spec.ts @@ -1,9 +1,13 @@ +/* eslint-disable sonarjs/no-duplicate-string */ import { HttpErrorCode } from '@teable/core'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { DataDbBindingService } from './data-db-binding.service'; +import { encryptDataDbUrl } from './data-db-url-secret'; const dataUrl = 'postgresql://teable:secret@example.com:5432/teable_data'; const initializeEmptyTargetMode = 'initialize-empty'; +const internalSchema = 'teable_meta_test'; +const schemaVersion = '20260421000000_init_data_db_baseline'; const capabilities = { createSchema: true, createTable: true, @@ -18,13 +22,23 @@ describe('DataDbBindingService', () => { const txClient = { dataDbConnection: { upsert: vi.fn(), + update: vi.fn(), }, spaceDataDbBinding: { create: vi.fn(), + upsert: vi.fn(), + updateMany: vi.fn(), }, }; const prismaService = { $tx: vi.fn(async (fn: (client: typeof txClient) => Promise) => fn(txClient)), + dataDbConnection: { + update: vi.fn(), + }, + spaceDataDbBinding: { + findUnique: vi.fn(), + updateMany: vi.fn(), + }, }; const preflightService = { preflight: vi.fn(), @@ -32,13 +46,46 @@ describe('DataDbBindingService', () => { const baselineService = { initialize: vi.fn(), }; + const dataDbClientManager = { + invalidateConnection: vi.fn(), + }; + const dataDbMigrationService = { + ensureConnectionMigrated: vi.fn(), + }; + const byodbBinding = { + mode: 'byodb', + state: 'ready', + dataDbConnection: { + id: 'dcnxxx', + provider: 'postgres', + encryptedUrl: encryptDataDbUrl(dataUrl), + urlFingerprint: 'dbfp_old', + displayHost: 'example.com:5432', + displayDatabase: 'teable_data', + internalSchema, + schemaVersion, + status: 'ready', + capabilities, + lastValidatedAt: new Date('2026-05-06T00:00:00.000Z'), + lastError: null, + createdBy: 'usrxxx', + }, + }; beforeEach(() => { txClient.dataDbConnection.upsert.mockReset().mockResolvedValue({ id: 'dcnxxx' }); + txClient.dataDbConnection.update.mockReset(); txClient.spaceDataDbBinding.create.mockReset(); + txClient.spaceDataDbBinding.upsert.mockReset(); + txClient.spaceDataDbBinding.updateMany.mockReset(); prismaService.$tx.mockClear(); preflightService.preflight.mockReset(); - baselineService.initialize.mockReset(); + baselineService.initialize.mockReset().mockResolvedValue(schemaVersion); + dataDbClientManager.invalidateConnection.mockReset(); + dataDbMigrationService.ensureConnectionMigrated.mockReset().mockResolvedValue([]); + prismaService.dataDbConnection.update.mockReset(); + prismaService.spaceDataDbBinding.findUnique.mockReset().mockResolvedValue(byodbBinding); + prismaService.spaceDataDbBinding.updateMany.mockReset(); }); it('creates an encrypted connection and BYODB binding after successful preflight', async () => { @@ -52,27 +99,35 @@ describe('DataDbBindingService', () => { const service = new DataDbBindingService( prismaService as never, preflightService as never, - baselineService as never + baselineService as never, + dataDbClientManager as never ); await service.createBindingForNewSpace('spcxxx', 'usrxxx', { mode: 'byodb', url: dataUrl, targetMode: initializeEmptyTargetMode, + internalSchema, }); expect(preflightService.preflight).toHaveBeenCalledWith({ url: dataUrl, targetMode: initializeEmptyTargetMode, + internalSchema, }); - expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl); + expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl, internalSchema); expect(txClient.dataDbConnection.upsert).toHaveBeenCalledWith( expect.objectContaining({ where: expect.objectContaining({ urlFingerprint: expect.stringMatching(/^dbfp_/) }), create: expect.objectContaining({ encryptedUrl: expect.not.stringContaining('secret'), + internalSchema, + schemaVersion, status: 'ready', }), + update: expect.objectContaining({ + schemaVersion, + }), }) ); expect(txClient.spaceDataDbBinding.create).toHaveBeenCalledWith({ @@ -84,6 +139,85 @@ describe('DataDbBindingService', () => { createdBy: 'usrxxx', }, }); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnxxx'); + }); + + it('generates an internal schema for new BYODB spaces when one is not provided', async () => { + preflightService.preflight.mockResolvedValue({ + ok: true, + provider: 'postgres', + classification: 'empty', + capabilities, + errors: [], + }); + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never + ); + + await service.createBindingForNewSpace('spcxxx', 'usrxxx', { + mode: 'byodb', + url: dataUrl, + targetMode: initializeEmptyTargetMode, + }); + + const generatedInternalSchema = expect.stringMatching(/^teable_[a-f0-9]{16}$/); + expect(preflightService.preflight).toHaveBeenCalledWith({ + url: dataUrl, + targetMode: initializeEmptyTargetMode, + internalSchema: generatedInternalSchema, + }); + expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl, generatedInternalSchema); + expect(txClient.dataDbConnection.upsert).toHaveBeenCalledWith( + expect.objectContaining({ + create: expect.objectContaining({ internalSchema: generatedInternalSchema }), + }) + ); + }); + + it('reuses the same data DB connection for multiple spaces with the same URL', async () => { + preflightService.preflight.mockResolvedValue({ + ok: true, + provider: 'postgres', + classification: 'teable-managed-compatible', + capabilities, + errors: [], + }); + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never + ); + + const dataDb = { + mode: 'byodb' as const, + url: dataUrl, + targetMode: initializeEmptyTargetMode, + internalSchema, + }; + await service.createBindingForNewSpace('spcxxx1', 'usrxxx', dataDb); + await service.createBindingForNewSpace('spcxxx2', 'usrxxx', dataDb); + + expect(txClient.dataDbConnection.upsert).toHaveBeenCalledTimes(2); + expect(txClient.dataDbConnection.upsert.mock.calls[0]?.[0].where).toEqual( + txClient.dataDbConnection.upsert.mock.calls[1]?.[0].where + ); + expect(txClient.spaceDataDbBinding.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + spaceId: 'spcxxx1', + dataDbConnectionId: 'dcnxxx', + }), + }); + expect(txClient.spaceDataDbBinding.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + spaceId: 'spcxxx2', + dataDbConnectionId: 'dcnxxx', + }), + }); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledTimes(2); }); it('rejects BYODB space creation when preflight fails', async () => { @@ -97,7 +231,8 @@ describe('DataDbBindingService', () => { const service = new DataDbBindingService( prismaService as never, preflightService as never, - baselineService as never + baselineService as never, + dataDbClientManager as never ); await expect( @@ -105,9 +240,192 @@ describe('DataDbBindingService', () => { mode: 'byodb', url: dataUrl, targetMode: initializeEmptyTargetMode, + internalSchema, }) ).rejects.toMatchObject({ code: HttpErrorCode.CONFLICT }); expect(baselineService.initialize).not.toHaveBeenCalled(); expect(prismaService.$tx).not.toHaveBeenCalled(); }); + + it('retests an existing BYODB binding without exposing encrypted URL material', async () => { + preflightService.preflight.mockResolvedValue({ + ok: true, + provider: 'postgres', + classification: 'teable-managed-compatible', + capabilities, + errors: [], + }); + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never, + dataDbMigrationService as never + ); + + await service.retestBinding('spcxxx'); + + expect(preflightService.preflight).toHaveBeenCalledWith({ + url: dataUrl, + targetMode: initializeEmptyTargetMode, + internalSchema, + }); + expect(txClient.dataDbConnection.upsert).not.toHaveBeenCalled(); + expect(txClient.dataDbConnection.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'dcnxxx' }, + data: expect.objectContaining({ + status: 'ready', + lastError: null, + }), + }) + ); + }); + + it('retries migration for an existing BYODB binding', async () => { + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never, + dataDbMigrationService as never + ); + + await service.retryMigrationForSpace('spcxxx'); + + expect(dataDbMigrationService.ensureConnectionMigrated).toHaveBeenCalledWith({ + connectionId: 'dcnxxx', + internalSchema, + url: dataUrl, + }); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnxxx'); + }); + + it('updates credentials for the same BYODB database identity', async () => { + const updatedUrl = 'postgresql://teable:new-secret@example.com:5432/teable_data'; + preflightService.preflight.mockResolvedValue({ + ok: true, + provider: 'postgres', + classification: 'teable-managed-compatible', + capabilities, + errors: [], + }); + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never, + dataDbMigrationService as never + ); + + await service.updateBindingForSpace('spcxxx', 'usrxxx', { + url: updatedUrl, + targetMode: initializeEmptyTargetMode, + }); + + expect(preflightService.preflight).toHaveBeenCalledWith({ + url: updatedUrl, + targetMode: initializeEmptyTargetMode, + internalSchema, + }); + expect(baselineService.initialize).toHaveBeenCalledWith(updatedUrl, internalSchema); + expect(prismaService.dataDbConnection.update).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'dcnxxx' }, + data: expect.objectContaining({ + encryptedUrl: expect.not.stringContaining('new-secret'), + schemaVersion, + status: 'ready', + }), + }) + ); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnxxx'); + }); + + it('adopts a copied database for an existing default space', async () => { + prismaService.spaceDataDbBinding.findUnique.mockResolvedValue(null); + preflightService.preflight.mockResolvedValue({ + ok: true, + provider: 'postgres', + classification: 'empty', + capabilities, + errors: [], + }); + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never, + dataDbMigrationService as never + ); + + await service.updateBindingForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: 'adopt-existing', + internalSchema, + }); + + expect(preflightService.preflight).toHaveBeenCalledWith({ + url: dataUrl, + targetMode: 'adopt-existing', + internalSchema, + }); + expect(baselineService.initialize).toHaveBeenCalledWith(dataUrl, internalSchema); + expect(txClient.spaceDataDbBinding.upsert).toHaveBeenCalledWith({ + where: { spaceId: 'spcxxx' }, + create: { + spaceId: 'spcxxx', + dataDbConnectionId: 'dcnxxx', + mode: 'byodb', + state: 'ready', + createdBy: 'usrxxx', + }, + update: { + dataDbConnectionId: 'dcnxxx', + mode: 'byodb', + state: 'ready', + }, + }); + expect(dataDbClientManager.invalidateConnection).toHaveBeenCalledWith('dcnxxx'); + }); + + it('does not create a BYODB binding for an existing default space without adopt-existing mode', async () => { + prismaService.spaceDataDbBinding.findUnique.mockResolvedValue(null); + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never, + dataDbMigrationService as never + ); + + await expect( + service.updateBindingForSpace('spcxxx', 'usrxxx', { + url: dataUrl, + targetMode: initializeEmptyTargetMode, + internalSchema, + }) + ).rejects.toMatchObject({ code: HttpErrorCode.NOT_FOUND }); + expect(preflightService.preflight).not.toHaveBeenCalled(); + expect(txClient.spaceDataDbBinding.upsert).not.toHaveBeenCalled(); + }); + + it('rejects credential updates that would move the space to a different data DB', async () => { + const service = new DataDbBindingService( + prismaService as never, + preflightService as never, + baselineService as never, + dataDbClientManager as never, + dataDbMigrationService as never + ); + + await expect( + service.updateBindingForSpace('spcxxx', 'usrxxx', { + url: 'postgresql://teable:secret@other.example.com:5432/teable_data', + targetMode: initializeEmptyTargetMode, + internalSchema, + }) + ).rejects.toMatchObject({ code: HttpErrorCode.VALIDATION_ERROR }); + expect(baselineService.initialize).not.toHaveBeenCalled(); + }); }); diff --git a/apps/nestjs-backend/src/features/space/data-db-binding.service.ts b/apps/nestjs-backend/src/features/space/data-db-binding.service.ts index 2612f6d04c..1bfffee7a9 100644 --- a/apps/nestjs-backend/src/features/space/data-db-binding.service.ts +++ b/apps/nestjs-backend/src/features/space/data-db-binding.service.ts @@ -1,15 +1,18 @@ import { Injectable } from '@nestjs/common'; import { HttpErrorCode } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import type { ICreateSpaceRo, IDataDbPreflightVo } from '@teable/openapi'; +import type { ICreateSpaceRo, IDataDbPreflightRo, IDataDbPreflightVo } from '@teable/openapi'; import { CustomHttpException } from '../../custom.exception'; +import { DataDbClientManager } from '../../global/data-db-client-manager.service'; import { DataDbBaselineService } from './data-db-baseline.service'; +import { resolveDataDbInternalSchema } from './data-db-internal-schema'; +import { DataDbMigrationService } from './data-db-migration.service'; import { DataDbPreflightService, - fingerprintDatabaseUrl, + fingerprintDataDbConnection, getDatabaseUrlDisplayParts, } from './data-db-preflight.service'; -import { encryptDataDbUrl } from './data-db-url-secret'; +import { decryptDataDbUrl, encryptDataDbUrl } from './data-db-url-secret'; type IDataDbCreateOptions = NonNullable; type IPreparedDataDbBinding = { @@ -17,9 +20,15 @@ type IPreparedDataDbBinding = { urlFingerprint: string; displayHost: string; displayDatabase: string; + internalSchema: string; + schemaVersion: string | null; capabilities: IDataDbPreflightVo['capabilities']; }; +const initializeEmptyTargetMode = 'initialize-empty'; +const adoptExistingTargetMode = 'adopt-existing'; +const dataDbUrlRequiredError = 'Data database URL is required'; + const buildPreflightErrorMessage = (preflight: IDataDbPreflightVo) => { const errorCodes = preflight.errors.map((error) => error.code).join(', '); return errorCodes @@ -32,7 +41,9 @@ export class DataDbBindingService { constructor( private readonly prismaService: PrismaService, private readonly preflightService: DataDbPreflightService, - private readonly baselineService: DataDbBaselineService + private readonly baselineService: DataDbBaselineService, + private readonly dataDbClientManager: DataDbClientManager, + private readonly dataDbMigrationService?: DataDbMigrationService ) {} async createBindingForNewSpace( @@ -51,7 +62,7 @@ export class DataDbBindingService { return null; } - if (dataDb.targetMode && dataDb.targetMode !== 'initialize-empty') { + if (dataDb.targetMode && dataDb.targetMode !== initializeEmptyTargetMode) { throw new CustomHttpException( 'Only initialize-empty BYODB target mode is supported for new spaces', HttpErrorCode.VALIDATION_ERROR @@ -59,15 +70,166 @@ export class DataDbBindingService { } if (!dataDb.url) { + throw new CustomHttpException(dataDbUrlRequiredError, HttpErrorCode.VALIDATION_ERROR); + } + + return await this.prepareByodbBinding( + { + url: dataDb.url, + targetMode: dataDb.targetMode ?? initializeEmptyTargetMode, + internalSchema: dataDb.internalSchema, + }, + dataDb.targetMode ?? initializeEmptyTargetMode + ); + } + + async createPreparedBindingForNewSpace( + spaceId: string, + createdBy: string, + prepared: IPreparedDataDbBinding | null + ) { + if (!prepared) { + return; + } + + let connectionId: string | undefined; + await this.prismaService.$tx(async (prisma) => { + const connection = await prisma.dataDbConnection.upsert({ + where: { urlFingerprint: prepared.urlFingerprint }, + create: { + provider: 'postgres', + encryptedUrl: prepared.encryptedUrl, + urlFingerprint: prepared.urlFingerprint, + displayHost: prepared.displayHost, + displayDatabase: prepared.displayDatabase, + internalSchema: prepared.internalSchema, + status: 'ready', + schemaVersion: prepared.schemaVersion, + capabilities: prepared.capabilities, + lastValidatedAt: new Date(), + createdBy, + }, + update: { + encryptedUrl: prepared.encryptedUrl, + displayHost: prepared.displayHost, + displayDatabase: prepared.displayDatabase, + internalSchema: prepared.internalSchema, + status: 'ready', + schemaVersion: prepared.schemaVersion, + capabilities: prepared.capabilities, + lastValidatedAt: new Date(), + lastError: null, + }, + select: { id: true }, + }); + connectionId = connection.id; + + await prisma.spaceDataDbBinding.create({ + data: { + spaceId, + dataDbConnectionId: connection.id, + mode: 'byodb', + state: 'ready', + createdBy, + }, + }); + }); + if (connectionId) { + await this.dataDbClientManager.invalidateConnection(connectionId); + } + } + + async retestBinding(spaceId: string) { + const binding = await this.getByodbBinding(spaceId); + const connection = binding.dataDbConnection; + const url = decryptDataDbUrl(connection.encryptedUrl); + const preflight = await this.preflightService.preflight({ + url, + targetMode: initializeEmptyTargetMode, + internalSchema: connection.internalSchema, + }); + + await this.prismaService.$tx(async (prisma) => { + await prisma.dataDbConnection.update({ + where: { id: connection.id }, + data: { + status: preflight.ok ? 'ready' : 'error', + capabilities: preflight.capabilities, + lastValidatedAt: new Date(), + lastError: preflight.ok ? null : buildPreflightErrorMessage(preflight), + }, + }); + await prisma.spaceDataDbBinding.updateMany({ + where: { dataDbConnectionId: connection.id, mode: 'byodb' }, + data: { state: preflight.ok ? 'ready' : 'error' }, + }); + }); + + return preflight; + } + + async retryMigrationForSpace(spaceId: string) { + if (!this.dataDbMigrationService) { throw new CustomHttpException( - 'Data database URL is required', + 'Data database migration service is unavailable', + HttpErrorCode.CONFLICT + ); + } + + const binding = await this.getByodbBinding(spaceId); + const connection = binding.dataDbConnection; + const applied = await this.dataDbMigrationService.ensureConnectionMigrated({ + connectionId: connection.id, + internalSchema: connection.internalSchema, + url: decryptDataDbUrl(connection.encryptedUrl), + }); + await this.dataDbClientManager.invalidateConnection(connection.id); + return applied; + } + + async updateBindingForSpace(spaceId: string, updatedBy: string, dataDb: IDataDbPreflightRo) { + if (!dataDb.url) { + throw new CustomHttpException(dataDbUrlRequiredError, HttpErrorCode.VALIDATION_ERROR); + } + + const binding = await this.prismaService.spaceDataDbBinding.findUnique({ + where: { spaceId }, + include: { dataDbConnection: true }, + }); + if (binding?.mode !== 'byodb' || !binding.dataDbConnection?.encryptedUrl) { + if (dataDb.targetMode === adoptExistingTargetMode) { + await this.createBindingForExistingSpace(spaceId, updatedBy, dataDb); + return; + } + + throw new CustomHttpException( + 'BYODB data database binding was not found', + HttpErrorCode.NOT_FOUND + ); + } + + const current = binding.dataDbConnection; + const internalSchema = resolveDataDbInternalSchema( + dataDb.internalSchema ?? current.internalSchema, + dataDb.url + ); + const nextDisplayParts = getDatabaseUrlDisplayParts(dataDb.url); + + if ( + current.internalSchema !== internalSchema || + current.displayHost !== nextDisplayParts.displayHost || + current.displayDatabase !== nextDisplayParts.displayDatabase + ) { + throw new CustomHttpException( + 'Changing the BYODB database or internal schema is not supported yet', HttpErrorCode.VALIDATION_ERROR ); } const preflight = await this.preflightService.preflight({ url: dataDb.url, - targetMode: dataDb.targetMode ?? 'initialize-empty', + targetMode: initializeEmptyTargetMode, + internalSchema, }); if (!preflight.ok) { throw new CustomHttpException(buildPreflightErrorMessage(preflight), HttpErrorCode.CONFLICT, { @@ -75,27 +237,41 @@ export class DataDbBindingService { }); } - await this.baselineService.initialize(dataDb.url); - - const { displayHost, displayDatabase } = getDatabaseUrlDisplayParts(dataDb.url); - return { - encryptedUrl: encryptDataDbUrl(dataDb.url), - urlFingerprint: fingerprintDatabaseUrl(dataDb.url), - displayHost, - displayDatabase, - capabilities: preflight.capabilities, - }; + const schemaVersion = await this.baselineService.initialize(dataDb.url, internalSchema); + await this.prismaService.dataDbConnection.update({ + where: { id: current.id }, + data: { + encryptedUrl: encryptDataDbUrl(dataDb.url), + urlFingerprint: fingerprintDataDbConnection(dataDb.url, internalSchema), + status: 'ready', + schemaVersion, + capabilities: preflight.capabilities, + lastValidatedAt: new Date(), + lastError: null, + createdBy: current.createdBy ?? updatedBy, + }, + }); + await this.prismaService.spaceDataDbBinding.updateMany({ + where: { dataDbConnectionId: current.id, mode: 'byodb' }, + data: { state: 'ready' }, + }); + await this.dataDbClientManager.invalidateConnection(current.id); } - async createPreparedBindingForNewSpace( + async createBindingForExistingSpace( spaceId: string, createdBy: string, - prepared: IPreparedDataDbBinding | null + dataDb: IDataDbPreflightRo ) { - if (!prepared) { - return; + if (dataDb.targetMode !== adoptExistingTargetMode) { + throw new CustomHttpException( + 'Only adopt-existing BYODB target mode is supported for existing spaces', + HttpErrorCode.VALIDATION_ERROR + ); } + const prepared = await this.prepareByodbBinding(dataDb, adoptExistingTargetMode); + let connectionId: string | undefined; await this.prismaService.$tx(async (prisma) => { const connection = await prisma.dataDbConnection.upsert({ where: { urlFingerprint: prepared.urlFingerprint }, @@ -105,7 +281,9 @@ export class DataDbBindingService { urlFingerprint: prepared.urlFingerprint, displayHost: prepared.displayHost, displayDatabase: prepared.displayDatabase, + internalSchema: prepared.internalSchema, status: 'ready', + schemaVersion: prepared.schemaVersion, capabilities: prepared.capabilities, lastValidatedAt: new Date(), createdBy, @@ -114,23 +292,81 @@ export class DataDbBindingService { encryptedUrl: prepared.encryptedUrl, displayHost: prepared.displayHost, displayDatabase: prepared.displayDatabase, + internalSchema: prepared.internalSchema, status: 'ready', + schemaVersion: prepared.schemaVersion, capabilities: prepared.capabilities, lastValidatedAt: new Date(), lastError: null, }, select: { id: true }, }); + connectionId = connection.id; - await prisma.spaceDataDbBinding.create({ - data: { + await prisma.spaceDataDbBinding.upsert({ + where: { spaceId }, + create: { spaceId, dataDbConnectionId: connection.id, mode: 'byodb', state: 'ready', createdBy, }, + update: { + dataDbConnectionId: connection.id, + mode: 'byodb', + state: 'ready', + }, }); }); + if (connectionId) { + await this.dataDbClientManager.invalidateConnection(connectionId); + } + } + + private async prepareByodbBinding( + dataDb: IDataDbPreflightRo, + targetMode: IDataDbPreflightRo['targetMode'] + ): Promise { + const internalSchema = resolveDataDbInternalSchema(dataDb.internalSchema, dataDb.url); + const preflight = await this.preflightService.preflight({ + url: dataDb.url, + targetMode, + internalSchema, + }); + if (!preflight.ok) { + throw new CustomHttpException(buildPreflightErrorMessage(preflight), HttpErrorCode.CONFLICT, { + preflight, + }); + } + + const schemaVersion = await this.baselineService.initialize(dataDb.url, internalSchema); + + const { displayHost, displayDatabase } = getDatabaseUrlDisplayParts(dataDb.url); + return { + encryptedUrl: encryptDataDbUrl(dataDb.url), + urlFingerprint: fingerprintDataDbConnection(dataDb.url, internalSchema), + displayHost, + displayDatabase, + internalSchema, + schemaVersion, + capabilities: preflight.capabilities, + }; + } + + private async getByodbBinding(spaceId: string) { + const binding = await this.prismaService.spaceDataDbBinding.findUnique({ + where: { spaceId }, + include: { dataDbConnection: true }, + }); + if (binding?.mode !== 'byodb' || !binding.dataDbConnection?.encryptedUrl) { + throw new CustomHttpException( + 'BYODB data database binding was not found', + HttpErrorCode.NOT_FOUND + ); + } + return binding as typeof binding & { + dataDbConnection: NonNullable; + }; } } diff --git a/apps/nestjs-backend/src/features/space/data-db-internal-schema.ts b/apps/nestjs-backend/src/features/space/data-db-internal-schema.ts new file mode 100644 index 0000000000..412b8fc032 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/data-db-internal-schema.ts @@ -0,0 +1,34 @@ +import { createHash } from 'crypto'; + +const postgresIdentifierPattern = /^[a-z_]\w*$/i; +const byodbInternalSchemaPrefix = + process.env.BYODB_DATA_DB_INTERNAL_SCHEMA_PREFIX?.trim() || 'teable'; + +const getDataDbIdentity = (url: string) => { + const parsed = new URL(url); + const database = parsed.pathname.replace(/^\//, ''); + return `${parsed.hostname}:${parsed.port}/${database}`; +}; + +export const generateDataDbInternalSchema = (url: string) => { + const digest = createHash('sha256').update(getDataDbIdentity(url)).digest('hex').slice(0, 16); + return `${byodbInternalSchemaPrefix}_${digest}`; +}; + +export const resolveDataDbInternalSchema = (internalSchema: string | undefined, url: string) => { + const resolved = internalSchema?.trim() || generateDataDbInternalSchema(url); + if (!postgresIdentifierPattern.test(resolved)) { + throw new Error('Invalid data database internal schema name'); + } + return resolved; +}; + +export const quoteDataDbIdentifier = (identifier: string) => + `"${identifier.replaceAll('"', '""')}"`; + +export const withDataDbInternalSchemaParam = (url: string, internalSchema: string) => { + const parsed = new URL(url); + parsed.searchParams.set('schema', internalSchema); + parsed.searchParams.set('options', `-c search_path=${internalSchema}`); + return parsed.toString(); +}; diff --git a/apps/nestjs-backend/src/features/space/data-db-migration.service.spec.ts b/apps/nestjs-backend/src/features/space/data-db-migration.service.spec.ts new file mode 100644 index 0000000000..f194606a1b --- /dev/null +++ b/apps/nestjs-backend/src/features/space/data-db-migration.service.spec.ts @@ -0,0 +1,259 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { createHash } from 'crypto'; +import { describe, expect, it, vi } from 'vitest'; +import { DataDbBaselineService } from './data-db-baseline.service'; +import { DataDbMigrationService, type IDataDbMigration } from './data-db-migration.service'; +import type { IDataDbPreflightClient } from './data-db-preflight.service'; +import { encryptDataDbUrl } from './data-db-url-secret'; + +class FakeMigrationClient implements IDataDbPreflightClient { + readonly calls: Array<{ bindings?: unknown[]; sql: string }> = []; + readonly executedMigrationSql: string[] = []; + + constructor(private readonly applied = new Map()) {} + + async raw(sql: string, bindings?: unknown[]) { + this.calls.push({ sql, bindings }); + + if (sql.includes('SELECT "id", "checksum" FROM "__teable_data_schema_migrations"')) { + return { + rows: Array.from(this.applied).map(([id, checksum]) => ({ id, checksum })) as T[], + }; + } + + if (sql.includes('INSERT INTO "__teable_data_schema_migrations"')) { + this.applied.set(String(bindings?.[0]), String(bindings?.[1])); + return { rows: [] as T[] }; + } + + if (sql.includes('"fixture_table"')) { + this.executedMigrationSql.push(sql); + } + + return { rows: [] as T[] }; + } + + async destroy() { + return undefined; + } +} + +const migrations: IDataDbMigration[] = [ + { + id: '20260421000000_init_data_db_baseline', + sql: 'CREATE TABLE "fixture_table" ("id" TEXT PRIMARY KEY);', + }, + { + id: '20260513000000_add_fixture_column', + sql: 'ALTER TABLE "fixture_table" ADD COLUMN IF NOT EXISTS "name" TEXT;', + }, +]; + +const checksumSql = (sql: string) => createHash('sha256').update(sql).digest('hex'); + +describe('DataDbMigrationService', () => { + it('creates the internal schema, locks, runs pending migrations, and records them', async () => { + const client = new FakeMigrationClient(); + const service = new DataDbMigrationService(migrations, () => client); + + await expect( + service.migrate('postgresql://teable:secret@example.com:5432/data', 'teable_test') + ).resolves.toEqual(migrations.map((migration) => migration.id)); + + expect(client.calls.map((call) => call.sql)).toEqual( + expect.arrayContaining([ + 'SET statement_timeout TO 300000', + 'SET lock_timeout TO 30000', + 'CREATE SCHEMA IF NOT EXISTS "teable_test"', + 'SET search_path TO "teable_test"', + 'SELECT pg_advisory_lock(hashtext(?))', + 'SELECT pg_advisory_unlock(hashtext(?))', + ]) + ); + expect(client.executedMigrationSql).toEqual(migrations.map((migration) => migration.sql)); + expect(client.calls).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + sql: expect.stringContaining('INSERT INTO "__teable_data_schema_migrations"'), + bindings: [migrations[0].id, expect.any(String)], + }), + expect.objectContaining({ + sql: expect.stringContaining('INSERT INTO "__teable_data_schema_migrations"'), + bindings: [migrations[1].id, expect.any(String)], + }), + ]) + ); + }); + + it('skips already applied migrations', async () => { + const client = new FakeMigrationClient( + new Map([[migrations[0].id, checksumSql(migrations[0].sql)]]) + ); + const service = new DataDbMigrationService(migrations, () => client); + + await expect( + service.migrate('postgresql://teable:secret@example.com:5432/data', 'teable_test') + ).resolves.toEqual([migrations[1].id]); + + expect(client.calls).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + sql: expect.stringContaining('INSERT INTO "__teable_data_schema_migrations"'), + bindings: [migrations[0].id, expect.any(String)], + }), + ]) + ); + }); + + it('marks the connection and bindings ready after a successful ensure', async () => { + const client = new FakeMigrationClient(); + const prismaService = { + dataDbConnection: { + findUnique: vi.fn().mockResolvedValue({ status: 'ready', schemaVersion: null }), + update: vi.fn().mockResolvedValue({}), + }, + spaceDataDbBinding: { + updateMany: vi.fn().mockResolvedValue({ count: 1 }), + }, + }; + const service = new DataDbMigrationService(migrations, () => client, prismaService as never); + + await expect( + service.ensureConnectionMigrated({ + connectionId: 'dcnxxx', + internalSchema: 'teable_test', + url: 'postgresql://teable:secret@example.com:5432/data', + }) + ).resolves.toEqual(migrations.map((migration) => migration.id)); + + expect(prismaService.dataDbConnection.update).toHaveBeenCalledWith({ + where: { id: 'dcnxxx' }, + data: { status: 'migrating' }, + }); + expect(prismaService.dataDbConnection.update).toHaveBeenCalledWith({ + where: { id: 'dcnxxx' }, + data: expect.objectContaining({ + status: 'ready', + schemaVersion: migrations[1].id, + lastError: null, + }), + }); + expect(prismaService.spaceDataDbBinding.updateMany).toHaveBeenLastCalledWith({ + where: { dataDbConnectionId: 'dcnxxx', mode: 'byodb' }, + data: { state: 'ready' }, + }); + }); + + it('stores a visible error when ensure fails', async () => { + const client = new FakeMigrationClient(); + const prismaService = { + dataDbConnection: { + findUnique: vi.fn().mockResolvedValue({ status: 'ready', schemaVersion: null }), + update: vi.fn().mockResolvedValue({}), + }, + spaceDataDbBinding: { + updateMany: vi.fn().mockResolvedValue({ count: 1 }), + }, + }; + const brokenClient: IDataDbPreflightClient = { + destroy: () => client.destroy(), + raw: async (sql: string, bindings?: unknown[]) => { + if (sql === 'SELECT broken') { + throw new Error('ECONNREFUSED'); + } + return await client.raw(sql, bindings); + }, + }; + const service = new DataDbMigrationService( + [{ id: migrations[0].id, sql: 'SELECT broken' }], + () => brokenClient, + prismaService as never + ); + + await expect( + service.ensureConnectionMigrated({ + connectionId: 'dcnxxx', + internalSchema: 'teable_test', + url: 'postgresql://teable:secret@example.com:5432/data', + }) + ).rejects.toThrow('ECONNREFUSED'); + + expect(prismaService.dataDbConnection.update).toHaveBeenLastCalledWith({ + where: { id: 'dcnxxx' }, + data: { + status: 'error', + lastError: 'ECONNREFUSED', + }, + }); + expect(prismaService.spaceDataDbBinding.updateMany).toHaveBeenLastCalledWith({ + where: { dataDbConnectionId: 'dcnxxx', mode: 'byodb' }, + data: { state: 'error' }, + }); + }); + + it('scans existing connections that need schema upgrades', async () => { + const dataUrl = 'postgresql://teable:secret@example.com:5432/data'; + const client = new FakeMigrationClient(); + const prismaService = { + dataDbConnection: { + findMany: vi.fn().mockResolvedValue([ + { + id: 'dcnxxx', + encryptedUrl: encryptDataDbUrl(dataUrl), + internalSchema: 'teable_test', + }, + ]), + findUnique: vi.fn().mockResolvedValue({ status: 'ready', schemaVersion: null }), + update: vi.fn().mockResolvedValue({}), + }, + spaceDataDbBinding: { + updateMany: vi.fn().mockResolvedValue({ count: 1 }), + }, + }; + const service = new DataDbMigrationService(migrations, () => client, prismaService as never); + + await service.migrateExistingConnections(); + + expect(prismaService.dataDbConnection.findMany).toHaveBeenCalledWith({ + where: { + status: { not: 'disabled' }, + OR: [ + { schemaVersion: null }, + { schemaVersion: { not: migrations[1].id } }, + { status: { in: ['migrating', 'error'] } }, + ], + }, + select: { + id: true, + encryptedUrl: true, + internalSchema: true, + }, + }); + expect(prismaService.dataDbConnection.update).toHaveBeenCalledWith({ + where: { id: 'dcnxxx' }, + data: expect.objectContaining({ + status: 'ready', + schemaVersion: migrations[1].id, + }), + }); + }); +}); + +describe('DataDbBaselineService', () => { + it('delegates baseline initialization to the migration service', async () => { + const migrationService = { + migrate: vi.fn().mockResolvedValue([]), + getLatestSchemaVersion: vi.fn().mockReturnValue(migrations[1].id), + }; + const service = new DataDbBaselineService(migrationService as never); + + await expect( + service.initialize('postgresql://teable:secret@example.com:5432/data', 'teable_test') + ).resolves.toBe(migrations[1].id); + + expect(migrationService.migrate).toHaveBeenCalledWith( + 'postgresql://teable:secret@example.com:5432/data', + 'teable_test' + ); + }); +}); diff --git a/apps/nestjs-backend/src/features/space/data-db-migration.service.ts b/apps/nestjs-backend/src/features/space/data-db-migration.service.ts new file mode 100644 index 0000000000..d38e22dcf2 --- /dev/null +++ b/apps/nestjs-backend/src/features/space/data-db-migration.service.ts @@ -0,0 +1,317 @@ +import { createHash } from 'crypto'; +import { existsSync, readdirSync, readFileSync } from 'fs'; +import { join } from 'path'; +import { Inject, Injectable, Logger, Optional } from '@nestjs/common'; +import type { OnApplicationBootstrap } from '@nestjs/common'; +import { PrismaService, type DataDbConnection } from '@teable/db-main-prisma'; +import { quoteDataDbIdentifier, resolveDataDbInternalSchema } from './data-db-internal-schema'; +import { + DATA_DB_PREFLIGHT_CLIENT_FACTORY, + dataDbKnexClientFactory, + type IDataDbPreflightClient, + type IDataDbPreflightClientFactory, +} from './data-db-preflight.service'; +import { decryptDataDbUrl } from './data-db-url-secret'; + +export interface IDataDbMigration { + id: string; + sql: string; +} + +export const dataDbMigrationsToken = Symbol('DATA_DB_MIGRATIONS'); + +export const DATA_DB_MIGRATION_TABLE = '__teable_data_schema_migrations'; + +const migrationsRootCandidates = [ + join(process.cwd(), 'community/packages/db-data-prisma/prisma/migrations'), + join(process.cwd(), '../../community/packages/db-data-prisma/prisma/migrations'), +]; +const defaultMigrationStatementTimeoutMs = 300_000; +const defaultMigrationLockTimeoutMs = 30_000; + +const readPositiveIntEnv = (key: string, fallback: number) => { + const value = Number(process.env[key]); + return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback; +}; + +const getMigrationsRoot = () => { + const found = migrationsRootCandidates.find((path) => existsSync(path)); + if (!found) { + throw new Error('Data DB migrations directory not found'); + } + return found; +}; + +const readDataDbMigrations = (): IDataDbMigration[] => + readdirSync(getMigrationsRoot(), { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => { + const sqlPath = join(getMigrationsRoot(), entry.name, 'migration.sql'); + return existsSync(sqlPath) + ? { + id: entry.name, + sql: readFileSync(sqlPath, 'utf8'), + } + : null; + }) + .filter((migration): migration is IDataDbMigration => Boolean(migration?.sql.trim())) + .sort((left, right) => left.id.localeCompare(right.id)); + +const checksumSql = (sql: string) => createHash('sha256').update(sql).digest('hex'); + +@Injectable() +export class DataDbMigrationService implements OnApplicationBootstrap { + private readonly logger = new Logger(DataDbMigrationService.name); + private readonly clientFactory: IDataDbPreflightClientFactory; + private readonly runningConnections = new Map>(); + + constructor( + @Optional() + @Inject(dataDbMigrationsToken) + private readonly migrations?: IDataDbMigration[], + @Optional() + @Inject(DATA_DB_PREFLIGHT_CLIENT_FACTORY) + clientFactory?: IDataDbPreflightClientFactory, + @Optional() + private readonly prismaService?: PrismaService + ) { + this.clientFactory = clientFactory ?? dataDbKnexClientFactory; + } + + async onApplicationBootstrap() { + if (!this.prismaService) { + return; + } + + void this.migrateExistingConnections().catch((error) => { + this.logger.error(`Failed to scan data database migrations: ${formatMigrationError(error)}`); + }); + } + + async migrate(url: string, internalSchema?: string) { + const migrations = this.resolveMigrations(); + if (migrations.length === 0) { + return []; + } + + const client = this.clientFactory(url); + const resolvedInternalSchema = resolveDataDbInternalSchema(internalSchema, url); + const quotedInternalSchema = quoteDataDbIdentifier(resolvedInternalSchema); + const lockKey = `teable:data-db-migration:${resolvedInternalSchema}`; + + try { + await client.raw( + `SET statement_timeout TO ${readPositiveIntEnv( + 'BYODB_DATA_DB_MIGRATION_STATEMENT_TIMEOUT_MS', + defaultMigrationStatementTimeoutMs + )}` + ); + await client.raw( + `SET lock_timeout TO ${readPositiveIntEnv( + 'BYODB_DATA_DB_MIGRATION_LOCK_TIMEOUT_MS', + defaultMigrationLockTimeoutMs + )}` + ); + await client.raw(`CREATE SCHEMA IF NOT EXISTS ${quotedInternalSchema}`); + await client.raw(`SET search_path TO ${quotedInternalSchema}`); + await client.raw('SELECT pg_advisory_lock(hashtext(?))', [lockKey]); + + try { + await this.ensureMigrationTable(client); + const applied = await this.getAppliedMigrations(client); + const appliedNow: string[] = []; + + for (const migration of migrations) { + const checksum = checksumSql(migration.sql); + const appliedChecksum = applied.get(migration.id); + if (appliedChecksum) { + if (appliedChecksum !== checksum) { + throw new Error(`Data DB migration ${migration.id} checksum mismatch`); + } + continue; + } + await client.raw(migration.sql); + await this.recordMigration(client, migration, checksum); + appliedNow.push(migration.id); + } + + return appliedNow; + } finally { + await client + .raw('SELECT pg_advisory_unlock(hashtext(?))', [lockKey]) + .catch(() => undefined); + } + } finally { + await client.destroy().catch(() => undefined); + } + } + + async ensureConnectionMigrated(input: { + connectionId: string; + internalSchema: string; + url: string; + }) { + const existing = this.runningConnections.get(input.connectionId); + if (existing) { + return await existing; + } + + const promise = this.migrateConnection(input).finally(() => { + this.runningConnections.delete(input.connectionId); + }); + this.runningConnections.set(input.connectionId, promise); + return await promise; + } + + async migrateExistingConnections() { + if (!this.prismaService) { + return; + } + + const latestSchemaVersion = this.getLatestSchemaVersion(); + if (!latestSchemaVersion) { + return; + } + + const connections = await this.prismaService.dataDbConnection.findMany({ + where: { + status: { + not: 'disabled', + }, + OR: [ + { schemaVersion: null }, + { schemaVersion: { not: latestSchemaVersion } }, + { status: { in: ['migrating', 'error'] } }, + ], + }, + select: { + id: true, + encryptedUrl: true, + internalSchema: true, + }, + }); + + for (const connection of connections) { + await this.ensureConnectionMigrated({ + connectionId: connection.id, + internalSchema: connection.internalSchema, + url: decryptDataDbUrl(connection.encryptedUrl), + }).catch((error) => { + this.logger.warn( + `Failed to migrate data database connection ${connection.id}: ${formatMigrationError(error)}` + ); + }); + } + } + + private async migrateConnection(input: { + connectionId: string; + internalSchema: string; + url: string; + }) { + const latestSchemaVersion = this.getLatestSchemaVersion(); + if (!latestSchemaVersion) { + return []; + } + + const current = await this.prismaService?.dataDbConnection.findUnique({ + where: { id: input.connectionId }, + select: { status: true, schemaVersion: true }, + }); + + if (current?.status === 'ready' && current.schemaVersion === latestSchemaVersion) { + return []; + } + + await this.updateConnectionState(input.connectionId, 'migrating'); + + try { + const applied = await this.migrate(input.url, input.internalSchema); + await this.prismaService?.dataDbConnection.update({ + where: { id: input.connectionId }, + data: { + status: 'ready', + schemaVersion: latestSchemaVersion, + lastValidatedAt: new Date(), + lastError: null, + }, + }); + await this.prismaService?.spaceDataDbBinding.updateMany({ + where: { dataDbConnectionId: input.connectionId, mode: 'byodb' }, + data: { state: 'ready' }, + }); + return applied; + } catch (error) { + const message = formatMigrationError(error); + await this.prismaService?.dataDbConnection.update({ + where: { id: input.connectionId }, + data: { + status: 'error', + lastError: message, + }, + }); + await this.prismaService?.spaceDataDbBinding.updateMany({ + where: { dataDbConnectionId: input.connectionId, mode: 'byodb' }, + data: { state: 'error' }, + }); + throw error; + } + } + + private async updateConnectionState(connectionId: DataDbConnection['id'], state: 'migrating') { + await this.prismaService?.dataDbConnection.update({ + where: { id: connectionId }, + data: { status: state }, + }); + await this.prismaService?.spaceDataDbBinding.updateMany({ + where: { dataDbConnectionId: connectionId, mode: 'byodb' }, + data: { state }, + }); + } + + private resolveMigrations() { + return this.migrations ?? readDataDbMigrations(); + } + + getLatestSchemaVersion() { + return this.resolveMigrations().at(-1)?.id ?? null; + } + + private async ensureMigrationTable(client: IDataDbPreflightClient) { + await client.raw(` + CREATE TABLE IF NOT EXISTS "${DATA_DB_MIGRATION_TABLE}" ( + "id" TEXT PRIMARY KEY, + "checksum" TEXT NOT NULL, + "applied_at" TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `); + } + + private async getAppliedMigrations(client: IDataDbPreflightClient) { + const result = await client.raw<{ id: string; checksum: string }>( + `SELECT "id", "checksum" FROM "${DATA_DB_MIGRATION_TABLE}" ORDER BY "id"` + ); + const rows = Array.isArray(result) ? result : result.rows ?? []; + return new Map(rows.map((row) => [row.id, row.checksum])); + } + + private async recordMigration( + client: IDataDbPreflightClient, + migration: IDataDbMigration, + checksum: string + ) { + await client.raw( + ` + INSERT INTO "${DATA_DB_MIGRATION_TABLE}" ("id", "checksum") + VALUES (?, ?) + ON CONFLICT ("id") DO NOTHING + `, + [migration.id, checksum] + ); + } +} + +const formatMigrationError = (error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + return message.length > 2000 ? `${message.slice(0, 1997)}...` : message; +}; diff --git a/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts b/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts index 79ab4f9b74..8e9941f77a 100644 --- a/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts +++ b/apps/nestjs-backend/src/features/space/data-db-preflight.service.spec.ts @@ -1,3 +1,6 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable sonarjs/no-duplicate-string */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { IDataDbPreflightClient } from './data-db-preflight.service'; import { @@ -10,9 +13,11 @@ type IFakeDbState = { schemas?: string[]; tables?: Array<{ table_schema: string; table_name: string }>; functions?: string[]; + databases?: string[]; failCreateSchema?: boolean; failConnect?: boolean; failMessage?: string; + failCode?: string; }; class FakePreflightClient implements IDataDbPreflightClient { @@ -20,7 +25,9 @@ class FakePreflightClient implements IDataDbPreflightClient { async raw(sql: string): Promise<{ rows: T[] }> { if (this.state.failConnect) { - throw new Error(this.state.failMessage ?? 'connection failed'); + const error = new Error(this.state.failMessage ?? 'connection failed'); + Object.assign(error, { code: this.state.failCode }); + throw error; } if (sql.includes('CREATE SCHEMA') && this.state.failCreateSchema) { throw new Error('permission denied for database'); @@ -37,6 +44,13 @@ class FakePreflightClient implements IDataDbPreflightClient { if (sql.includes('pg_stat_activity')) { return { rows: [{ count: '0' }] as T[] }; } + if (sql.includes('pg_database')) { + return { + rows: (this.state.databases ?? ['postgres', 'teable_data']).map((datname) => ({ + datname, + })) as T[], + }; + } if (sql.includes('information_schema.schemata')) { return { rows: (this.state.schemas ?? ['public']).map((schema_name) => ({ schema_name })) as T[], @@ -59,6 +73,7 @@ class FakePreflightClient implements IDataDbPreflightClient { } const DATA_URL = 'postgresql://teable:secret@example.com:5432/teable_data'; +const internalSchema = 'teable_meta_test'; const BASELINE_TABLES = [ 'computed_update_outbox', 'computed_update_outbox_seed', @@ -69,6 +84,7 @@ const BASELINE_TABLES = [ 'record_trash', '__undo_log', ]; +const DATA_SCHEMA_MIGRATION_TABLE = '__teable_data_schema_migrations'; const createService = (state: IFakeDbState) => new DataDbPreflightService(undefined, () => new FakePreflightClient(state)); @@ -113,26 +129,50 @@ describe('DataDbPreflightService', () => { it('classifies a compatible Teable data database', async () => { const result = await createService({ - schemas: ['public', 'bseabc'], - tables: BASELINE_TABLES.map((table_name) => ({ table_schema: 'public', table_name })), + schemas: ['public', internalSchema, 'bseabc'], + tables: [...BASELINE_TABLES, DATA_SCHEMA_MIGRATION_TABLE].map((table_name) => ({ + table_schema: internalSchema, + table_name, + })), functions: ['__teable_capture_undo_row'], }).preflight({ url: DATA_URL, targetMode: 'adopt-existing', + internalSchema, }); - expect(result.ok).toBe(false); + expect(result.ok).toBe(true); + expect(result.classification).toBe('teable-managed-compatible'); + expect(result.errors).toEqual([]); + }); + + it('allows the internal data schema migration history table in Teable-managed schemas', async () => { + const result = await createService({ + schemas: ['public', internalSchema], + tables: [...BASELINE_TABLES, DATA_SCHEMA_MIGRATION_TABLE].map((table_name) => ({ + table_schema: internalSchema, + table_name, + })), + functions: ['__teable_capture_undo_row'], + }).preflight({ + url: DATA_URL, + targetMode: 'initialize-empty', + internalSchema, + }); + + expect(result.ok).toBe(true); expect(result.classification).toBe('teable-managed-compatible'); expect(result.errors).toEqual([]); }); it('rejects a partial Teable data database as incompatible', async () => { const result = await createService({ - schemas: ['public'], - tables: [{ table_schema: 'public', table_name: 'record_history' }], + schemas: ['public', internalSchema], + tables: [{ table_schema: internalSchema, table_name: 'record_history' }], }).preflight({ url: DATA_URL, targetMode: 'initialize-empty', + internalSchema, }); expect(result.ok).toBe(false); @@ -140,7 +180,7 @@ describe('DataDbPreflightService', () => { expect(result.errors.map((error) => error.code)).toContain('INCOMPATIBLE_TEABLE_DATABASE'); }); - it('rejects non-empty unknown databases', async () => { + it('allows non-empty public schemas because BYODB uses Teable internal schemas', async () => { const result = await createService({ schemas: ['public'], tables: [{ table_schema: 'public', table_name: 'customer_table' }], @@ -149,6 +189,39 @@ describe('DataDbPreflightService', () => { targetMode: 'initialize-empty', }); + expect(result.ok).toBe(true); + expect(result.classification).toBe('empty'); + expect(result.errors).toEqual([]); + }); + + it('allows other base schemas while initializing an empty internal schema', async () => { + const result = await createService({ + schemas: ['public', internalSchema, 'bse_existing_base'], + tables: [ + { table_schema: 'bse_existing_base', table_name: 'sheet_table' }, + { table_schema: 'public', table_name: 'customer_table' }, + ], + }).preflight({ + url: DATA_URL, + targetMode: 'initialize-empty', + internalSchema, + }); + + expect(result.ok).toBe(true); + expect(result.classification).toBe('empty'); + expect(result.errors).toEqual([]); + }); + + it('rejects unknown objects inside the Teable internal schema', async () => { + const result = await createService({ + schemas: ['public', internalSchema], + tables: [{ table_schema: internalSchema, table_name: 'customer_table' }], + }).preflight({ + url: DATA_URL, + targetMode: 'initialize-empty', + internalSchema, + }); + expect(result.ok).toBe(false); expect(result.classification).toBe('non-empty-unknown'); expect(result.errors.map((error) => error.code)).toContain('NON_EMPTY_UNKNOWN_DATABASE'); @@ -181,6 +254,74 @@ describe('DataDbPreflightService', () => { expect(result.errors.map((error) => error.code)).toContain('CONNECTION_FAILED'); }); + it('returns a specific error when an IPv6 address is unreachable', async () => { + const result = await createService({ + failConnect: true, + failCode: 'ENETUNREACH', + failMessage: 'connect ENETUNREACH 2406:da1c:4c7:f800:9d0b:f8d1:c668:930:5432', + }).preflight({ + url: DATA_URL, + targetMode: 'initialize-empty', + }); + + expect(result.errors.map((error) => error.code)).toContain('IPV6_NETWORK_UNREACHABLE'); + expect(result.errors[0]?.remediation).toContain('IPv4-reachable'); + }); + + it('returns database choices when the URL database does not exist', async () => { + const requestedUrl = 'postgresql://teable:secret@example.com:5432/missing_db'; + const clients: Record = { + [requestedUrl]: { + failConnect: true, + failCode: '3D000', + failMessage: 'database "missing_db" does not exist', + }, + ['postgresql://teable:secret@example.com:5432/postgres']: { + databases: ['postgres', 'teable_data'], + }, + }; + const service = new DataDbPreflightService( + undefined, + (url) => new FakePreflightClient(clients[url] ?? {}) + ); + + const result = await service.preflight({ + url: requestedUrl, + targetMode: 'initialize-empty', + }); + + expect(result.ok).toBe(false); + expect(result.displayDatabase).toBe('missing_db'); + expect(result.serverVersion).toBe('14.12'); + expect(result.availableDatabases).toEqual(['postgres', 'teable_data']); + expect(result.errors.map((error) => error.code)).toContain('CONNECTION_FAILED'); + expect(JSON.stringify(result)).not.toContain('secret'); + }); + + it('returns database choices when the URL omits the database name', async () => { + const requestedUrl = 'postgresql://teable:secret@example.com:5432'; + const clients: Record = { + ['postgresql://teable:secret@example.com:5432/postgres']: { + databases: ['postgres', 'teable_data'], + }, + }; + const service = new DataDbPreflightService( + undefined, + (url) => new FakePreflightClient(clients[url] ?? {}) + ); + + const result = await service.preflight({ + url: requestedUrl, + targetMode: 'initialize-empty', + }); + + expect(result.ok).toBe(false); + expect(result.displayDatabase).toBe(''); + expect(result.availableDatabases).toEqual(['postgres', 'teable_data']); + expect(result.requiresDatabaseSelection).toBe(true); + expect(result.errors).toEqual([]); + }); + it('rejects unsupported database URL drivers', async () => { const result = await createService({}).preflight({ url: 'mysql://teable:secret@example.com:3306/teable_data', @@ -215,6 +356,8 @@ describe('DataDbPreflightService', () => { provider: 'postgres', displayHost: 'example.com:5432', displayDatabase: 'teable_data', + internalSchema, + schemaVersion: '20260421000000_init_data_db_baseline', lastValidatedAt: new Date('2026-05-06T00:00:00.000Z'), lastError: null, encryptedUrl: 'encrypted-secret', @@ -240,6 +383,8 @@ describe('DataDbPreflightService', () => { provider: 'postgres', displayHost: 'example.com:5432', displayDatabase: 'teable_data', + internalSchema, + schemaVersion: '20260421000000_init_data_db_baseline', lastValidatedAt: '2026-05-06T00:00:00.000Z', }); expect(JSON.stringify(summary)).not.toContain('encrypted-secret'); diff --git a/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts b/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts index 9f8d7d7e7d..195f6dc361 100644 --- a/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts +++ b/apps/nestjs-backend/src/features/space/data-db-preflight.service.ts @@ -1,16 +1,19 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +import { createHash } from 'crypto'; +import { promises as dns } from 'dns'; +import { isIP } from 'net'; import { Inject, Injectable, Optional } from '@nestjs/common'; import { parseDsn } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; import type { IDataDbConnectionSummaryVo, IDataDbPreflightRo, IDataDbPreflightVo, } from '@teable/openapi'; -import { PrismaService } from '@teable/db-main-prisma'; -import { createHash } from 'crypto'; -import { promises as dns } from 'dns'; -import { isIP } from 'net'; import type { Knex } from 'knex'; import createKnex from 'knex'; +import { resolveDataDbInternalSchema } from './data-db-internal-schema'; type IPreflightCapabilities = IDataDbPreflightVo['capabilities']; type IPreflightClassification = IDataDbPreflightVo['classification']; @@ -35,8 +38,14 @@ const DATA_PLANE_TABLES = [ '__undo_log', ]; +const DATA_SCHEMA_MIGRATION_TABLE = '__teable_data_schema_migrations'; const DATA_PLANE_FUNCTIONS = ['__teable_capture_undo_row']; -const ALLOWED_PUBLIC_TABLES = new Set([...DATA_PLANE_TABLES, '_prisma_migrations']); +const ALLOWED_INTERNAL_TABLES = new Set([ + ...DATA_PLANE_TABLES, + DATA_SCHEMA_MIGRATION_TABLE, + '_prisma_migrations', +]); +const MAINTENANCE_DATABASE_CANDIDATES = ['postgres', 'template1']; const DEFAULT_CAPABILITIES: IPreflightCapabilities = { createSchema: false, createTable: false, @@ -53,6 +62,14 @@ const PRIVATE_NETWORK_ERROR: IPreflightError = { remediation: 'Set TEABLE_SSRF_PROTECTION_DISABLED=true only in trusted self-hosted deployments.', }; +const IPV6_NETWORK_UNREACHABLE_ERROR: IPreflightError = { + code: 'IPV6_NETWORK_UNREACHABLE', + message: + 'The database host resolved to an IPv6 address, but this Teable deployment cannot reach IPv6 networks.', + remediation: + 'Use an IPv4-reachable database endpoint or connection pooler, or enable IPv6 outbound networking for this Teable deployment.', +}; + const normalizeRawRows = (result: { rows?: T[] } | T[]): T[] => { if (Array.isArray(result)) { return result; @@ -72,14 +89,24 @@ export const fingerprintDatabaseUrl = (url: string): string => { return `dbfp_${createHash('sha256').update(url).digest('hex')}`; }; +export const fingerprintDataDbConnection = (url: string, internalSchema: string): string => { + return `dbfp_${createHash('sha256').update(`${url}\n${internalSchema}`).digest('hex')}`; +}; + export const getDatabaseUrlDisplayParts = (url: string) => { const parsed = new URL(url); return { displayHost: parsed.port ? `${parsed.hostname}:${parsed.port}` : parsed.hostname, - displayDatabase: parsed.pathname.replace(/^\//, ''), + displayDatabase: decodeURIComponent(parsed.pathname.replace(/^\//, '')), }; }; +export const replaceDatabaseUrlDatabase = (url: string, database: string): string => { + const parsed = new URL(url); + parsed.pathname = `/${encodeURIComponent(database)}`; + return parsed.toString(); +}; + const isPrivateNetworkAllowed = () => process.env.TEABLE_SSRF_PROTECTION_DISABLED === 'true'; const isPrivateIp = (address: string): boolean => { @@ -130,6 +157,7 @@ export class DataDbPreflightService { async preflight(input: IDataDbPreflightRo): Promise { const errors: IPreflightError[] = []; + let internalSchema = ''; let maskedUrl: string | undefined; let urlFingerprint: string | undefined; let displayHost: string | undefined; @@ -137,8 +165,9 @@ export class DataDbPreflightService { try { parseDsn(input.url); + internalSchema = resolveDataDbInternalSchema(input.internalSchema, input.url); maskedUrl = maskDatabaseUrl(input.url); - urlFingerprint = fingerprintDatabaseUrl(input.url); + urlFingerprint = fingerprintDataDbConnection(input.url, internalSchema); const displayParts = getDatabaseUrlDisplayParts(input.url); displayHost = displayParts.displayHost; displayDatabase = displayParts.displayDatabase; @@ -154,6 +183,7 @@ export class DataDbPreflightService { }, ], classification: 'non-empty-unknown', + internalSchema, }); } @@ -167,14 +197,32 @@ export class DataDbPreflightService { displayHost, displayDatabase, classification: 'non-empty-unknown', + internalSchema, + }); + } + + if (!displayDatabase) { + const databaseResult = await this.inspectServerDatabases(input.url); + return this.buildResult({ + errors, + maskedUrl, + urlFingerprint, + displayHost, + displayDatabase, + serverVersion: databaseResult?.serverVersion, + availableDatabases: databaseResult?.availableDatabases, + requiresDatabaseSelection: true, + classification: 'non-empty-unknown', + internalSchema, }); } const client = this.clientFactory(input.url); try { const serverVersion = await this.getServerVersion(client); + const availableDatabases = await this.listAvailableDatabases(client).catch(() => undefined); const capabilities = await this.detectCapabilities(client, errors); - const classification = await this.classifyTarget(client, errors); + const classification = await this.classifyTarget(client, errors, internalSchema); return this.buildResult({ errors, @@ -183,14 +231,23 @@ export class DataDbPreflightService { displayHost, displayDatabase, serverVersion, + availableDatabases, capabilities, classification, + internalSchema, }); } catch (error) { + const missingDatabaseResult = this.isMissingDatabaseError(error) + ? await this.inspectServerDatabases(input.url) + : undefined; errors.push({ - code: 'CONNECTION_FAILED', - message: this.sanitizeErrorMessage(error, input.url), - remediation: 'Verify host, port, database name, credentials, and SSL settings.', + ...(this.isIpv6NetworkUnreachableError(error) + ? IPV6_NETWORK_UNREACHABLE_ERROR + : { + code: 'CONNECTION_FAILED', + message: this.sanitizeErrorMessage(error, input.url), + remediation: 'Verify host, port, database name, credentials, and SSL settings.', + }), }); return this.buildResult({ errors, @@ -198,7 +255,10 @@ export class DataDbPreflightService { urlFingerprint, displayHost, displayDatabase, + serverVersion: missingDatabaseResult?.serverVersion, + availableDatabases: missingDatabaseResult?.availableDatabases, classification: 'non-empty-unknown', + internalSchema, }); } finally { await client.destroy().catch(() => undefined); @@ -218,6 +278,8 @@ export class DataDbPreflightService { provider: binding.dataDbConnection.provider, displayHost: binding.dataDbConnection.displayHost ?? undefined, displayDatabase: binding.dataDbConnection.displayDatabase ?? undefined, + internalSchema: binding.dataDbConnection.internalSchema ?? undefined, + schemaVersion: binding.dataDbConnection.schemaVersion ?? null, lastValidatedAt: binding.dataDbConnection.lastValidatedAt?.toISOString(), lastError: binding.dataDbConnection.lastError ?? undefined, capabilities: binding.dataDbConnection.capabilities as @@ -238,28 +300,39 @@ export class DataDbPreflightService { urlFingerprint, displayHost, displayDatabase, + internalSchema, serverVersion, capabilities = DEFAULT_CAPABILITIES, classification, + availableDatabases, + requiresDatabaseSelection, }: { errors: IPreflightError[]; maskedUrl?: string; urlFingerprint?: string; displayHost?: string; displayDatabase?: string; + internalSchema?: string; serverVersion?: string; + availableDatabases?: string[]; + requiresDatabaseSelection?: boolean; capabilities?: IPreflightCapabilities; classification: IPreflightClassification; }): IDataDbPreflightVo { + const isUsableForNewSpace = + classification === 'empty' || classification === 'teable-managed-compatible'; return { - ok: errors.length === 0 && classification === 'empty', + ok: errors.length === 0 && isUsableForNewSpace, provider: 'postgres', maskedUrl, urlFingerprint, displayHost, displayDatabase, + internalSchema, serverVersion, classification, + availableDatabases, + requiresDatabaseSelection, capabilities, errors, }; @@ -271,6 +344,16 @@ export class DataDbPreflightService { return withoutRawUrl.replace(/:[^:@/]+@/g, ':***@'); } + private isIpv6NetworkUnreachableError(error: unknown) { + const code = + error && typeof error === 'object' ? (error as { code?: unknown }).code : undefined; + const message = error instanceof Error ? error.message : String(error); + return ( + (code === 'ENETUNREACH' || message.includes('ENETUNREACH')) && + /\b(?:[a-f0-9]{1,4}:){2,}[a-f0-9]{1,4}\b/i.test(message) + ); + } + private async validateNetwork(url: string): Promise { if (isPrivateNetworkAllowed()) { return null; @@ -294,6 +377,53 @@ export class DataDbPreflightService { return rows[0]?.server_version; } + private async listAvailableDatabases(client: IDataDbPreflightClient) { + const rows = normalizeRawRows<{ datname: string }>( + await client.raw(` + SELECT datname + FROM pg_database + WHERE datallowconn = true + AND datistemplate = false + ORDER BY datname ASC + `) + ); + return rows.map((row) => row.datname).filter(Boolean); + } + + private isMissingDatabaseError(error: unknown) { + if (typeof error === 'object' && error !== null && 'code' in error && error.code === '3D000') { + return true; + } + + const message = error instanceof Error ? error.message : String(error); + return /database ".+" does not exist/i.test(message); + } + + private getMaintenanceDatabaseUrls(url: string) { + const parsed = new URL(url); + const requestedDatabase = decodeURIComponent(parsed.pathname.replace(/^\//, '')); + const username = parsed.username ? decodeURIComponent(parsed.username) : undefined; + const candidates = [...MAINTENANCE_DATABASE_CANDIDATES, username].filter( + (database): database is string => Boolean(database && database !== requestedDatabase) + ); + return [...new Set(candidates)].map((database) => replaceDatabaseUrlDatabase(url, database)); + } + + private async inspectServerDatabases(url: string) { + for (const maintenanceUrl of this.getMaintenanceDatabaseUrls(url)) { + const client = this.clientFactory(maintenanceUrl); + try { + const serverVersion = await this.getServerVersion(client).catch(() => undefined); + const availableDatabases = await this.listAvailableDatabases(client).catch(() => []); + return { serverVersion, availableDatabases }; + } catch { + // Try the next well-known maintenance database. + } finally { + await client.destroy().catch(() => undefined); + } + } + } + private async detectCapabilities( client: IDataDbPreflightClient, errors: IPreflightError[] @@ -373,15 +503,10 @@ export class DataDbPreflightService { private async classifyTarget( client: IDataDbPreflightClient, - errors: IPreflightError[] + errors: IPreflightError[], + internalSchema: string ): Promise { - const [schemaRows, tableRows, functionRows] = await Promise.all([ - client.raw<{ schema_name: string }>(` - SELECT schema_name - FROM information_schema.schemata - WHERE schema_name NOT IN ('information_schema', 'pg_catalog', 'pg_toast') - AND schema_name NOT LIKE 'pg_%' - `), + const [tableRows, functionRows] = await Promise.all([ client.raw<{ table_schema: string; table_name: string }>(` SELECT table_schema, table_name FROM information_schema.tables @@ -392,40 +517,34 @@ export class DataDbPreflightService { client.raw<{ routine_name: string }>(` SELECT routine_name FROM information_schema.routines - WHERE routine_schema = 'public' + WHERE routine_schema = '${internalSchema}' `), ]); - const schemas = normalizeRawRows(schemaRows).map((row) => row.schema_name); const tables = normalizeRawRows(tableRows); const functions = normalizeRawRows(functionRows).map((row) => row.routine_name); - const bseSchemas = schemas.filter((schema) => schema.startsWith('bse')); - const publicTables = tables - .filter((table) => table.table_schema === 'public') + const internalTables = tables + .filter((table) => table.table_schema === internalSchema) .map((table) => table.table_name); - const unknownTables = tables.filter((table) => { - if (table.table_schema.startsWith('bse')) { - return false; - } - return table.table_schema !== 'public' || !ALLOWED_PUBLIC_TABLES.has(table.table_name); - }); - const managedTables = publicTables.filter((table) => DATA_PLANE_TABLES.includes(table)); + const unknownInternalTables = internalTables.filter( + (table) => !ALLOWED_INTERNAL_TABLES.has(table) + ); + const managedTables = internalTables.filter((table) => DATA_PLANE_TABLES.includes(table)); const managedFunctions = functions.filter((name) => DATA_PLANE_FUNCTIONS.includes(name)); - const hasManagedObjects = - bseSchemas.length > 0 || managedTables.length > 0 || managedFunctions.length > 0; + const hasManagedObjects = managedTables.length > 0 || managedFunctions.length > 0; const hasAllBaselineObjects = DATA_PLANE_TABLES.every((table) => managedTables.includes(table)) && DATA_PLANE_FUNCTIONS.every((func) => managedFunctions.includes(func)); - if (!hasManagedObjects && unknownTables.length === 0) { + if (!hasManagedObjects && unknownInternalTables.length === 0) { return 'empty'; } - if (unknownTables.length > 0) { + if (unknownInternalTables.length > 0) { errors.push({ code: 'NON_EMPTY_UNKNOWN_DATABASE', - message: 'The target database contains objects that are not managed by Teable', - remediation: 'Use an empty database or run a dedicated migration/adopt flow.', + message: `The ${internalSchema} schema already contains objects outside Teable management`, + remediation: `Use a database without a conflicting ${internalSchema} schema, or remove the unknown objects from that schema.`, }); return 'non-empty-unknown'; } diff --git a/apps/nestjs-backend/src/features/space/space.service.ts b/apps/nestjs-backend/src/features/space/space.service.ts index 8c6e43626e..f981f7f4cf 100644 --- a/apps/nestjs-backend/src/features/space/space.service.ts +++ b/apps/nestjs-backend/src/features/space/space.service.ts @@ -63,7 +63,14 @@ export class SpaceService { return false; } - private async createSpaceByParams(spaceCreateInput: Prisma.SpaceCreateInput) { + protected createOrganizationSpace( + _space: { id: string; name: string }, + _spaceCreateInput: Prisma.SpaceCreateInput + ): Promise { + return Promise.resolve(); + } + + protected async createSpaceByParams(spaceCreateInput: Prisma.SpaceCreateInput) { return await this.prismaService.$tx(async (prisma) => { const result = await prisma.space.create({ select: { @@ -72,6 +79,7 @@ export class SpaceService { }, data: spaceCreateInput, }); + await this.createOrganizationSpace(result, spaceCreateInput); await this.collaboratorService.createSpaceCollaborator({ collaborators: [ { diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.spec.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.spec.ts index 45ad79f0a5..0f92bcb018 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.spec.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.spec.ts @@ -5,11 +5,13 @@ const { executeCreateTableEndpoint, executeDeleteTableEndpoint, executeDuplicateTableEndpoint, + executeListTableRecordsEndpoint, executeRestoreTableEndpoint, } = vi.hoisted(() => ({ executeCreateTableEndpoint: vi.fn(), executeDeleteTableEndpoint: vi.fn(), executeDuplicateTableEndpoint: vi.fn(), + executeListTableRecordsEndpoint: vi.fn(), executeRestoreTableEndpoint: vi.fn(), })); @@ -17,6 +19,7 @@ vi.mock('@teable/v2-contract-http-implementation/handlers', () => ({ executeCreateTableEndpoint, executeDeleteTableEndpoint, executeDuplicateTableEndpoint, + executeListTableRecordsEndpoint, executeRestoreTableEndpoint, })); @@ -55,12 +58,14 @@ describe('TableOpenApiV2Service.createTable', () => { tableService?: Record; fieldOpenApiService?: Record; viewService?: Record; - recordService?: Record; prismaService?: Record; dbProvider?: Record; }) => new TableOpenApiV2Service( { + getContainerForBase: vi.fn().mockResolvedValue({ + resolve: vi.fn().mockReturnValue({}), + }), getContainer: vi.fn().mockResolvedValue({ resolve: vi.fn().mockReturnValue({}), }), @@ -71,14 +76,14 @@ describe('TableOpenApiV2Service.createTable', () => { (overrides?.tableService ?? {}) as never, (overrides?.fieldOpenApiService ?? {}) as never, (overrides?.viewService ?? {}) as never, - (overrides?.recordService ?? {}) as never, (overrides?.prismaService ?? {}) as never, { generateDbTableName: vi .fn() .mockImplementation((baseId: string, name: string) => `${baseId}.${name}`), ...overrides?.dbProvider, - } as never + } as never, + {} as never ); it('fills missing legacy link lookupFieldId and prefixes legacy dbTableName before calling v2', async () => { @@ -164,6 +169,33 @@ describe('TableOpenApiV2Service.createTable', () => { }); const recordIds = Array.from({ length: 1001 }, (_, index) => `rec${index + 1}`); + executeListTableRecordsEndpoint + .mockResolvedValueOnce({ + status: 200, + body: { + ok: true, + data: { + records: recordIds.slice(0, 1000).map((recordId) => ({ + id: recordId, + fields: {}, + })), + pagination: { hasMore: true }, + }, + }, + }) + .mockResolvedValueOnce({ + status: 200, + body: { + ok: true, + data: { + records: recordIds.slice(1000).map((recordId) => ({ + id: recordId, + fields: {}, + })), + pagination: { hasMore: false }, + }, + }, + }); const tableService = { getTableMeta: vi.fn().mockResolvedValue({ id: 'tblTest', @@ -190,27 +222,10 @@ describe('TableOpenApiV2Service.createTable', () => { }, ]), }; - const recordService = { - getDocIdsByQuery: vi - .fn() - .mockResolvedValueOnce({ ids: recordIds.slice(0, 1000) }) - .mockResolvedValueOnce({ ids: recordIds.slice(1000) }), - getSnapshotBulkWithPermission: vi.fn().mockResolvedValue( - [...recordIds].reverse().map((recordId) => ({ - data: { - id: recordId, - name: recordId, - fields: {}, - }, - })) - ), - }; - const service = createService({ tableService, fieldOpenApiService, viewService, - recordService, }); const result = await service.createTable('bseTest', { @@ -222,16 +237,32 @@ describe('TableOpenApiV2Service.createTable', () => { })), }); - expect(recordService.getDocIdsByQuery).toHaveBeenNthCalledWith(1, 'tblTest', { - viewId: 'viwDefault', - skip: 0, - take: 1000, - }); - expect(recordService.getDocIdsByQuery).toHaveBeenNthCalledWith(2, 'tblTest', { - viewId: 'viwDefault', - skip: 1000, - take: 1, - }); + expect(executeListTableRecordsEndpoint).toHaveBeenNthCalledWith( + 1, + {}, + { + tableId: 'tblTest', + viewId: 'viwDefault', + fieldKeyType: 'name', + cellFormat: 'json', + limit: 1000, + offset: 0, + }, + {} + ); + expect(executeListTableRecordsEndpoint).toHaveBeenNthCalledWith( + 2, + {}, + { + tableId: 'tblTest', + viewId: 'viwDefault', + fieldKeyType: 'name', + cellFormat: 'json', + limit: 1, + offset: 1000, + }, + {} + ); expect(result.records).toHaveLength(1001); expect(result.records[0]?.id).toBe('rec1'); expect(result.records[1000]?.id).toBe('rec1001'); @@ -247,12 +278,14 @@ describe('TableOpenApiV2Service.duplicateTable', () => { tableService?: Record; fieldOpenApiService?: Record; viewService?: Record; - recordService?: Record; prismaService?: Record; dbProvider?: Record; }) => new TableOpenApiV2Service( { + getContainerForBase: vi.fn().mockResolvedValue({ + resolve: vi.fn().mockReturnValue({}), + }), getContainer: vi.fn().mockResolvedValue({ resolve: vi.fn().mockReturnValue({}), }), @@ -263,14 +296,14 @@ describe('TableOpenApiV2Service.duplicateTable', () => { (overrides?.tableService ?? {}) as never, (overrides?.fieldOpenApiService ?? {}) as never, (overrides?.viewService ?? {}) as never, - (overrides?.recordService ?? {}) as never, (overrides?.prismaService ?? {}) as never, { generateDbTableName: vi .fn() .mockImplementation((baseId: string, name: string) => `${baseId}.${name}`), ...overrides?.dbProvider, - } as never + } as never, + {} as never ); it('rebuilds the legacy duplicate-table response from the duplicated v2 table', async () => { diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts index d815271c14..d46b57144c 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts @@ -13,18 +13,19 @@ import { executeCreateTableEndpoint, executeDeleteTableEndpoint, executeDuplicateTableEndpoint, + executeListTableRecordsEndpoint, executeRestoreTableEndpoint, } from '@teable/v2-contract-http-implementation/handlers'; import { v2CoreTokens } from '@teable/v2-core'; -import type { ICommandBus } from '@teable/v2-core'; +import type { ICommandBus, IExecutionContext, IQueryBus } from '@teable/v2-core'; import { CustomHttpException, getDefaultCodeByStatus } from '../../../custom.exception'; import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; import { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; -import { RecordService } from '../../record/record.service'; import { V2ContainerService } from '../../v2/v2-container.service'; import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; import { ViewService } from '../../view/view.service'; +import { TableDuplicateService } from '../table-duplicate.service'; import { TableService } from '../table.service'; import { mapLegacyCreateTableToV2Input } from './table-open-api-v2.mapper'; @@ -38,11 +39,21 @@ export class TableOpenApiV2Service { private readonly tableService: TableService, private readonly fieldOpenApiService: FieldOpenApiService, private readonly viewService: ViewService, - private readonly recordService: RecordService, private readonly prismaService: PrismaService, - @InjectDbProvider() private readonly dbProvider: IDbProvider + @InjectDbProvider() private readonly dbProvider: IDbProvider, + private readonly tableDuplicateLegacyService: TableDuplicateService ) {} + private async collectCrossSpaceAffectedFields( + tableId: string + ): Promise> { + // Delegate to the v1 service so cross-space link, conditional lookup, + // conditional rollup, and their transitive lookup/rollup dependents are + // all detected consistently with the duplicate-check endpoint and the + // v1 downgrade path. Keeping detection in one place avoids drift. + return this.tableDuplicateLegacyService.previewCrossSpaceAffectedFields(tableId); + } + private throwV2Error( error: { code: string; @@ -60,9 +71,9 @@ export class TableOpenApiV2Service { } async createTable(baseId: string, createTableRo: ICreateTableWithDefault): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForBase(baseId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const normalizedCreateTableRo = await this.normalizeLegacyCreateTableRo(baseId, createTableRo); const result = await executeCreateTableEndpoint( context, @@ -74,7 +85,9 @@ export class TableOpenApiV2Service { return await this.buildLegacyCreateTableResponse( baseId, normalizedCreateTableRo, - result.body.data.table.id + result.body.data.table.id, + context, + container.resolve(v2CoreTokens.queryBus) ); } @@ -90,9 +103,9 @@ export class TableOpenApiV2Service { tableId: string, mode: 'soft' | 'permanent' = 'soft' ): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForBase(baseId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const result = await executeDeleteTableEndpoint( context, @@ -116,9 +129,9 @@ export class TableOpenApiV2Service { } async restoreTable(baseId: string, tableId: string): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForBase(baseId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const result = await executeRestoreTableEndpoint( context, @@ -145,9 +158,20 @@ export class TableOpenApiV2Service { tableId: string, duplicateTableRo: IDuplicateTableRo ): Promise { - const container = await this.v2ContainerService.getContainer(); + // The v2 duplicate command does not run cross-space validation when + // creating fields, so a table containing any cross-space link would + // silently produce another cross-space copy. Delegate to the v1 path, + // which downgrades cross-space link/lookup/rollup fields to single line + // text. Callers should hit `duplicate-check` first to preview which + // fields will be downgraded. + const affected = await this.collectCrossSpaceAffectedFields(tableId); + if (affected.length > 0) { + return this.tableDuplicateLegacyService.duplicateTable(baseId, tableId, duplicateTableRo); + } + + const container = await this.v2ContainerService.getContainerForBase(baseId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const result = await executeDuplicateTableEndpoint( context, { @@ -185,14 +209,16 @@ export class TableOpenApiV2Service { private async buildLegacyCreateTableResponse( baseId: string, createTableRo: ICreateTableWithDefault, - tableId: string + tableId: string, + context: IExecutionContext, + queryBus: IQueryBus ): Promise { const table = await this.tableService.getTableMeta(baseId, tableId); const fields = await this.fieldOpenApiService.getFields(tableId, { filterHidden: false, }); const views = await this.viewService.getViews(tableId); - const records = await this.getCreatedRecords(table, createTableRo); + const records = await this.getCreatedRecords(table, createTableRo, context, queryBus); return { ...table, @@ -224,41 +250,46 @@ export class TableOpenApiV2Service { private async getCreatedRecords( table: ITableVo, - createTableRo: ICreateTableWithDefault + createTableRo: ICreateTableWithDefault, + context: IExecutionContext, + queryBus: IQueryBus ): Promise { const total = createTableRo.records?.length ?? 0; if (total === 0) { return []; } - const recordIds: string[] = []; - for (let skip = 0; skip < total; skip += 1000) { - const take = Math.min(1000, total - skip); - const { ids } = await this.recordService.getDocIdsByQuery(table.id, { - viewId: table.defaultViewId, - skip, - take, - }); - recordIds.push(...ids); - } + const records: IRecord[] = []; + for (let offset = 0; offset < total; offset += 1000) { + const limit = Math.min(1000, total - offset); + const result = await executeListTableRecordsEndpoint( + context, + { + tableId: table.id, + viewId: table.defaultViewId, + fieldKeyType: createTableRo.fieldKeyType ?? FieldKeyType.Name, + cellFormat: CellFormat.Json, + limit, + offset, + }, + queryBus + ); - if (recordIds.length === 0) { - return []; - } + if (result.status === 200 && result.body.ok) { + records.push(...(result.body.data.records as IRecord[])); + continue; + } - const snapshots = await this.recordService.getSnapshotBulkWithPermission( - table.id, - recordIds, - undefined, - createTableRo.fieldKeyType ?? FieldKeyType.Name, - CellFormat.Json - ); - const recordById = new Map( - snapshots.map((snapshot) => [snapshot.data.id, snapshot.data] as const) - ); + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } - return recordIds - .map((recordId) => recordById.get(recordId)) + const recordById = new Map(records.map((record) => [record.id, record] as const)); + return records + .map((record) => recordById.get(record.id)) .filter((record): record is IRecord => record != null); } diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts index a03c356273..a48eca4a89 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts @@ -13,6 +13,8 @@ import { UseInterceptors, } from '@nestjs/common'; import type { + IDuplicateFieldCheckVo, + IDuplicateTableCheckVo, IDuplicateTableVo, IGetAbnormalVo, ITableFullVo, @@ -46,6 +48,7 @@ import { Permissions } from '../../auth/decorators/permissions.decorator'; import { UseV2Feature } from '../../canary/decorators/use-v2-feature.decorator'; import { V2FeatureGuard } from '../../canary/guards/v2-feature.guard'; import { V2IndicatorInterceptor } from '../../canary/interceptors/v2-indicator.interceptor'; +import { TableDuplicateService } from '../table-duplicate.service'; import { TableIndexService } from '../table-index.service'; import { TablePermissionService } from '../table-permission.service'; import { TableService } from '../table.service'; @@ -64,6 +67,7 @@ export class TableController { private readonly tableIndexService: TableIndexService, private readonly tablePermissionService: TablePermissionService, private readonly tableOpenApiV2Service: TableOpenApiV2Service, + private readonly tableDuplicateService: TableDuplicateService, private readonly cls: ClsService ) {} @@ -175,6 +179,27 @@ export class TableController { return await this.tableOpenApiService.duplicateTable(baseId, tableId, duplicateTableRo); } + @Permissions('table|read') + @Get(':tableId/duplicate-check') + async duplicateTableCheck(@Param('tableId') tableId: string): Promise { + const affectedFields = + await this.tableDuplicateService.previewCrossSpaceAffectedFields(tableId); + return { affectedFields }; + } + + @Permissions('field|create') + @Get(':tableId/field/:fieldId/duplicate-check') + async duplicateFieldCheck( + @Param('tableId') tableId: string, + @Param('fieldId') fieldId: string + ): Promise { + const affectedFields = await this.tableDuplicateService.previewFieldDuplicateCrossSpace( + tableId, + fieldId + ); + return { affectedFields }; + } + @UseV2Feature('deleteTable') @Delete(':tableId') @Permissions('table|delete') diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.spec.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.spec.ts index ff15974285..b2cdb973a9 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.spec.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.spec.ts @@ -133,16 +133,94 @@ describe('TableOpenApiService.prepareFields', () => { } ).prepareFields('tblTest', [nameFieldRo, linkFieldRo, lookupFieldRo, rollupFieldRo]); - expect(fieldSupplementService.prepareCreateFields).toHaveBeenCalledWith('tblTest', [ - nameFieldRo, - linkFieldRo, - ]); + expect(fieldSupplementService.prepareCreateFields).toHaveBeenCalledWith( + 'tblTest', + [nameFieldRo, linkFieldRo], + undefined, + { useTransaction: true } + ); expect(fieldSupplementService.prepareCreateField).toHaveBeenCalledTimes(2); expect(fields).toHaveLength(4); }); }); describe('TableOpenApiService.createTable', () => { + it('records legacy table creation schema operations in meta prisma', async () => { + const upsert = vi.fn().mockResolvedValue(undefined); + const prismaService = { + txClient: vi.fn().mockReturnValue({ + schemaOperation: { upsert }, + }), + }; + const cls = { + get: vi.fn().mockReturnValue('usrTest'), + }; + + const service = new TableOpenApiService( + prismaService as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + cls as never, + {} as never, + {} as never + ); + + await ( + service as unknown as { + completeTableCreateSchemaOperation: ( + baseId: string, + tableId: string, + recordCount: number + ) => Promise; + } + ).completeTableCreateSchemaOperation('bseTest', 'tblTest', 2); + + expect(upsert).toHaveBeenCalledWith( + expect.objectContaining({ + where: { idempotencyKey: 'table.create:table:tblTest' }, + create: expect.objectContaining({ + type: 'table.create', + status: 'ready', + phase: 'ready', + resourceType: 'table', + resourceId: 'tblTest', + baseId: 'bseTest', + tableId: 'tblTest', + idempotencyKey: 'table.create:table:tblTest', + payload: { recordCount: 2 }, + createdBy: 'usrTest', + lastModifiedBy: 'usrTest', + }), + update: expect.objectContaining({ + type: 'table.create', + status: 'ready', + phase: 'ready', + resourceType: 'table', + resourceId: 'tblTest', + baseId: 'bseTest', + tableId: 'tblTest', + payload: { recordCount: 2 }, + lockedAt: null, + lockedBy: null, + lastError: null, + lastModifiedBy: 'usrTest', + }), + }) + ); + }); + it('drops the data table when metadata transaction rolls back after physical creation', async () => { const projectsTable = 'bseTest.projects'; const createError = new Error('field create failed'); @@ -176,8 +254,10 @@ describe('TableOpenApiService.createTable', () => { const prismaService = { $tx: vi.fn(async (fn: () => Promise) => fn()), }; - const dataPrismaService = { - $executeRawUnsafe: executeRawUnsafe, + const databaseRouter = { + executeDataPrismaForBase: vi.fn(async (_baseId: string, sql: string) => + executeRawUnsafe(sql) + ), }; const dbProvider = { dropTable: vi.fn().mockReturnValue('drop table "bseTest"."projects"'), @@ -185,7 +265,7 @@ describe('TableOpenApiService.createTable', () => { const service = new TableOpenApiService( prismaService as never, - dataPrismaService as never, + databaseRouter as never, {} as never, {} as never, {} as never, @@ -237,13 +317,13 @@ describe('TableOpenApiService.cleanTablesRelatedData', () => { const prismaService = { txClient: vi.fn().mockReturnValue(metaTxClient), }; - const dataPrismaService = { - txClient: vi.fn().mockReturnValue(dataTxClient), + const databaseRouter = { + dataPrismaForBase: vi.fn().mockResolvedValue(dataTxClient), }; const service = new TableOpenApiService( prismaService as never, - dataPrismaService as never, + databaseRouter as never, {} as never, {} as never, {} as never, @@ -282,6 +362,67 @@ describe('TableOpenApiService.cleanTablesRelatedData', () => { expect(dataTxClient.recordTrash.deleteMany).toHaveBeenCalledWith({ where: { tableId: { in: ['tblA', 'tblB'] } }, }); + expect(databaseRouter.dataPrismaForBase).toHaveBeenCalledWith('bseTest', undefined); + }); + + it('uses the routed data transaction client when requested', async () => { + const metaTxClient = { + field: { deleteMany: vi.fn().mockResolvedValue(undefined) }, + view: { deleteMany: vi.fn().mockResolvedValue(undefined) }, + attachmentsTable: { deleteMany: vi.fn().mockResolvedValue(undefined) }, + ops: { deleteMany: vi.fn().mockResolvedValue(undefined) }, + tableMeta: { deleteMany: vi.fn().mockResolvedValue(undefined) }, + trash: { deleteMany: vi.fn().mockResolvedValue(undefined) }, + }; + const dataTxClient = { + recordHistory: { deleteMany: vi.fn().mockResolvedValue(undefined) }, + tableTrash: { deleteMany: vi.fn().mockResolvedValue(undefined) }, + recordTrash: { deleteMany: vi.fn().mockResolvedValue(undefined) }, + }; + const dataRootClient = { + txClient: vi.fn().mockReturnValue(dataTxClient), + recordHistory: { deleteMany: vi.fn() }, + tableTrash: { deleteMany: vi.fn() }, + recordTrash: { deleteMany: vi.fn() }, + }; + const prismaService = { + txClient: vi.fn().mockReturnValue(metaTxClient), + }; + const databaseRouter = { + dataPrismaForBase: vi.fn().mockResolvedValue(dataRootClient), + }; + + const service = new TableOpenApiService( + prismaService as never, + databaseRouter as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never + ); + + await service.cleanTablesRelatedData('bseTest', ['tblA'], { useTransaction: true }); + + expect(databaseRouter.dataPrismaForBase).toHaveBeenCalledWith('bseTest', { + useTransaction: true, + }); + expect(dataRootClient.txClient).toHaveBeenCalled(); + expect(dataTxClient.recordHistory.deleteMany).toHaveBeenCalledWith({ + where: { tableId: { in: ['tblA'] } }, + }); + expect(dataRootClient.recordHistory.deleteMany).not.toHaveBeenCalled(); }); }); @@ -304,10 +445,10 @@ describe('TableOpenApiService.dropTables', () => { const prismaService = { txClient: vi.fn().mockReturnValue(metaTxClient), }; - const dataPrismaService = { - txClient: vi.fn().mockReturnValue({ - $executeRawUnsafe: executeRawUnsafe, - }), + const databaseRouter = { + executeDataPrismaForTable: vi.fn(async (_tableId: string, sql: string) => + executeRawUnsafe(sql) + ), }; const batchService = { saveRawOps: vi.fn().mockResolvedValue(undefined), @@ -321,7 +462,7 @@ describe('TableOpenApiService.dropTables', () => { const service = new TableOpenApiService( prismaService as never, - dataPrismaService as never, + databaseRouter as never, {} as never, {} as never, {} as never, @@ -366,8 +507,8 @@ describe('TableOpenApiService.sqlQuery', () => { }, $queryRawUnsafe: metaQueryRawUnsafe, }; - const dataPrismaService = { - $queryRawUnsafe: dataQueryRawUnsafe, + const databaseRouter = { + queryDataPrismaForTable: vi.fn((_tableId: string, sql: string) => dataQueryRawUnsafe(sql)), }; const recordService = { buildFilterSortQuery: vi.fn().mockResolvedValue({ @@ -379,7 +520,7 @@ describe('TableOpenApiService.sqlQuery', () => { const service = new TableOpenApiService( prismaService as never, - dataPrismaService as never, + databaseRouter as never, {} as never, {} as never, recordService as never, @@ -440,8 +581,11 @@ describe('TableOpenApiService.updateDbTableName', () => { const dataTxClient = { $executeRawUnsafe: dataExecuteRawUnsafe, }; - const dataPrismaService = { - $tx: vi.fn(async (fn: (prisma: typeof dataTxClient) => Promise) => fn(dataTxClient)), + const databaseRouter = { + dataPrismaTransactionForTable: vi.fn( + async (_tableId: string, fn: (prisma: typeof dataTxClient) => Promise) => + fn(dataTxClient) + ), }; const tableService = { updateTable: vi.fn().mockResolvedValue(undefined), @@ -457,7 +601,7 @@ describe('TableOpenApiService.updateDbTableName', () => { const service = new TableOpenApiService( prismaService as never, - dataPrismaService as never, + databaseRouter as never, {} as never, {} as never, {} as never, @@ -513,8 +657,11 @@ describe('TableOpenApiService.updateDbTableName', () => { const dataTxClient = { $executeRawUnsafe: dataExecuteRawUnsafe, }; - const dataPrismaService = { - $tx: vi.fn(async (fn: (prisma: typeof dataTxClient) => Promise) => fn(dataTxClient)), + const databaseRouter = { + dataPrismaTransactionForTable: vi.fn( + async (_tableId: string, fn: (prisma: typeof dataTxClient) => Promise) => + fn(dataTxClient) + ), }; const dbProvider = { joinDbTableName: vi.fn().mockReturnValue(renamedOrdersTable), @@ -527,7 +674,7 @@ describe('TableOpenApiService.updateDbTableName', () => { const service = new TableOpenApiService( prismaService as never, - dataPrismaService as never, + databaseRouter as never, {} as never, {} as never, {} as never, diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts index c11d44bbdb..2e18fb3475 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts @@ -20,11 +20,11 @@ import { IdPrefix, TemplateRolePermission, actionPrefixMap, + getRandomString, getBasePermission, isLinkLookupOptions, } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; -import { PrismaService } from '@teable/db-main-prisma'; +import { PrismaService, ProvisionState } from '@teable/db-main-prisma'; import type { ICreateRecordsRo, ICreateTableRo, @@ -44,6 +44,8 @@ import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; import { Events } from '../../../event-emitter/events'; +import type { IDataDbRoutingOptions } from '../../../global/data-db-client-manager.service'; +import { DatabaseRouter } from '../../../global/database-router.service'; import { RawOpType } from '../../../share-db/interface'; import type { IClsStore } from '../../../types/cls'; import { updateOrder } from '../../../utils/update-order'; @@ -66,7 +68,7 @@ export class TableOpenApiService { private logger = new Logger(TableOpenApiService.name); constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly recordOpenApiService: RecordOpenApiService, private readonly viewOpenApiService: ViewOpenApiService, private readonly recordService: RecordService, @@ -144,7 +146,58 @@ export class TableOpenApiService { return this.recordOpenApiService.createRecords(tableId, data); } + private async completeTableCreateSchemaOperation( + baseId: string, + tableId: string, + recordCount: number + ) { + const now = new Date(); + const userId = this.cls.get('user.id'); + + await this.prismaService.txClient().schemaOperation.upsert({ + where: { + idempotencyKey: `table.create:table:${tableId}`, + }, + create: { + id: `sgo${getRandomString(16)}`, + type: 'table.create', + status: 'ready', + phase: 'ready', + resourceType: 'table', + resourceId: tableId, + baseId, + tableId, + idempotencyKey: `table.create:table:${tableId}`, + payload: { recordCount }, + attempts: 0, + maxAttempts: 8, + nextRunAt: now, + createdBy: userId, + lastModifiedTime: now, + lastModifiedBy: userId, + }, + update: { + type: 'table.create', + status: 'ready', + phase: 'ready', + resourceType: 'table', + resourceId: tableId, + baseId, + tableId, + payload: { recordCount }, + attempts: 0, + maxAttempts: 8, + nextRunAt: now, + lockedAt: null, + lockedBy: null, + lastError: null, + lastModifiedBy: userId, + }, + }); + } + private async cleanupCreatedDataTable( + baseId: string, table: Pick | undefined, reason: unknown ) { @@ -153,7 +206,10 @@ export class TableOpenApiService { } try { - await this.dataPrismaService.$executeRawUnsafe(this.dbProvider.dropTable(table.dbTableName)); + await this.databaseRouter.executeDataPrismaForBase( + baseId, + this.dbProvider.dropTable(table.dbTableName) + ); await this.tableMutationCacheInvalidator.invalidateDroppedTable(table.dbTableName); } catch (cleanupError) { this.logger.error( @@ -178,7 +234,9 @@ export class TableOpenApiService { const fields: IFieldVo[] = await this.fieldSupplementService.prepareCreateFields( tableId, - independentFields + independentFields, + undefined, + { useTransaction: true } ); const allFieldRos = independentFields.concat(dependentFields); @@ -193,7 +251,8 @@ export class TableOpenApiService { const computedFieldVo = await this.fieldSupplementService.prepareCreateField( tableId, fieldRo, - batchFieldVos + batchFieldVos, + { useTransaction: true } ); fieldVoMap.set(fieldRo, computedFieldVo); } @@ -253,6 +312,11 @@ export class TableOpenApiService { const orderB = fieldIdOrder.get(b.id) ?? Number.MAX_SAFE_INTEGER; return orderA - orderB; }); + await this.completeTableCreateSchemaOperation( + baseId, + tableId, + tableRo.records?.length ?? 0 + ); return { ...tableVo, @@ -263,7 +327,7 @@ export class TableOpenApiService { }; }) .catch(async (error) => { - await this.cleanupCreatedDataTable(createdTable, error); + await this.cleanupCreatedDataTable(baseId, createdTable, error); throw error; }); @@ -315,6 +379,7 @@ export class TableOpenApiService { where: { baseId, deletedTime: null, + provisionState: ProvisionState.ready, id: includeTableIds ? { in: includeTableIds } : undefined, }, }); @@ -375,7 +440,7 @@ export class TableOpenApiService { async () => { await this.dropTables(tableIds); await this.cleanTaskRelatedData(tableIds); - await this.cleanTablesRelatedData(baseId, tableIds); + await this.cleanTablesRelatedData(baseId, tableIds, { useTransaction: true }); }, { timeout: this.thresholdConfig.bigTransactionTimeout, @@ -388,15 +453,17 @@ export class TableOpenApiService { where: { id: { in: tableIds } }, select: { dbTableName: true, version: true, id: true, baseId: true, deletedTime: true }, }); - const dataPrisma = this.dataPrismaService.txClient(); - for (const table of tables) { if (!table.deletedTime) { await this.batchService.saveRawOps(table.baseId, RawOpType.Del, IdPrefix.Table, [ { docId: table.id, version: table.version }, ]); } - await dataPrisma.$executeRawUnsafe(this.dbProvider.dropTable(table.dbTableName)); + await this.databaseRouter.executeDataPrismaForTable( + table.id, + this.dbProvider.dropTable(table.dbTableName), + { useTransaction: true } + ); await this.tableMutationCacheInvalidator.invalidateDroppedTable(table.dbTableName); } } @@ -441,7 +508,11 @@ export class TableOpenApiService { }); } - async cleanTablesRelatedData(baseId: string, tableIds: string[]) { + async cleanTablesRelatedData( + baseId: string, + tableIds: string[], + routingOptions?: IDataDbRoutingOptions + ) { const metaPrisma = this.prismaService.txClient(); // delete field for table @@ -474,7 +545,11 @@ export class TableOpenApiService { }); // record history and trash snapshots live with the physical record tables on the data DB. - const dataPrisma = this.dataPrismaService.txClient(); + const routedDataPrisma = await this.databaseRouter.dataPrismaForBase(baseId, routingOptions); + const dataPrisma = + 'txClient' in routedDataPrisma && typeof routedDataPrisma.txClient === 'function' + ? routedDataPrisma.txClient() + : routedDataPrisma; // clean record history for table await dataPrisma.recordHistory.deleteMany({ @@ -585,7 +660,7 @@ export class TableOpenApiService { `; this.logger.log('sqlQuery:sql:combine: ' + combinedQuery); - return this.dataPrismaService.$queryRawUnsafe(combinedQuery); + return this.databaseRouter.queryDataPrismaForTable(tableId, combinedQuery); } async updateName(baseId: string, tableId: string, name: string) { @@ -652,7 +727,8 @@ export class TableOpenApiService { const renameSql = this.dbProvider.renameTableName(oldDbTableName, dbTableName); const rollbackRenameSql = this.dbProvider.renameTableName(dbTableName, oldDbTableName); - await this.dataPrismaService.$tx( + await this.databaseRouter.dataPrismaTransactionForTable( + tableId, async (prisma) => { for (const sql of renameSql) { await prisma.$executeRawUnsafe(sql); @@ -697,8 +773,9 @@ export class TableOpenApiService { await this.tableService.updateTable(baseId, tableId, { dbTableName }); }); } catch (error) { - await this.dataPrismaService - .$tx( + await this.databaseRouter + .dataPrismaTransactionForTable( + tableId, async (prisma) => { for (const sql of rollbackRenameSql) { await prisma.$executeRawUnsafe(sql); @@ -722,7 +799,7 @@ export class TableOpenApiService { async shuffle(baseId: string) { const tables = await this.prismaService.tableMeta.findMany({ - where: { baseId, deletedTime: null }, + where: { baseId, deletedTime: null, provisionState: ProvisionState.ready }, select: { id: true }, orderBy: { order: 'asc' }, }); @@ -744,6 +821,7 @@ export class TableOpenApiService { where: { baseId, deletedTime: null, + provisionState: ProvisionState.ready, }, select: { order: true, @@ -762,7 +840,7 @@ export class TableOpenApiService { const table = await this.prismaService.tableMeta .findFirstOrThrow({ select: { order: true, id: true }, - where: { baseId, id: tableId, deletedTime: null }, + where: { baseId, id: tableId, deletedTime: null, provisionState: ProvisionState.ready }, }) .catch(() => { throw new CustomHttpException(`Table ${tableId} not found`, HttpErrorCode.NOT_FOUND, { @@ -775,7 +853,7 @@ export class TableOpenApiService { const anchorTable = await this.prismaService.tableMeta .findFirstOrThrow({ select: { order: true, id: true }, - where: { baseId, id: anchorId, deletedTime: null }, + where: { baseId, id: anchorId, deletedTime: null, provisionState: ProvisionState.ready }, }) .catch(() => { throw new CustomHttpException(`Anchor ${anchorId} not found`, HttpErrorCode.NOT_FOUND, { @@ -796,6 +874,7 @@ export class TableOpenApiService { where: { baseId, deletedTime: null, + provisionState: ProvisionState.ready, order: whereOrder, }, orderBy: { order: align }, diff --git a/apps/nestjs-backend/src/features/table/open-api/v2-table-mutation-cache-invalidator.service.ts b/apps/nestjs-backend/src/features/table/open-api/v2-table-mutation-cache-invalidator.service.ts index 4fdcf30c65..0f9ef2c5d5 100644 --- a/apps/nestjs-backend/src/features/table/open-api/v2-table-mutation-cache-invalidator.service.ts +++ b/apps/nestjs-backend/src/features/table/open-api/v2-table-mutation-cache-invalidator.service.ts @@ -9,7 +9,10 @@ export class V2TableMutationCacheInvalidatorService implements TableMutationCach constructor(private readonly v2ContainerService: V2ContainerService) {} async invalidateDroppedTable(dbTableName: string): Promise { - const container = await this.v2ContainerService.getContainer(); + const baseId = dbTableName.split('.')[0]; + const container = baseId + ? await this.v2ContainerService.getContainerForBase(baseId) + : await this.v2ContainerService.getContainer(); const rootDb = container.resolve(v2DataDbTokens.db); invalidateUndoCaptureTableCache(dbTableName, rootDb); } diff --git a/apps/nestjs-backend/src/features/table/table-duplicate.service.ts b/apps/nestjs-backend/src/features/table/table-duplicate.service.ts index 9a2748ca93..0c4fd0f21d 100644 --- a/apps/nestjs-backend/src/features/table/table-duplicate.service.ts +++ b/apps/nestjs-backend/src/features/table/table-duplicate.service.ts @@ -10,9 +10,9 @@ import { } from '@teable/core'; import type { View } from '@teable/db-main-prisma'; import { PrismaService, ProvisionState } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { CreateRecordAction, + type ICrossSpaceTableAffectedField, type IDuplicateTableRo, type IDuplicateTableVo, type IFieldWithTableIdJson, @@ -26,8 +26,14 @@ import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; import { EventEmitterService } from '../../event-emitter/event-emitter.service'; import { Events } from '../../event-emitter/events'; +import { DataDbClientManager } from '../../global/data-db-client-manager.service'; import { CUSTOM_KNEX, DATA_KNEX } from '../../global/knex/knex.module'; import type { IClsStore } from '../../types/cls'; +import { + collectCrossSpaceAffectedFieldIds, + extractForeignTableId, +} from '../base/cross-space-detection.util'; +import type { ILinkFieldTableInfo } from '../base/utils'; import { DataLoaderService } from '../data-loader/data-loader.service'; import { FieldDuplicateService } from '../field/field-duplicate/field-duplicate.service'; import { createFieldInstanceByRaw, rawField2FieldObj } from '../field/model/factory'; @@ -37,6 +43,30 @@ import { ROW_ORDER_FIELD_PREFIX } from '../view/constant'; import { createViewVoByRaw } from '../view/model/factory'; import { TableService } from './table.service'; +type IDataPrismaExecutor = { + $executeRawUnsafe(query: string, ...values: unknown[]): Promise; + $queryRawUnsafe(query: string, ...values: unknown[]): Promise; +}; + +type IDataPrismaScopedClient = IDataPrismaExecutor & { + txClient?: () => IDataPrismaExecutor; +}; + +type IDuplicateTableDataProgress = { + processedRows: number; + batchProcessedRows: number; + currentBatch: number; + totalRows: number; +}; + +type IDuplicateTableDataOptions = { + batchSize?: number; + onProgress?: (progress: IDuplicateTableDataProgress) => void; +}; + +const duplicateTableDataDefaultBatchSize = 500; +const autoNumberFieldName = '__auto_number'; + @Injectable() export class TableDuplicateService { private logger = new Logger(TableDuplicateService.name); @@ -44,7 +74,6 @@ export class TableDuplicateService { constructor( private readonly cls: ClsService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, private readonly tableService: TableService, private readonly fieldOpenService: FieldOpenApiService, private readonly fieldDuplicateService: FieldDuplicateService, @@ -52,9 +81,30 @@ export class TableDuplicateService { @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel(CUSTOM_KNEX) private readonly knex: Knex, @InjectModel(DATA_KNEX) private readonly dataKnex: Knex, - private readonly eventEmitterService: EventEmitterService + private readonly eventEmitterService: EventEmitterService, + private readonly dataDbClientManager: DataDbClientManager ) {} + private getDataPrismaExecutor(prisma: IDataPrismaScopedClient): IDataPrismaExecutor { + return prisma.txClient?.() ?? prisma; + } + + private async assertSameDataDatabaseForRecordCopy(sourceTableId: string, targetBaseId: string) { + const [source, target] = await Promise.all([ + this.dataDbClientManager.getDataDatabaseForTable(sourceTableId, { useTransaction: true }), + this.dataDbClientManager.getDataDatabaseForBase(targetBaseId, { useTransaction: true }), + ]); + + if (source.cacheKey === target.cacheKey) { + return; + } + + throw new CustomHttpException( + 'Duplicating records across different space data databases is not supported yet', + HttpErrorCode.VALIDATION_ERROR + ); + } + private disableTableDomainDataLoader() { if (!this.cls.isActive()) { return; @@ -76,6 +126,7 @@ export class TableDuplicateService { } = await this.prismaService.tableMeta.findUniqueOrThrow({ where: { id: tableId }, }); + const userId = this.cls.get('user.id'); let newTableVo: | { @@ -108,18 +159,32 @@ export class TableDuplicateService { await this.repairDuplicateOmit(sourceToTargetFieldMap, sourceToTargetViewMap, newTableVo.id); if (includeRecords) { + await this.assertSameDataDatabaseForRecordCopy(tableId, baseId); + const dataPrisma = this.getDataPrismaExecutor( + await this.dataDbClientManager.dataPrismaForTable(newTableVo.id, { + useTransaction: true, + }) + ); const count = await this.duplicateTableData( dbTableName, newTableVo.dbTableName, sourceToTargetViewMap, sourceToTargetFieldMap, - [] + [], + dataPrisma ); - await this.duplicateAttachments(sourceTableId, newTableVo.id, sourceToTargetFieldMap); + await this.duplicateAttachments( + sourceTableId, + newTableVo.id, + sourceToTargetFieldMap, + dataPrisma + ); await this.duplicateLinkJunction( { [sourceTableId]: newTableVo.id }, - sourceToTargetFieldMap + sourceToTargetFieldMap, + true, + dataPrisma ); await this.emitTableDuplicateAuditLog(newTableVo.id, count, duplicateRo); } @@ -181,9 +246,11 @@ export class TableDuplicateService { targetDbTableName: string, sourceToTargetViewMap: Record, sourceToTargetFieldMap: Record, - crossBaseLinkInfo: { dbFieldName: string; selfKeyName: string; isMultipleCellValue: boolean }[] + crossBaseLinkInfo: ILinkFieldTableInfo[], + dataPrisma: IDataPrismaExecutor, + options?: IDuplicateTableDataOptions ) { - const prisma = this.dataPrismaService.txClient(); + const prisma = dataPrisma; const metaPrisma = this.prismaService.txClient(); const qb = this.dataKnex.queryBuilder(); @@ -262,11 +329,11 @@ export class TableDuplicateService { ); for (const name of newRowColumns) { - await this.createRowOrderField(targetDbTableName, name.slice(6)); + await this.createRowOrderField(targetDbTableName, name.slice(6), prisma); } for (const name of newFkColumns) { - await this.createFkField(targetDbTableName, name.slice(5)); + await this.createFkField(targetDbTableName, name.slice(5), prisma); } // following field should not be duplicated @@ -307,6 +374,22 @@ export class TableDuplicateService { .concat(newFkColumns) .filter((dbFieldName) => !excludeColumnsSet.has(dbFieldName)); + const buildDuplicateSql = (range?: { + minAutoNumberExclusive?: number; + maxAutoNumberInclusive?: number; + }) => + this.dbProvider + .duplicateTableQuery(this.dataKnex.queryBuilder()) + .duplicateTableData( + sourceDbTableName, + targetDbTableName, + newColumns, + oldColumns, + crossBaseLinkDbFieldNames, + range + ) + .toQuery(); + const sql = this.dbProvider .duplicateTableQuery(qb) .duplicateTableData( @@ -324,14 +407,63 @@ export class TableDuplicateService { const sourceTableCountResult = await prisma.$queryRawUnsafe<[{ count: bigint | number }]>(sourceTableCountSql); + const totalRows = Number(sourceTableCountResult[0]?.count || 0); + + if (!options?.onProgress || totalRows === 0) { + await prisma.$executeRawUnsafe(sql); + return totalRows; + } + + const batchSize = options.batchSize ?? duplicateTableDataDefaultBatchSize; + let lastAutoNumber = 0; + let processedRows = 0; + let currentBatch = 0; + + while (processedRows < totalRows) { + const autoNumberRowsSql = this.dataKnex(sourceDbTableName) + .select(autoNumberFieldName) + .where(autoNumberFieldName, '>', lastAutoNumber) + .orderBy(autoNumberFieldName, 'asc') + .limit(batchSize) + .toQuery(); + const autoNumberRows = + await prisma.$queryRawUnsafe< + Array> + >(autoNumberRowsSql); + if (!autoNumberRows.length) { + break; + } + + const batchLastAutoNumber = Number( + autoNumberRows[autoNumberRows.length - 1]![autoNumberFieldName] + ); + await prisma.$executeRawUnsafe( + buildDuplicateSql({ + minAutoNumberExclusive: lastAutoNumber, + maxAutoNumberInclusive: batchLastAutoNumber, + }) + ); - await prisma.$executeRawUnsafe(sql); + currentBatch += 1; + processedRows += autoNumberRows.length; + options.onProgress({ + processedRows, + batchProcessedRows: autoNumberRows.length, + currentBatch, + totalRows, + }); + lastAutoNumber = batchLastAutoNumber; + } - return Number(sourceTableCountResult[0]?.count || 0); + return totalRows; } - private async createRowOrderField(dbTableName: string, viewId: string) { - const prisma = this.dataPrismaService.txClient(); + private async createRowOrderField( + dbTableName: string, + viewId: string, + dataPrisma: IDataPrismaExecutor + ) { + const prisma = dataPrisma; const rowIndexFieldName = `${ROW_ORDER_FIELD_PREFIX}_${viewId}`; @@ -365,8 +497,12 @@ export class TableDuplicateService { await prisma.$executeRawUnsafe(createRowIndexSQL); } - private async createFkField(dbTableName: string, fieldId: string) { - const prisma = this.dataPrismaService.txClient(); + private async createFkField( + dbTableName: string, + fieldId: string, + dataPrisma: IDataPrismaExecutor + ) { + const prisma = dataPrisma; const fkFieldName = `__fk_${fieldId}`; @@ -382,6 +518,88 @@ export class TableDuplicateService { } } + async previewFieldDuplicateCrossSpace( + tableId: string, + fieldId: string + ): Promise { + return (await this.previewCrossSpaceAffectedFields(tableId)).filter( + (f) => f.fieldId === fieldId + ); + } + + async previewCrossSpaceAffectedFields( + sourceTableId: string, + targetTableId: string = sourceTableId + ): Promise { + const prisma = this.prismaService.txClient(); + + const fieldsRaw = await prisma.field.findMany({ + where: { tableId: sourceTableId, deletedTime: null }, + }); + if (!fieldsRaw.length) return []; + + const foreignTableIds = Array.from( + new Set( + fieldsRaw + .map((f) => extractForeignTableId(f)) + .filter((ft): ft is string => !!ft && ft !== targetTableId) + ) + ); + if (foreignTableIds.length === 0) return []; + + const rows = await prisma.tableMeta.findMany({ + where: { id: { in: [targetTableId, ...foreignTableIds] }, deletedTime: null }, + select: { id: true, base: { select: { spaceId: true } } }, + }); + const spaceMap = new Map(rows.map((r) => [r.id, r.base.spaceId])); + const targetSpace = spaceMap.get(targetTableId); + if (!targetSpace) return []; + + const affected = collectCrossSpaceAffectedFieldIds({ + fields: fieldsRaw, + isForeignInternal: (ft) => ft === targetTableId, + isForeignCrossSpace: (ft) => { + const s = spaceMap.get(ft); + return Boolean(s && s !== targetSpace); + }, + }); + + return fieldsRaw + .filter((f) => affected.has(f.id)) + .map((f) => ({ fieldId: f.id, fieldName: f.name, type: f.type })); + } + + private async identifyCrossSpaceFieldIds( + targetTableId: string, + fields: IFieldWithTableIdJson[] + ): Promise> { + const foreignTableIds = Array.from( + new Set( + fields + .map((f) => extractForeignTableId(f)) + .filter((ft): ft is string => !!ft && ft !== targetTableId) + ) + ); + if (!foreignTableIds.length) return new Set(); + + const rows = await this.prismaService.txClient().tableMeta.findMany({ + where: { id: { in: [targetTableId, ...foreignTableIds] }, deletedTime: null }, + select: { id: true, base: { select: { spaceId: true } } }, + }); + const spaceMap = new Map(rows.map((r) => [r.id, r.base.spaceId])); + const targetSpace = spaceMap.get(targetTableId); + if (!targetSpace) return new Set(); + + return collectCrossSpaceAffectedFieldIds({ + fields, + isForeignInternal: (ft) => ft === targetTableId, + isForeignCrossSpace: (ft) => { + const s = spaceMap.get(ft); + return Boolean(s && s !== targetSpace); + }, + }); + } + private async duplicateFields(sourceTableId: string, targetTableId: string) { const fieldsRaw = await this.prismaService.txClient().field.findMany({ where: { tableId: sourceTableId, deletedTime: null }, @@ -416,33 +634,62 @@ export class TableDuplicateService { FieldType.Button, ]; + // Identify cross-space link fields and their direct lookup/rollup dependents. + // After duplication these would otherwise be rejected by the cross-space + // assertion in field-supplement; we route them through createCommonFields as + // single line text instead. + const crossSpaceIds = await this.identifyCrossSpaceFieldIds(targetTableId, fieldsInstances); + + const downgradeToText = (f: IFieldWithTableIdJson): IFieldWithTableIdJson => { + const mutable: Record = { ...f }; + mutable.type = FieldType.SingleLineText; + delete mutable.options; + delete mutable.lookupOptions; + delete mutable.isLookup; + delete mutable.isConditionalLookup; + delete mutable.isComputed; + delete mutable.isMultipleCellValue; + delete mutable.dbFieldType; + delete mutable.cellValueType; + return mutable as IFieldWithTableIdJson; + }; + + const downgradedFields = fieldsInstances + .filter(({ id }) => crossSpaceIds.has(id)) + .map(downgradeToText); + const commonFields = fieldsInstances.filter( - ({ type, isLookup, aiConfig }) => - !nonCommonFieldTypes.includes(type) && !isLookup && !aiConfig + ({ id, type, isLookup, aiConfig }) => + !crossSpaceIds.has(id) && !nonCommonFieldTypes.includes(type) && !isLookup && !aiConfig ); // the primary formula which rely on other fields const primaryFormulaFields = fieldsInstances.filter( - ({ type, isLookup }) => type === FieldType.Formula && !isLookup + ({ id, type, isLookup }) => !crossSpaceIds.has(id) && type === FieldType.Formula && !isLookup ); // these field require other field, we need to merge them and ensure a specific order const linkFields = fieldsInstances.filter( - ({ type, isLookup }) => type === FieldType.Link && !isLookup + ({ id, type, isLookup }) => !crossSpaceIds.has(id) && type === FieldType.Link && !isLookup ); const buttonFields = fieldsInstances.filter( - ({ type, isLookup }) => type === FieldType.Button && !isLookup + ({ id, type, isLookup }) => !crossSpaceIds.has(id) && type === FieldType.Button && !isLookup ); // rest fields, like formula, rollup, lookup fields const dependencyFields = fieldsInstances.filter( ({ id }) => + !crossSpaceIds.has(id) && ![...primaryFormulaFields, ...linkFields, ...buttonFields, ...commonFields] .map(({ id }) => id) .includes(id) ); + if (downgradedFields.length) { + await this.fieldDuplicateService.createCommonFields(downgradedFields, sourceToTargetFieldMap); + } + await this.fieldDuplicateService.createCommonFields(commonFields, sourceToTargetFieldMap); await this.fieldDuplicateService.createButtonFields(buttonFields, sourceToTargetFieldMap); @@ -585,7 +832,9 @@ export class TableDuplicateService { // Only attempt to rename if a physical column exists. // Link fields do not create standard columns; self-link symmetric side definitely doesn't. - const dataPrisma = this.dataPrismaService.txClient(); + const dataPrisma = this.getDataPrismaExecutor( + await this.dataDbClientManager.dataPrismaForTable(targetTableId, { useTransaction: true }) + ); const exists = await this.dbProvider.checkColumnExist( targetDbTableName, genDbFieldName, @@ -856,10 +1105,12 @@ export class TableDuplicateService { async duplicateAttachments( sourceTableId: string, targetTableId: string, - fieldIdMap: Record + fieldIdMap: Record, + dataPrisma: IDataPrismaExecutor ) { - const prisma = this.prismaService.txClient(); - const attachmentFieldRaws = await prisma.field.findMany({ + const prisma = dataPrisma; + const metaPrisma = this.prismaService.txClient(); + const attachmentFieldRaws = await metaPrisma.field.findMany({ where: { tableId: sourceTableId, type: FieldType.Attachment, @@ -895,11 +1146,12 @@ export class TableDuplicateService { async duplicateLinkJunction( tableIdMap: Record, fieldIdMap: Record, - allowCrossBase: boolean = true, + allowCrossBase: boolean, + routedDataPrisma: IDataPrismaExecutor, disconnectedLinkFieldIds?: string[] ) { const metaPrisma = this.prismaService.txClient(); - const dataPrisma = this.dataPrismaService.txClient(); + const dataPrisma = routedDataPrisma; const sourceLinkFieldRaws = await metaPrisma.field.findMany({ where: { tableId: { in: Object.keys(tableIdMap) }, @@ -916,6 +1168,8 @@ export class TableDuplicateService { }, }); + const targetFields = targetLinkFieldRaws.map((f) => createFieldInstanceByRaw(f)); + const targetLinkFieldIds = new Set(targetFields.map((f) => f.id)); const sourceFields = sourceLinkFieldRaws .filter(({ isLookup }) => !isLookup) .map((f) => createFieldInstanceByRaw(f)) @@ -931,8 +1185,12 @@ export class TableDuplicateService { return true; } return !disconnectedLinkFieldIds.includes(field.id); - }); - const targetFields = targetLinkFieldRaws.map((f) => createFieldInstanceByRaw(f)); + }) + // Drop source links whose target is no longer a Link in the new base — + // e.g. cross-space links and excluded-table links that base-export + // structurally degrades to SingleLineText. Without their target column, + // there's no junction to copy. + .filter((field) => targetLinkFieldIds.has(fieldIdMap[field.id])); const junctionDbTableNameMap = {} as Record< string, diff --git a/apps/nestjs-backend/src/features/table/table-index.service.ts b/apps/nestjs-backend/src/features/table/table-index.service.ts index 563e608fe4..e8c1fb3170 100644 --- a/apps/nestjs-backend/src/features/table/table-index.service.ts +++ b/apps/nestjs-backend/src/features/table/table-index.service.ts @@ -2,7 +2,6 @@ import { Injectable, Logger } from '@nestjs/common'; import { FieldType, HttpErrorCode } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { TableIndex } from '@teable/openapi'; import type { IGetAbnormalVo, ITableIndexType, IToggleIndexRo } from '@teable/openapi'; import { ClsService } from 'nestjs-cls'; @@ -10,6 +9,8 @@ import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.confi import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import type { IDataDbRoutingOptions } from '../../global/data-db-client-manager.service'; +import { DatabaseRouter } from '../../global/database-router.service'; import type { IClsStore } from '../../types/cls'; import type { IFieldInstance } from '../field/model/factory'; import { createFieldInstanceByRaw } from '../field/model/factory'; @@ -21,7 +22,7 @@ export class TableIndexService { constructor( private readonly cls: ClsService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, @InjectDbProvider() private readonly dbProvider: IDbProvider ) {} @@ -44,7 +45,8 @@ export class TableIndexService { async getActivatedTableIndexes( tableId: string, - type: TableIndex = TableIndex.search + type: TableIndex = TableIndex.search, + routingOptions?: IDataDbRoutingOptions ): Promise { const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ where: { @@ -57,11 +59,11 @@ export class TableIndexService { if (type === TableIndex.search) { const searchIndexSql = this.dbProvider.searchIndex().getExistTableIndexSql(dbTableName); - const [{ exists: searchIndexExist }] = await this.dataPrismaService.$queryRawUnsafe< + const [{ exists: searchIndexExist }] = await this.databaseRouter.queryDataPrismaForTable< { exists: boolean; }[] - >(searchIndexSql); + >(tableId, searchIndexSql, routingOptions); const result: ITableIndexType[] = []; @@ -110,13 +112,19 @@ export class TableIndexService { }, }); - await this.toggleSearchIndex(dbTableName, fields, !index.includes(type)); + await this.toggleSearchIndex(tableId, dbTableName, fields, !index.includes(type)); } - async toggleSearchIndex(dbTableName: string, fields: IFieldInstance[], toEnable: boolean) { + async toggleSearchIndex( + tableId: string, + dbTableName: string, + fields: IFieldInstance[], + toEnable: boolean + ) { if (toEnable) { const sqls = this.dbProvider.searchIndex().getCreateIndexSql(dbTableName, fields); - return await this.dataPrismaService.$tx( + return await this.databaseRouter.dataPrismaTransactionForTable( + tableId, async (prisma) => { for (let i = 0; i < sqls.length; i++) { const sql = sqls[i]; @@ -142,7 +150,7 @@ export class TableIndexService { const sql = this.dbProvider.searchIndex().getDropIndexSql(dbTableName); try { - return await this.dataPrismaService.$executeRawUnsafe(sql); + return await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } catch (error) { console.error('toggleSearchIndex:drop:error', sql); throw new CustomHttpException( @@ -167,11 +175,15 @@ export class TableIndexService { if (index.includes(TableIndex.search)) { const sql = this.dbProvider.searchIndex().getDeleteSingleIndexSql(dbTableName, field); // Execute within current transaction if present to keep boundaries consistent - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } } - async createSearchFieldSingleIndex(tableId: string, fieldInstance: IFieldInstance) { + async createSearchFieldSingleIndex( + tableId: string, + fieldInstance: IFieldInstance, + routingOptions?: IDataDbRoutingOptions + ) { if (fieldInstance.type === FieldType.Button) { return; } @@ -180,10 +192,10 @@ export class TableIndexService { select: { dbTableName: true }, }); const { dbTableName } = tableRaw; - const index = await this.getActivatedTableIndexes(tableId); + const index = await this.getActivatedTableIndexes(tableId, TableIndex.search, routingOptions); const sql = this.dbProvider.searchIndex().createSingleIndexSql(dbTableName, fieldInstance); if (index.includes(TableIndex.search) && sql) { - await this.dataPrismaService.txClient().$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql, routingOptions); } } @@ -202,7 +214,7 @@ export class TableIndexService { const sql = this.dbProvider .searchIndex() .getUpdateSingleIndexNameSql(dbTableName, oldField, newField); - await this.dataPrismaService.$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } } @@ -214,7 +226,7 @@ export class TableIndexService { const { dbTableName } = tableRaw; const sql = this.dbProvider.searchIndex().getIndexInfoSql(dbTableName); - return this.dataPrismaService.$queryRawUnsafe(sql); + return this.databaseRouter.queryDataPrismaForTable(tableId, sql); } async getAbnormalTableIndex(tableId: string, type: TableIndex) { @@ -267,7 +279,8 @@ export class TableIndexService { const dropSql = this.dbProvider.searchIndex().getDropIndexSql(dbTableName); const fieldInstances = await this.getSearchIndexFields(tableId); const createSqls = this.dbProvider.searchIndex().getCreateIndexSql(dbTableName, fieldInstances); - await this.dataPrismaService.$tx( + await this.databaseRouter.dataPrismaTransactionForTable( + tableId, async (prisma) => { await prisma.$executeRawUnsafe(dropSql); for (let i = 0; i < createSqls.length; i++) { diff --git a/apps/nestjs-backend/src/features/table/table.service.spec.ts b/apps/nestjs-backend/src/features/table/table.service.spec.ts index 87f3ee1883..7bb512ead2 100644 --- a/apps/nestjs-backend/src/features/table/table.service.spec.ts +++ b/apps/nestjs-backend/src/features/table/table.service.spec.ts @@ -38,4 +38,56 @@ describe('TableService', () => { const dbTableName = service.generateValidName(''); expect(dbTableName).toBe('unnamed'); }); + + it('uses the routed data transaction client when creating a physical table', async () => { + const dataTxClient = { + $executeRawUnsafe: vi.fn().mockResolvedValue(0), + }; + const dataRootClient = { + txClient: vi.fn().mockReturnValue(dataTxClient), + $executeRawUnsafe: vi.fn().mockResolvedValue(0), + }; + const metaTxClient = { + tableMeta: { + findMany: vi.fn().mockResolvedValue([]), + findFirst: vi.fn().mockResolvedValue(null), + create: vi.fn().mockResolvedValue({ id: 'tblTest', dbTableName: 'bseTest.orders' }), + update: vi.fn().mockResolvedValue(undefined), + }, + }; + const mockedService = new TableService( + { get: vi.fn().mockReturnValue('usrTest') } as never, + { txClient: vi.fn().mockReturnValue(metaTxClient) } as never, + { dataPrismaForBase: vi.fn().mockResolvedValue(dataRootClient) } as never, + {} as never, + { + driver: 'sqlite', + generateDbTableName: vi.fn((_baseId: string, name: string) => `bseTest.${name}`), + dropTable: vi.fn((name: string) => `drop table ${name}`), + } as never, + { + schema: { + createTable: vi.fn().mockReturnValue({ + toSQL: () => [{ sql: 'create table "bseTest"."orders" ("__id" text)' }], + }), + }, + } as never + ); + + await ( + mockedService as unknown as { + createDBTable( + baseId: string, + tableRo: { name: string }, + createTable?: boolean + ): Promise; + } + ).createDBTable('bseTest', { name: 'orders' }); + + expect(dataRootClient.txClient).toHaveBeenCalled(); + expect(dataTxClient.$executeRawUnsafe).toHaveBeenCalledWith( + 'create table "bseTest"."orders" ("__id" text)' + ); + expect(dataRootClient.$executeRawUnsafe).not.toHaveBeenCalled(); + }); }); diff --git a/apps/nestjs-backend/src/features/table/table.service.ts b/apps/nestjs-backend/src/features/table/table.service.ts index 1c1bb7fef1..1e4808a6cc 100644 --- a/apps/nestjs-backend/src/features/table/table.service.ts +++ b/apps/nestjs-backend/src/features/table/table.service.ts @@ -12,7 +12,6 @@ import { } from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; import { PrismaService, ProvisionState } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { ICreateTableRo, ITableVo } from '@teable/openapi'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; @@ -20,13 +19,22 @@ import { ClsService } from 'nestjs-cls'; import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { DataDbClientManager } from '../../global/data-db-client-manager.service'; +import { DATA_KNEX } from '../../global/knex'; import type { IReadonlyAdapterService } from '../../share-db/interface'; import { RawOpType } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; import { convertNameToValidCharacter } from '../../utils/name-conversion'; -import { DATA_KNEX } from '../../global/knex'; import { BatchService } from '../calculation/batch.service'; +type IDataPrismaExecutor = { + $executeRawUnsafe(query: string, ...values: unknown[]): PromiseLike; +}; + +type IDataPrismaScopedClient = IDataPrismaExecutor & { + txClient?: () => IDataPrismaExecutor; +}; + @Injectable() export class TableService implements IReadonlyAdapterService { private logger = new Logger(TableService.name); @@ -34,7 +42,7 @@ export class TableService implements IReadonlyAdapterService { constructor( private readonly cls: ClsService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly dataDbClientManager: DataDbClientManager, private readonly batchService: BatchService, @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel(DATA_KNEX) private readonly knex: Knex @@ -51,11 +59,13 @@ export class TableService implements IReadonlyAdapterService { .$executeRaw`select id from base where id = ${baseId} for update`; } - private async cleanupCreatedDataTable(dbTableName: string, reason: unknown) { + private async cleanupCreatedDataTable( + dataPrisma: IDataPrismaExecutor, + dbTableName: string, + reason: unknown + ) { try { - await this.dataPrismaService - .txClient() - .$executeRawUnsafe(this.dbProvider.dropTable(dbTableName)); + await dataPrisma.$executeRawUnsafe(this.dbProvider.dropTable(dbTableName)); } catch (cleanupError) { this.logger.error( `Failed to clean up data table ${dbTableName} after table metadata provisioning error: ${ @@ -66,6 +76,10 @@ export class TableService implements IReadonlyAdapterService { } } + private getDataPrismaExecutor(prisma: IDataPrismaScopedClient): IDataPrismaExecutor { + return prisma.txClient?.() ?? prisma; + } + private async createDBTable(baseId: string, tableRo: ICreateTableRo, createTable = true) { const userId = this.cls.get('user.id'); await this.lockBaseRow(baseId); @@ -152,8 +166,12 @@ export class TableService implements IReadonlyAdapterService { table.integer('__version').notNullable(); }); + let dataPrisma: IDataPrismaExecutor | undefined; try { - const dataPrisma = this.dataPrismaService.txClient(); + const scopedDataPrisma = await this.dataDbClientManager.dataPrismaForBase(baseId, { + useTransaction: true, + }); + dataPrisma = this.getDataPrismaExecutor(scopedDataPrisma); for (const sql of createTableSchema.toSQL()) { await dataPrisma.$executeRawUnsafe(sql.sql); } @@ -165,7 +183,9 @@ export class TableService implements IReadonlyAdapterService { }, }); } catch (error) { - await this.cleanupCreatedDataTable(dbTableName, error); + if (dataPrisma) { + await this.cleanupCreatedDataTable(dataPrisma, dbTableName, error); + } await this.prismaService.txClient().tableMeta.update({ where: { id: tableMeta.id }, data: { @@ -210,7 +230,7 @@ export class TableService implements IReadonlyAdapterService { async getTableMeta(baseId: string, tableId: string): Promise { const tableMeta = await this.prismaService.txClient().tableMeta.findFirst({ - where: { id: tableId, baseId, deletedTime: null }, + where: { id: tableId, baseId, deletedTime: null, provisionState: ProvisionState.ready }, }); if (!tableMeta) { @@ -440,7 +460,7 @@ export class TableService implements IReadonlyAdapterService { ): Promise[]> { const { ignoreDefaultViewId } = ops; const tables = await this.prismaService.txClient().tableMeta.findMany({ - where: { baseId, id: { in: ids }, deletedTime: null }, + where: { baseId, id: { in: ids }, deletedTime: null, provisionState: ProvisionState.ready }, orderBy: { order: 'asc' }, }); @@ -473,6 +493,7 @@ export class TableService implements IReadonlyAdapterService { where: { deletedTime: null, baseId, + provisionState: ProvisionState.ready, ...(projectionTableIds ? { id: { in: projectionTableIds }, diff --git a/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.spec.ts b/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.spec.ts index f243cce325..47dd92e2d8 100644 --- a/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.spec.ts +++ b/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.spec.ts @@ -26,8 +26,11 @@ describe('TableTrashListener', () => { create: vi.fn(), }, }; + const dataDbClientManager = { + dataPrismaForTable: vi.fn().mockResolvedValue(dataPrismaService), + }; const listener = new TableTrashListener( - dataPrismaService as never, + dataDbClientManager as never, { bigTransactionTimeout: 30_000, } as never @@ -44,6 +47,9 @@ describe('TableTrashListener', () => { await listener.recordDeleteListener(payload); + expect(dataDbClientManager.dataPrismaForTable).toHaveBeenCalledWith('tblTrashListenerTable', { + useTransaction: true, + }); expect(dataPrismaService.$tx).toHaveBeenCalledWith(expect.any(Function), { timeout: 30_000, }); @@ -95,14 +101,18 @@ describe('TableTrashListener', () => { create: tableTrashCreate, }, }; + const dataDbClientManager = { + dataPrismaForTable: vi.fn().mockResolvedValue(dataPrismaService), + }; const listener = new TableTrashListener( - dataPrismaService as never, + dataDbClientManager as never, { bigTransactionTimeout: 30_000, } as never ); const fieldPayload: IDeleteFieldsPayload = { operationId: 'oprTrashListenerField', + windowId: 'winTrashListenerWindow', tableId: 'tblTrashListenerTable', userId: 'usrTrashListenerUser', fields: [{ id: 'fldTrashListenerField', name: 'Name' }] as never, @@ -110,6 +120,7 @@ describe('TableTrashListener', () => { }; const viewPayload: IDeleteViewPayload = { operationId: 'oprTrashListenerView', + windowId: 'winTrashListenerWindow', tableId: 'tblTrashListenerTable', userId: 'usrTrashListenerUser', viewId: 'viwTrashListenerView', @@ -118,6 +129,16 @@ describe('TableTrashListener', () => { await listener.fieldDeleteListener(fieldPayload); await listener.viewDeleteListener(viewPayload); + expect(dataDbClientManager.dataPrismaForTable).toHaveBeenNthCalledWith( + 1, + 'tblTrashListenerTable', + { useTransaction: true } + ); + expect(dataDbClientManager.dataPrismaForTable).toHaveBeenNthCalledWith( + 2, + 'tblTrashListenerTable', + { useTransaction: true } + ); expect(tableTrashCreate).toHaveBeenNthCalledWith(1, { data: { id: 'oprTrashListenerField', diff --git a/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.ts b/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.ts index 385cc3d6a8..806d12dff3 100644 --- a/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.ts +++ b/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.ts @@ -1,21 +1,70 @@ import { Injectable } from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; import { generateRecordTrashId } from '@teable/core'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { ResourceType } from '@teable/openapi'; import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.config'; import { Events } from '../../../event-emitter/events'; +import { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import { IDeleteFieldsPayload } from '../../undo-redo/operations/delete-fields.operation'; import { IDeleteRecordsPayload } from '../../undo-redo/operations/delete-records.operation'; import { IDeleteViewPayload } from '../../undo-redo/operations/delete-view.operation'; +type ITableTrashDataPrisma = { + tableTrash: { + create(args: unknown): PromiseLike; + }; + recordTrash: { + createMany(args: unknown): PromiseLike; + }; +}; + +type IScopedTableTrashDataPrisma = ITableTrashDataPrisma & { + txClient?: () => ITableTrashDataPrisma; + $tx?: ( + fn: (prisma: ITableTrashDataPrisma) => Promise, + options?: { timeout?: number } + ) => Promise; + $transaction?: ( + fn: (prisma: ITableTrashDataPrisma) => Promise, + options?: { timeout?: number } + ) => Promise; +}; + @Injectable() export class TableTrashListener { constructor( - private readonly dataPrismaService: DataPrismaService, + private readonly dataDbClientManager: DataDbClientManager, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig ) {} + private getDataPrismaExecutor(prisma: IScopedTableTrashDataPrisma): ITableTrashDataPrisma { + return prisma.txClient?.() ?? prisma; + } + + private async dataPrismaForTable(tableId: string): Promise { + return (await this.dataDbClientManager.dataPrismaForTable(tableId, { + useTransaction: true, + })) as IScopedTableTrashDataPrisma; + } + + private async dataPrismaTransactionForTable( + tableId: string, + fn: (prisma: ITableTrashDataPrisma) => Promise, + options?: { timeout?: number } + ): Promise { + const prisma = await this.dataPrismaForTable(tableId); + + if (prisma.$tx) { + return await prisma.$tx(fn, options); + } + + if (prisma.$transaction) { + return await prisma.$transaction(fn, options); + } + + return await fn(this.getDataPrismaExecutor(prisma)); + } + @OnEvent(Events.OPERATION_RECORDS_DELETE) async recordDeleteListener(payload: IDeleteRecordsPayload) { const { operationId, userId, tableId, records } = payload; @@ -25,7 +74,8 @@ export class TableTrashListener { const recordIds = records.map((record) => record.id); const createdTime = new Date(); - await this.dataPrismaService.$tx( + await this.dataPrismaTransactionForTable( + tableId, async (prisma) => { await prisma.tableTrash.create({ data: { @@ -65,7 +115,9 @@ export class TableTrashListener { if (!operationId) return; - await this.dataPrismaService.tableTrash.create({ + const dataPrisma = this.getDataPrismaExecutor(await this.dataPrismaForTable(tableId)); + + await dataPrisma.tableTrash.create({ data: { id: operationId, tableId, @@ -82,7 +134,9 @@ export class TableTrashListener { if (!operationId) return; - await this.dataPrismaService.tableTrash.create({ + const dataPrisma = this.getDataPrismaExecutor(await this.dataPrismaForTable(tableId)); + + await dataPrisma.tableTrash.create({ data: { id: operationId, tableId, diff --git a/apps/nestjs-backend/src/features/trash/trash.controller.ts b/apps/nestjs-backend/src/features/trash/trash.controller.ts index adbdf7c896..b397c97031 100644 --- a/apps/nestjs-backend/src/features/trash/trash.controller.ts +++ b/apps/nestjs-backend/src/features/trash/trash.controller.ts @@ -46,13 +46,14 @@ export class TrashController { @TokenAccess() async restoreTrash( @Param('trashId') trashId: string, + @Query('tableId') tableId: string | undefined, @Res({ passthrough: true }) response: Response ): Promise { await this.prepareRestoreTableCanary(trashId, response); if (this.cls.get('useV2')) { return await this.trashService.restoreTrashV2(trashId); } - return await this.trashService.restoreTrash(trashId); + return await this.trashService.restoreTrash(trashId, tableId); } @Delete('reset-items') diff --git a/apps/nestjs-backend/src/features/trash/trash.service.ts b/apps/nestjs-backend/src/features/trash/trash.service.ts index c367c4e84a..56f6056010 100644 --- a/apps/nestjs-backend/src/features/trash/trash.service.ts +++ b/apps/nestjs-backend/src/features/trash/trash.service.ts @@ -28,6 +28,7 @@ import { ClsService } from 'nestjs-cls'; import type { ICreateFieldsOperation } from '../../cache/types'; import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; import { CustomHttpException } from '../../custom.exception'; +import { DataDbClientManager } from '../../global/data-db-client-manager.service'; import { META_KNEX } from '../../global/knex'; import type { IPerformanceCacheStore } from '../../performance-cache'; import { PerformanceCacheService } from '../../performance-cache'; @@ -51,12 +52,22 @@ import { resolveV2TrashRecordDisplayName } from './v2-trash-record-name'; type IRecordTrashSnapshot = IDeleteRecordsPayload['records'][number]; +type ITrashDataPrisma = { + tableTrash: DataPrismaService['tableTrash']; + recordTrash: DataPrismaService['recordTrash']; +}; + +type IScopedTrashDataPrisma = ITrashDataPrisma & { + txClient?: () => ITrashDataPrisma; + $tx?: DataPrismaService['$tx']; + $transaction?: DataPrismaService['$transaction']; +}; + @Injectable() export class TrashService { constructor( protected readonly performanceCacheService: PerformanceCacheService, protected readonly prismaService: PrismaService, - protected readonly dataPrismaService: DataPrismaService, protected readonly cls: ClsService, protected readonly userService: UserService, protected readonly permissionService: PermissionService, @@ -71,10 +82,42 @@ export class TrashService { protected readonly v2ContainerService: V2ContainerService, protected readonly v2ExecutionContextFactory: V2ExecutionContextFactory, protected readonly canaryService: CanaryService, + protected readonly dataDbClientManager: DataDbClientManager, @ThresholdConfig() protected readonly thresholdConfig: IThresholdConfig, @InjectModel(META_KNEX) protected readonly knex: Knex ) {} + private getTrashDataPrismaExecutor(prisma: IScopedTrashDataPrisma): ITrashDataPrisma { + return prisma.txClient?.() ?? prisma; + } + + private async trashDataPrismaForTable(tableId: string): Promise { + return (await this.dataDbClientManager.dataPrismaForTable(tableId, { + useTransaction: true, + })) as IScopedTrashDataPrisma; + } + + private async trashDataPrismaTransactionForTable( + tableId: string, + fn: (prisma: ITrashDataPrisma) => Promise + ): Promise { + const prisma = await this.trashDataPrismaForTable(tableId); + + if (prisma.$tx) { + return await prisma.$tx(fn, { + timeout: this.thresholdConfig.bigTransactionTimeout, + }); + } + + if (prisma.$transaction) { + return await prisma.$transaction(fn, { + timeout: this.thresholdConfig.bigTransactionTimeout, + }); + } + + return await fn(this.getTrashDataPrismaExecutor(prisma)); + } + async getAuthorizedSpacesAndBases() { const userId = this.cls.get('user.id'); const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); @@ -273,11 +316,11 @@ export class TrashService { } try { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const tableQueryService = container.resolve( v2CoreTokens.tableQueryService ); - const queryContext = await this.v2ExecutionContextFactory.createContext(); + const queryContext = await this.v2ExecutionContextFactory.createContext(container); const tableResult = await tableQueryService.getById(queryContext, tableIdResult.value); return tableResult.isOk() ? tableResult.value : null; @@ -393,7 +436,10 @@ export class TrashService { }, {} as IResourceMapVo); } case TableTrashType.Record: { - const recordList = await this.dataPrismaService.recordTrash.findMany({ + const dataPrisma = this.getTrashDataPrismaExecutor( + await this.trashDataPrismaForTable(tableId) + ); + const recordList = await dataPrisma.recordTrash.findMany({ where: { tableId, recordId: { in: resourceIds } }, select: { recordId: true, @@ -428,7 +474,8 @@ export class TrashService { true ); - const list = await this.dataPrismaService.tableTrash.findMany({ + const dataPrisma = this.getTrashDataPrismaExecutor(await this.trashDataPrismaForTable(tableId)); + const list = await dataPrisma.tableTrash.findMany({ where: { tableId, }, @@ -756,15 +803,29 @@ export class TrashService { } } - async restoreTableResource(trashId: string) { + async restoreTableResource(trashId: string, routedTableId?: string) { const accessTokenId = this.cls.get('accessTokenId'); + if (!routedTableId) { + throw new CustomHttpException( + `Table id is required to restore table trash ${trashId}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.tableNotFound', + }, + } + ); + } + const lookupDataPrisma = this.getTrashDataPrismaExecutor( + await this.trashDataPrismaForTable(routedTableId) + ); const { tableId, resourceType, snapshot: originSnapshot, createdTime, - } = await this.dataPrismaService.tableTrash + } = await lookupDataPrisma.tableTrash .findUniqueOrThrow({ where: { id: trashId }, select: { @@ -785,6 +846,9 @@ export class TrashService { } ); }); + const dataPrisma = routedTableId + ? lookupDataPrisma + : this.getTrashDataPrismaExecutor(await this.trashDataPrismaForTable(tableId)); await this.permissionService.validPermissions( tableId, @@ -829,7 +893,7 @@ export class TrashService { createdTime: true; }; }>; - const recordTrashRows = await this.dataPrismaService.recordTrash.findMany({ + const recordTrashRows = await dataPrisma.recordTrash.findMany({ where: { tableId, recordId: { in: recordIds } }, select: { id: true, @@ -871,19 +935,14 @@ export class TrashService { }, true ); - await this.dataPrismaService.$tx( - async (prisma) => { - await prisma.recordTrash.deleteMany({ - where: { id: { in: matchedRecordTrashRows.map(({ id }) => id) } }, - }); - await prisma.tableTrash.delete({ - where: { id: trashId }, - }); - }, - { - timeout: this.thresholdConfig.bigTransactionTimeout, - } - ); + await this.trashDataPrismaTransactionForTable(tableId, async (prisma) => { + await prisma.recordTrash.deleteMany({ + where: { id: { in: matchedRecordTrashRows.map(({ id }) => id) } }, + }); + await prisma.tableTrash.delete({ + where: { id: trashId }, + }); + }); return; } default: @@ -898,7 +957,7 @@ export class TrashService { ); } - await this.dataPrismaService.tableTrash.delete({ + await dataPrisma.tableTrash.delete({ where: { id: trashId }, }); } @@ -969,9 +1028,9 @@ export class TrashService { }; } - async restoreTrash(trashId: string) { + async restoreTrash(trashId: string, tableId?: string) { if (trashId.startsWith(IdPrefix.Operation)) { - return await this.restoreTableResource(trashId); + return await this.restoreTableResource(trashId, tableId); } await this.prismaService.$tx(async (prisma) => { @@ -1066,7 +1125,8 @@ export class TrashService { true ); - const deletedList = await this.dataPrismaService.tableTrash.findMany({ + const dataPrisma = this.getTrashDataPrismaExecutor(await this.trashDataPrismaForTable(tableId)); + const deletedList = await dataPrisma.tableTrash.findMany({ where: { tableId }, select: { resourceType: true, snapshot: true }, }); @@ -1117,7 +1177,7 @@ export class TrashService { }); }); - await this.dataPrismaService.$tx(async (prisma) => { + await this.trashDataPrismaTransactionForTable(tableId, async (prisma) => { await prisma.recordTrash.deleteMany({ where: { tableId }, }); diff --git a/apps/nestjs-backend/src/features/trash/v2-record-trash.service.ts b/apps/nestjs-backend/src/features/trash/v2-record-trash.service.ts index aacff76fe7..d3359bc387 100644 --- a/apps/nestjs-backend/src/features/trash/v2-record-trash.service.ts +++ b/apps/nestjs-backend/src/features/trash/v2-record-trash.service.ts @@ -59,7 +59,7 @@ export class V2RecordTrashService { return; } - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const db = container.resolve(v2DataDbTokens.db) as TrashDbClient; const recordIds = records.map((record) => record.id); const createdTime = new Date(); diff --git a/apps/nestjs-backend/src/features/trash/v2-table-trash.service.spec.ts b/apps/nestjs-backend/src/features/trash/v2-table-trash.service.spec.ts index 2167c417ba..908c09583f 100644 --- a/apps/nestjs-backend/src/features/trash/v2-table-trash.service.spec.ts +++ b/apps/nestjs-backend/src/features/trash/v2-table-trash.service.spec.ts @@ -84,6 +84,8 @@ const createV2ContainerService = () => { selectFrom: vi.fn().mockReturnValue(selectQuery), }; const dataDb = { + deleteFrom: vi.fn().mockReturnValue(deleteQuery), + insertInto: vi.fn().mockReturnValue(insertQuery), transaction: vi.fn(() => ({ execute: vi.fn(async () => undefined), })), @@ -108,6 +110,7 @@ const createV2ContainerService = () => { selectQuery, service: { getContainer: vi.fn().mockResolvedValue(container), + getContainerForTable: vi.fn().mockResolvedValue(container), }, }; }; @@ -117,6 +120,7 @@ describe('V2TableTrashedProjection', () => { const deletedTime = new Date('2026-03-12T00:00:00.000Z'); const { db, + dataDb, deleteQuery, insertQuery, selectQuery, @@ -152,12 +156,29 @@ describe('V2TableTrashedProjection', () => { deleted_time: deletedTime, deleted_by: 'usrTestUserId', }); + expect(v2ContainerService.getContainerForTable).toHaveBeenCalledWith('tblaaaaaaaaaaaaaaaa'); + expect(dataDb.deleteFrom).toHaveBeenCalledWith('table_trash'); + expect(dataDb.insertInto).toHaveBeenCalledWith('table_trash'); + expect(insertQuery.values).toHaveBeenCalledWith({ + id: expect.any(String), + table_id: 'tblaaaaaaaaaaaaaaaa', + resource_type: ResourceType.Table, + snapshot: JSON.stringify({ + tableId: 'tblaaaaaaaaaaaaaaaa', + baseId: 'bseaaaaaaaaaaaaaaaa', + name: 'Trash Me', + fieldIds: [], + viewIds: [], + }), + created_by: 'usrTestUserId', + created_time: deletedTime, + }); }); }); describe('V2TableRestoredProjection', () => { it('removes a table trash entry after restore', async () => { - const { db, deleteQuery, service: v2ContainerService } = createV2ContainerService(); + const { db, dataDb, deleteQuery, service: v2ContainerService } = createV2ContainerService(); const projection = new V2TableRestoredProjection(v2ContainerService as never); const context = { actorId: ActorId.create('usrTestUserId')._unsafeUnwrap(), @@ -176,12 +197,17 @@ describe('V2TableRestoredProjection', () => { expect(db.deleteFrom).toHaveBeenCalledWith('trash'); expect(deleteQuery.where).toHaveBeenNthCalledWith(1, 'resource_id', '=', 'tblaaaaaaaaaaaaaaaa'); expect(deleteQuery.where).toHaveBeenNthCalledWith(2, 'resource_type', '=', ResourceType.Table); + expect(v2ContainerService.getContainerForTable).toHaveBeenCalledWith('tblaaaaaaaaaaaaaaaa'); + expect(dataDb.deleteFrom).toHaveBeenCalledWith('table_trash'); }); }); describe('V2RecordTrashService', () => { it('persists deleted records through the v2 Kysely db transaction', async () => { const operations: Array<{ table: string; values: unknown }> = []; + type ITrashTransaction = { + insertInto: ReturnType; + }; const trx = { insertInto: vi.fn((table: string) => ({ values: (values: unknown) => ({ @@ -194,10 +220,10 @@ describe('V2RecordTrashService', () => { }), }), })), - }; + } satisfies ITrashTransaction; const db = { transaction: vi.fn(() => ({ - execute: async (callback: (trx: typeof trx) => Promise) => callback(trx), + execute: async (callback: (trx: ITrashTransaction) => Promise) => callback(trx), })), }; const container = { @@ -209,7 +235,7 @@ describe('V2RecordTrashService', () => { }), }; const v2ContainerService = { - getContainer: vi.fn().mockResolvedValue(container), + getContainerForTable: vi.fn().mockResolvedValue(container), }; const service = new V2RecordTrashService(v2ContainerService as never); const tracer = new FakeTracer(); @@ -231,7 +257,7 @@ describe('V2RecordTrashService', () => { await service.persistDeletedRecords(payload, { tracer } as Pick); - expect(v2ContainerService.getContainer).toHaveBeenCalled(); + expect(v2ContainerService.getContainerForTable).toHaveBeenCalledWith('tblaaaaaaaaaaaaaaaa'); expect(db.transaction).toHaveBeenCalled(); expect(operations).toHaveLength(2); expect(operations[0]).toEqual({ diff --git a/apps/nestjs-backend/src/features/trash/v2-table-trash.service.ts b/apps/nestjs-backend/src/features/trash/v2-table-trash.service.ts index e6a1de793d..b5cb59b919 100644 --- a/apps/nestjs-backend/src/features/trash/v2-table-trash.service.ts +++ b/apps/nestjs-backend/src/features/trash/v2-table-trash.service.ts @@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'; import type { IRecord } from '@teable/core'; import { generateOperationId } from '@teable/core'; import { ResourceType } from '@teable/openapi'; -import { v2MetaDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { v2DataDbTokens, v2MetaDbTokens } from '@teable/v2-adapter-db-postgres-pg'; import { ProjectionHandler, RecordsDeleted, @@ -45,6 +45,19 @@ type IAttachmentsTableDb = V1TeableDatabase & { }; /* eslint-enable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/naming-convention */ +type ITableTrashDataDb = V1TeableDatabase & { + table_trash: { + id: string; + table_id: string; + resource_type: string; + snapshot: string; + created_by: string; + created_time: Date; + }; +}; +/* eslint-enable @typescript-eslint/naming-convention */ + @ProjectionHandler(RecordsDeleted) export class V2RecordsDeletedTableTrashProjection implements IEventHandler { constructor(private readonly v2RecordTrashService: V2RecordTrashService) {} @@ -209,6 +222,34 @@ export class V2TableTrashedProjection implements IEventHandler { }) .execute(); + const dataContainer = await this.v2ContainerService.getContainerForTable( + event.tableId.toString() + ); + const dataDb = dataContainer.resolve>(v2DataDbTokens.db); + await dataDb + .deleteFrom('table_trash') + .where('table_id', '=', event.tableId.toString()) + .where('resource_type', '=', ResourceType.Table) + .execute(); + + await dataDb + .insertInto('table_trash') + .values({ + id: nanoid(), + table_id: event.tableId.toString(), + resource_type: ResourceType.Table, + snapshot: JSON.stringify({ + tableId: event.tableId.toString(), + baseId: event.baseId.toString(), + name: event.tableName.toString(), + fieldIds: event.fieldIds.map((fieldId) => fieldId.toString()), + viewIds: event.viewIds.map((viewId) => viewId.toString()), + }), + created_by: context.actorId.toString(), + created_time: table.deleted_time, + }) + .execute(); + return ok(undefined); } } @@ -229,6 +270,16 @@ export class V2TableRestoredProjection implements IEventHandler { .where('resource_type', '=', ResourceType.Table) .execute(); + const dataContainer = await this.v2ContainerService.getContainerForTable( + event.tableId.toString() + ); + const dataDb = dataContainer.resolve>(v2DataDbTokens.db); + await dataDb + .deleteFrom('table_trash') + .where('table_id', '=', event.tableId.toString()) + .where('resource_type', '=', ResourceType.Table) + .execute(); + return ok(undefined); } } diff --git a/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts index 13ce9c36bf..89a48f1acf 100644 --- a/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts +++ b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts @@ -306,9 +306,9 @@ export class UndoRedoService { mode: 'undo' | 'redo' ): Promise | undefined> { try { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); context.windowId = windowId; const commandResult = @@ -393,11 +393,11 @@ export class UndoRedoService { return; } - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const stackService = container.resolve( v2CoreTokens.undoRedoService ); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); context.windowId = windowId; const replayContext = toUndoRedoStackReplayContext(context); diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/delete-fields.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/delete-fields.operation.ts index a561e90154..c2897a3156 100644 --- a/apps/nestjs-backend/src/features/undo-redo/operations/delete-fields.operation.ts +++ b/apps/nestjs-backend/src/features/undo-redo/operations/delete-fields.operation.ts @@ -2,18 +2,31 @@ import { FieldKeyType } from '@teable/core'; import type { DataPrismaService } from '@teable/db-data-prisma'; import type { IDeleteFieldsOperation } from '../../../cache/types'; import { OperationName } from '../../../cache/types'; +import type { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import type { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; import type { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; import type { ICreateFieldsPayload } from './create-fields.operation'; export type IDeleteFieldsPayload = ICreateFieldsPayload & { operationId: string }; + +type IScopedDataPrismaService = DataPrismaService & { + txClient?: () => DataPrismaService; +}; + export class DeleteFieldsOperation { constructor( private readonly fieldOpenApiService: FieldOpenApiService, private readonly recordOpenApiService: RecordOpenApiService, - private readonly dataPrismaService: DataPrismaService + private readonly dataDbClientManager: DataDbClientManager ) {} + private async dataPrismaForTable(tableId: string): Promise { + const dataPrisma = (await this.dataDbClientManager.dataPrismaForTable(tableId, { + useTransaction: true, + })) as IScopedDataPrismaService; + return (dataPrisma.txClient?.() ?? dataPrisma) as DataPrismaService; + } + async event2Operation(payload: IDeleteFieldsPayload): Promise { return { name: OperationName.DeleteFields, @@ -32,14 +45,17 @@ export class DeleteFieldsOperation { const { params, result, operationId = '' } = operation; const { tableId } = params; const { fields, records } = result; + const dataPrisma = await this.dataPrismaForTable(tableId); - const count = await this.dataPrismaService.tableTrash.count({ + const count = await dataPrisma.tableTrash.count({ where: { id: operationId }, }); if (operationId && Number(count) === 0) return operation; - await this.fieldOpenApiService.createFields(tableId, fields); + await this.fieldOpenApiService.createFields(tableId, fields, undefined, { + restoreViewOrder: true, + }); if (records) { await this.recordOpenApiService.updateRecords(tableId, { @@ -49,7 +65,7 @@ export class DeleteFieldsOperation { } if (operationId) { - await this.dataPrismaService.tableTrash.delete({ + await dataPrisma.tableTrash.delete({ where: { id: operationId }, }); } diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/delete-records.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/delete-records.operation.ts index 9a16777a21..748fc9be8a 100644 --- a/apps/nestjs-backend/src/features/undo-redo/operations/delete-records.operation.ts +++ b/apps/nestjs-backend/src/features/undo-redo/operations/delete-records.operation.ts @@ -4,6 +4,7 @@ import type { DataPrismaService } from '@teable/db-data-prisma'; import type { IDeleteRecordsOperation } from '../../../cache/types'; import { OperationName } from '../../../cache/types'; import type { IThresholdConfig } from '../../../configs/threshold.config'; +import type { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import type { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; export interface IDeleteRecordsPayload { @@ -17,10 +18,42 @@ export interface IDeleteRecordsPayload { export class DeleteRecordsOperation { constructor( private readonly recordOpenApiService: RecordOpenApiService, - private readonly dataPrismaService: DataPrismaService, - private readonly thresholdConfig: IThresholdConfig + private readonly thresholdConfig: IThresholdConfig, + private readonly dataDbClientManager: DataDbClientManager ) {} + private async dataPrismaForTable(tableId: string): Promise { + return (await this.dataDbClientManager.dataPrismaForTable(tableId, { + useTransaction: true, + })) as DataPrismaService; + } + + private async dataPrismaExecutorForTable(tableId: string): Promise { + const dataPrisma = await this.dataPrismaForTable(tableId); + return (dataPrisma.txClient?.() ?? dataPrisma) as DataPrismaService; + } + + private async dataPrismaTransactionForTable( + tableId: string, + fn: (prisma: DataPrismaService) => Promise + ): Promise { + const dataPrisma = await this.dataPrismaForTable(tableId); + + if (dataPrisma.$tx) { + return await dataPrisma.$tx(fn as never, { + timeout: this.thresholdConfig.bigTransactionTimeout, + }); + } + + if (dataPrisma.$transaction) { + return await dataPrisma.$transaction(fn as never, { + timeout: this.thresholdConfig.bigTransactionTimeout, + }); + } + + return await fn((dataPrisma.txClient?.() ?? dataPrisma) as DataPrismaService); + } + async event2Operation(payload: IDeleteRecordsPayload): Promise { return { name: OperationName.DeleteRecords, @@ -36,8 +69,9 @@ export class DeleteRecordsOperation { async undo(operation: IDeleteRecordsOperation) { const { params, result, operationId = '' } = operation; + const dataPrisma = await this.dataPrismaExecutorForTable(params.tableId); - const count = await this.dataPrismaService.tableTrash.count({ + const count = await dataPrisma.tableTrash.count({ where: { id: operationId }, }); @@ -51,22 +85,17 @@ export class DeleteRecordsOperation { if (operationId) { const recordIds = result.records.map((record) => record.id); - await this.dataPrismaService.$tx( - async (prisma) => { - await prisma.tableTrash.delete({ - where: { id: operationId }, - }); - await prisma.recordTrash.deleteMany({ - where: { - tableId: params.tableId, - recordId: { in: recordIds }, - }, - }); - }, - { - timeout: this.thresholdConfig.bigTransactionTimeout, - } - ); + await this.dataPrismaTransactionForTable(params.tableId, async (prisma) => { + await prisma.tableTrash.delete({ + where: { id: operationId }, + }); + await prisma.recordTrash.deleteMany({ + where: { + tableId: params.tableId, + recordId: { in: recordIds }, + }, + }); + }); } return operation; diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/delete-trash-routing.spec.ts b/apps/nestjs-backend/src/features/undo-redo/operations/delete-trash-routing.spec.ts index e343be3831..6614e82b86 100644 --- a/apps/nestjs-backend/src/features/undo-redo/operations/delete-trash-routing.spec.ts +++ b/apps/nestjs-backend/src/features/undo-redo/operations/delete-trash-routing.spec.ts @@ -18,10 +18,13 @@ describe('trash-backed undo operations', () => { delete: vi.fn().mockResolvedValue(undefined), }, }; + const dataDbClientManager = { + dataPrismaForTable: vi.fn().mockResolvedValue(dataPrismaService), + }; const operation = new DeleteFieldsOperation( fieldOpenApiService as never, recordOpenApiService as never, - dataPrismaService as never + dataDbClientManager as never ); await operation.undo({ @@ -37,9 +40,15 @@ describe('trash-backed undo operations', () => { expect(dataPrismaService.tableTrash.count).toHaveBeenCalledWith({ where: { id: 'otrash1' }, }); - expect(fieldOpenApiService.createFields).toHaveBeenCalledWith('tbl1', [ - { id: 'fld1', name: 'Name' }, - ]); + expect(dataDbClientManager.dataPrismaForTable).toHaveBeenCalledWith('tbl1', { + useTransaction: true, + }); + expect(fieldOpenApiService.createFields).toHaveBeenCalledWith( + 'tbl1', + [{ id: 'fld1', name: 'Name' }], + undefined, + { restoreViewOrder: true } + ); expect(recordOpenApiService.updateRecords).toHaveBeenCalledWith('tbl1', { fieldKeyType: FieldKeyType.Id, records: [{ id: 'rec1' }], @@ -66,10 +75,13 @@ describe('trash-backed undo operations', () => { }); }), }; + const dataDbClientManager = { + dataPrismaForTable: vi.fn().mockResolvedValue(dataPrismaService), + }; const operation = new DeleteRecordsOperation( recordOpenApiService as never, - dataPrismaService as never, - { bigTransactionTimeout: 60_000 } as never + { bigTransactionTimeout: 60_000 } as never, + dataDbClientManager as never ); await operation.undo({ @@ -85,6 +97,9 @@ describe('trash-backed undo operations', () => { fieldKeyType: FieldKeyType.Id, records: [{ id: 'rec1' }, { id: 'rec2' }], }); + expect(dataDbClientManager.dataPrismaForTable).toHaveBeenCalledWith('tbl1', { + useTransaction: true, + }); expect(dataPrismaService.$tx).toHaveBeenCalled(); expect(tableTrashDelete).toHaveBeenCalledWith({ where: { id: 'otrash2' }, @@ -108,10 +123,13 @@ describe('trash-backed undo operations', () => { delete: vi.fn().mockResolvedValue(undefined), }, }; + const dataDbClientManager = { + dataPrismaForTable: vi.fn().mockResolvedValue(dataPrismaService), + }; const operation = new DeleteViewOperation( viewOpenApiService as never, viewService as never, - dataPrismaService as never + dataDbClientManager as never ); await operation.undo({ diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/delete-view.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/delete-view.operation.ts index 11ba0a4022..f5b0984054 100644 --- a/apps/nestjs-backend/src/features/undo-redo/operations/delete-view.operation.ts +++ b/apps/nestjs-backend/src/features/undo-redo/operations/delete-view.operation.ts @@ -1,6 +1,7 @@ import type { DataPrismaService } from '@teable/db-data-prisma'; import type { IDeleteViewOperation } from '../../../cache/types'; import { OperationName } from '../../../cache/types'; +import type { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import type { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; import type { ViewService } from '../../view/view.service'; @@ -12,13 +13,24 @@ export interface IDeleteViewPayload { userId: string; } +type IScopedDataPrismaService = DataPrismaService & { + txClient?: () => DataPrismaService; +}; + export class DeleteViewOperation { constructor( private readonly viewOpenApiService: ViewOpenApiService, private readonly viewService: ViewService, - private readonly dataPrismaService: DataPrismaService + private readonly dataDbClientManager: DataDbClientManager ) {} + private async dataPrismaForTable(tableId: string): Promise { + const dataPrisma = (await this.dataDbClientManager.dataPrismaForTable(tableId, { + useTransaction: true, + })) as IScopedDataPrismaService; + return (dataPrisma.txClient?.() ?? dataPrisma) as DataPrismaService; + } + async event2Operation(payload: IDeleteViewPayload): Promise { return { name: OperationName.DeleteView, @@ -33,8 +45,9 @@ export class DeleteViewOperation { async undo(operation: IDeleteViewOperation) { const { params, operationId = '' } = operation; const { tableId, viewId } = params; + const dataPrisma = await this.dataPrismaForTable(tableId); - const count = await this.dataPrismaService.tableTrash.count({ + const count = await dataPrisma.tableTrash.count({ where: { id: operationId }, }); @@ -43,7 +56,7 @@ export class DeleteViewOperation { await this.viewService.restoreView(tableId, viewId); if (operationId) { - await this.dataPrismaService.tableTrash.delete({ + await dataPrisma.tableTrash.delete({ where: { id: operationId }, }); } diff --git a/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-operation.service.ts b/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-operation.service.ts index d674eaa89e..c6eaf601fb 100644 --- a/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-operation.service.ts +++ b/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-operation.service.ts @@ -3,11 +3,11 @@ import { Injectable } from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; import { assertNever } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { IUndoRedoOperation } from '../../../cache/types'; import { OperationName } from '../../../cache/types'; import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.config'; import { Events, IEventRawContext } from '../../../event-emitter/events'; +import { DataDbClientManager } from '../../../global/data-db-client-manager.service'; import { FieldOpenApiV2Service } from '../../field/open-api/field-open-api-v2.service'; import { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; import { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; @@ -67,7 +67,7 @@ export class UndoRedoOperationService { private readonly recordService: RecordService, private readonly viewService: ViewService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly dataDbClientManager: DataDbClientManager, private readonly tableDomainQueryService: TableDomainQueryService, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig ) { @@ -78,8 +78,8 @@ export class UndoRedoOperationService { ); this.deleteRecords = new DeleteRecordsOperation( this.recordOpenApiService, - this.dataPrismaService, - this.thresholdConfig + this.thresholdConfig, + this.dataDbClientManager ); this.updateRecords = new UpdateRecordsOperation(this.recordOpenApiService, this.recordService); this.updateRecordsOrder = new UpdateRecordsOrderOperation(this.viewOpenApiService); @@ -90,7 +90,7 @@ export class UndoRedoOperationService { this.deleteFields = new DeleteFieldsOperation( this.fieldOpenApiService, this.recordOpenApiService, - this.dataPrismaService + this.dataDbClientManager ); this.convertField = new ConvertFieldOperation( this.fieldOpenApiService, @@ -105,7 +105,7 @@ export class UndoRedoOperationService { this.deleteView = new DeleteViewOperation( this.viewOpenApiService, this.viewService, - this.dataPrismaService + this.dataDbClientManager ); this.createView = new CreateViewOperation(this.viewOpenApiService, this.viewService); this.updateView = new UpdateViewOperation(this.viewOpenApiService); diff --git a/apps/nestjs-backend/src/features/user/user.service.ts b/apps/nestjs-backend/src/features/user/user.service.ts index aedf63cffd..d0105cf40f 100644 --- a/apps/nestjs-backend/src/features/user/user.service.ts +++ b/apps/nestjs-backend/src/features/user/user.service.ts @@ -54,6 +54,23 @@ export class UserService { ); } + async getUsersByIdsOrEmails(params: { ids?: string[]; emails?: string[] }) { + const { ids = [], emails = [] } = params; + const conditions = []; + if (ids.length > 0) conditions.push({ id: { in: ids } }); + if (emails.length > 0) conditions.push({ email: { in: emails.map((e) => e.toLowerCase()) } }); + if (conditions.length === 0) return []; + + const users = await this.prismaService.user.findMany({ + where: { OR: conditions, deletedTime: null }, + }); + return users.map((u) => ({ + ...u, + avatar: u.avatar && getPublicFullStorageUrl(u.avatar), + notifyMeta: u.notifyMeta ? (JSON.parse(u.notifyMeta) as IUserNotifyMeta) : null, + })); + } + async getUserByEmail(email: string) { return await this.prismaService.txClient().user.findUnique({ where: { email: email.toLowerCase(), deletedTime: null }, diff --git a/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts index 85ee0f03da..62f647f4d5 100644 --- a/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts +++ b/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts @@ -17,6 +17,8 @@ vi.mock('../attachments/attachments-storage.service', () => ({ import { CacheService } from '../../cache/cache.service'; import { thresholdConfig } from '../../configs/threshold.config'; +import { DataDbClientManager } from '../../global/data-db-client-manager.service'; +import { DataDbRuntimeCacheService } from '../../global/data-db-runtime-cache.service'; import { ShareDbService } from '../../share-db/share-db.service'; import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; import { V2ContainerService } from './v2-container.service'; @@ -139,6 +141,12 @@ const createService = (providers: InstanceWrapper[] = []) => { getPreviewUrlByPath: vi.fn(), getTableThumbnailUrl: vi.fn(), }; + const dataDbClientManager = { + getDataDatabaseForSpace: vi.fn(), + getDataDatabaseForBase: vi.fn(), + getDataDatabaseForTable: vi.fn(), + }; + const runtimeCache = new DataDbRuntimeCacheService(); const reflector = new Reflector(); const discoveryService = { getProviders: vi.fn().mockReturnValue(providers), @@ -152,7 +160,9 @@ const createService = (providers: InstanceWrapper[] = []) => { attachmentsStorageService as never, { undoExpirationTime: 60, maxUndoStackSize: 20 } as never, reflector, - discoveryService + discoveryService, + dataDbClientManager as never, + runtimeCache ); return { @@ -161,6 +171,8 @@ const createService = (providers: InstanceWrapper[] = []) => { shareDbService, cacheService, attachmentsStorageService, + dataDbClientManager, + runtimeCache, discoveryService, }; }; @@ -176,6 +188,12 @@ const createTestingModule = async (providers: InstanceWrapper[] = []) => { getPreviewUrlByPath: vi.fn(), getTableThumbnailUrl: vi.fn(), }; + const dataDbClientManager = { + getDataDatabaseForSpace: vi.fn(), + getDataDatabaseForBase: vi.fn(), + getDataDatabaseForTable: vi.fn(), + }; + const runtimeCache = new DataDbRuntimeCacheService(); const reflector = new Reflector(); const discoveryService = { getProviders: vi.fn().mockReturnValue(providers), @@ -189,6 +207,8 @@ const createTestingModule = async (providers: InstanceWrapper[] = []) => { { provide: ShareDbService, useValue: shareDbService }, { provide: CacheService, useValue: cacheService }, { provide: AttachmentsStorageService, useValue: attachmentsStorageService }, + { provide: DataDbClientManager, useValue: dataDbClientManager }, + { provide: DataDbRuntimeCacheService, useValue: runtimeCache }, { provide: thresholdConfig.KEY, useValue: { undoExpirationTime: 60, maxUndoStackSize: 20 }, @@ -249,7 +269,7 @@ describe('V2ContainerService', () => { expect(registrar.registerProjections).toHaveBeenCalledTimes(1); }); - it('accepts split meta/data envs without requiring the legacy alias', async () => { + it('uses the meta database as the default data database without a global data env', async () => { const container = createContainerMock(); mocks.createV2NodePgContainer.mockResolvedValue(container); const { service, configService } = createService(); @@ -258,9 +278,6 @@ describe('V2ContainerService', () => { if (key === 'PRISMA_META_DATABASE_URL') { return 'postgres://meta-db'; } - if (key === 'PRISMA_DATA_DATABASE_URL') { - return 'postgres://data-db'; - } return undefined; }); @@ -269,7 +286,7 @@ describe('V2ContainerService', () => { expect(mocks.createV2NodePgContainer).toHaveBeenCalledWith( expect.objectContaining({ metaConnectionString: 'postgres://meta-db', - dataConnectionString: 'postgres://data-db', + dataConnectionString: 'postgres://meta-db', }) ); }); @@ -319,6 +336,39 @@ describe('V2ContainerService', () => { ); }); + it('creates a scoped container from the base data database binding', async () => { + const defaultContainer = createContainerMock(); + const scopedContainer = createContainerMock(); + mocks.createV2NodePgContainer + .mockResolvedValueOnce(defaultContainer) + .mockResolvedValueOnce(scopedContainer); + const { service, configService, dataDbClientManager } = createService(); + + configService.get.mockImplementation((key: string) => + key === 'PRISMA_META_DATABASE_URL' ? 'postgres://meta-db' : undefined + ); + dataDbClientManager.getDataDatabaseForBase.mockResolvedValue({ + cacheKey: 'dcnxxx', + url: 'postgres://space-data-db', + isMetaFallback: false, + connectionId: 'dcnxxx', + }); + + await expect(service.getContainer()).resolves.toBe(defaultContainer); + await expect(service.getContainerForBase('bsexxx')).resolves.toBe(scopedContainer); + await expect(service.getContainerForBase('bsexxx')).resolves.toBe(scopedContainer); + + expect(dataDbClientManager.getDataDatabaseForBase).toHaveBeenCalledWith('bsexxx'); + expect(mocks.createV2NodePgContainer).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + metaConnectionString: 'postgres://meta-db', + dataConnectionString: 'postgres://space-data-db', + }) + ); + expect(mocks.createV2NodePgContainer).toHaveBeenCalledTimes(2); + }); + it('falls back to DATABASE_URL when neither meta nor legacy alias is configured', async () => { const container = createContainerMock(); mocks.createV2NodePgContainer.mockResolvedValue(container); diff --git a/apps/nestjs-backend/src/features/v2/v2-container.service.ts b/apps/nestjs-backend/src/features/v2/v2-container.service.ts index 235bdbae2a..d13a77631d 100644 --- a/apps/nestjs-backend/src/features/v2/v2-container.service.ts +++ b/apps/nestjs-backend/src/features/v2/v2-container.service.ts @@ -18,6 +18,11 @@ import { registerV2ImportServices } from '@teable/v2-import'; import { PinoLogger } from 'nestjs-pino'; import { CacheService } from '../../cache/cache.service'; import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { DataDbClientManager } from '../../global/data-db-client-manager.service'; +import { + DataDbRuntimeCacheService, + V2_CONTAINER_CACHE_NAMESPACE, +} from '../../global/data-db-runtime-cache.service'; import { ShareDbService } from '../../share-db/share-db.service'; import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; import { V2AttachmentUrlSignerService } from './v2-attachment-url-signer.service'; @@ -41,7 +46,6 @@ const resolvePositiveInteger = (value: unknown): number | undefined => { @Injectable() export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestroy { private readonly logger = new Logger(V2ContainerService.name); - private containerPromise?: Promise; constructor( private readonly configService: ConfigService, @@ -51,7 +55,9 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr private readonly attachmentsStorageService: AttachmentsStorageService, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, private readonly reflector: Reflector, - private readonly discoveryService: DiscoveryService + private readonly discoveryService: DiscoveryService, + private readonly dataDbClientManager: DataDbClientManager, + private readonly runtimeCache: DataDbRuntimeCacheService ) {} async onApplicationBootstrap(): Promise { @@ -59,23 +65,46 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr } async getContainer(): Promise { - if (!this.containerPromise) { - this.containerPromise = this.createContainer().catch((error) => { - this.containerPromise = undefined; - throw error; - }); - } + return await this.getContainerForDataDb('default', this.getMetaConnectionString()); + } + + async getContainerForSpace(spaceId: string): Promise { + const dataDb = await this.dataDbClientManager.getDataDatabaseForSpace(spaceId); + return await this.getContainerForDataDb(dataDb.cacheKey, dataDb.url); + } + + async getContainerForBase(baseId: string): Promise { + const dataDb = await this.dataDbClientManager.getDataDatabaseForBase(baseId); + return await this.getContainerForDataDb(dataDb.cacheKey, dataDb.url); + } - return this.containerPromise; + async getContainerForTable(tableId: string): Promise { + const dataDb = await this.dataDbClientManager.getDataDatabaseForTable(tableId); + return await this.getContainerForDataDb(dataDb.cacheKey, dataDb.url); } - private async createContainer(): Promise { - const metaConnectionString = + private async getContainerForDataDb( + cacheKey: string, + dataConnectionString: string + ): Promise { + return await this.runtimeCache.getOrCreate( + V2_CONTAINER_CACHE_NAMESPACE, + cacheKey, + () => this.createContainer(dataConnectionString), + (container) => this.destroyContainer(container) + ); + } + + private getMetaConnectionString(): string { + return ( this.configService.get('PRISMA_META_DATABASE_URL') ?? this.configService.get('PRISMA_DATABASE_URL') ?? - this.configService.getOrThrow('DATABASE_URL'); - const dataConnectionString = - this.configService.get('PRISMA_DATA_DATABASE_URL') ?? metaConnectionString; + this.configService.getOrThrow('DATABASE_URL') + ); + } + + private async createContainer(dataConnectionString: string): Promise { + const metaConnectionString = this.getMetaConnectionString(); const logger = new PinoLoggerAdapter(this.pinoLogger); const tracer = new OpenTelemetryTracer(); const commandBusMiddlewares = [new CommandBusTracingMiddleware()]; @@ -88,7 +117,7 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr this.configService.get('MAX_FREE_ROW_LIMIT') ); - this.logger.log('Initializing shared V2 container'); + this.logger.log('Initializing V2 container'); const container = await createV2NodePgContainer({ metaConnectionString, @@ -141,7 +170,7 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr registrar.registerProjections(container); } - this.logger.log('Shared V2 container initialized'); + this.logger.log('V2 container initialized'); return container; } @@ -182,9 +211,10 @@ export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestr } async onModuleDestroy(): Promise { - if (!this.containerPromise) return; + await this.runtimeCache.deleteByNamespace(V2_CONTAINER_CACHE_NAMESPACE); + } - const container = await this.containerPromise; + private async destroyContainer(container: DependencyContainer): Promise { await this.stopComputedUpdatePolling(container); const closers = Array.from( new Set([ diff --git a/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts b/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts index 0b13313359..803238e70c 100644 --- a/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts +++ b/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts @@ -1,6 +1,7 @@ import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; import type { IExecutionContext, ITracer } from '@teable/v2-core'; import { ActorId, v2CoreTokens, type TableDataSafetyLimitConfig } from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; import { ClsService } from 'nestjs-cls'; import { I18nContext, I18nService } from 'nestjs-i18n'; @@ -24,8 +25,8 @@ export class V2ExecutionContextFactory { * Creates a complete execution context with actorId, tracer, and requestId. * @throws HttpException if user.id is not available or ActorId creation fails */ - async createContext(): Promise { - const container = await this.v2ContainerService.getContainer(); + async createContext(container?: DependencyContainer): Promise { + container ??= await this.v2ContainerService.getContainer(); const tracer = container.resolve(v2CoreTokens.tracer); const tableLimits = container.isRegistered(v2CoreTokens.tableDataSafetyLimits) ? container.resolve(v2CoreTokens.tableDataSafetyLimits) diff --git a/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.spec.ts index 50160c5f66..f311252794 100644 --- a/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.spec.ts +++ b/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.spec.ts @@ -1,7 +1,25 @@ import { ViewOpBuilder } from '@teable/core'; -import { v2DataDbTokens } from '@teable/v2-adapter-db-postgres-pg'; import { describe, expect, it, vi } from 'vitest'; import { V2_FIELD_DELETE_COMPAT_CONTEXT_KEY } from './v2-field-delete-compat.constants'; + +const mockV2Tokens = vi.hoisted(() => ({ + v2DataDbTokens: { + db: Symbol('v2.data.db'), + }, +})); + +vi.mock('@teable/v2-adapter-db-postgres-pg', () => ({ + v2DataDbTokens: mockV2Tokens.v2DataDbTokens, +})); + +vi.mock('./v2-container.service', () => ({ + V2ContainerService: class V2ContainerService {}, +})); + +vi.mock('./v2-view-compat.service', () => ({ + V2ViewCompatService: class V2ViewCompatService {}, +})); + import { V2FieldDeletedCompatProjection } from './v2-field-delete-compat.service'; const createInsertDb = () => { @@ -19,7 +37,7 @@ const createInsertDb = () => { const createV2ContainerService = (db: unknown) => ({ getContainer: vi.fn().mockResolvedValue({ resolve: vi.fn((token: symbol) => { - if (token !== v2DataDbTokens.db) { + if (token !== mockV2Tokens.v2DataDbTokens.db) { throw new Error(`Unexpected token ${String(token)}`); } @@ -58,15 +76,14 @@ describe('V2FieldDeletedCompatProjection', () => { }, }; - const result = await projection.handle( - { - [V2_FIELD_DELETE_COMPAT_CONTEXT_KEY]: compatContext, - } as never, - { - tableId: { toString: () => 'tblCompatTable0001' }, - fieldId: { toString: () => 'fldCompatA00000001' }, - } as never - ); + const executionContext = { + [V2_FIELD_DELETE_COMPAT_CONTEXT_KEY]: compatContext, + } as never; + + const result = await projection.handle(executionContext, { + tableId: { toString: () => 'tblCompatTable0001' }, + fieldId: { toString: () => 'fldCompatA00000001' }, + } as never); expect(result._unsafeUnwrap()).toBeUndefined(); expect(compatContext.completed).toBeUndefined(); @@ -105,21 +122,21 @@ describe('V2FieldDeletedCompatProjection', () => { }, }; - const result = await projection.handle( - { - [V2_FIELD_DELETE_COMPAT_CONTEXT_KEY]: compatContext, - } as never, - { - tableId: { toString: () => 'tblCompatTable0001' }, - fieldId: { toString: () => 'fldCompatA00000001' }, - } as never - ); + const executionContext = { + [V2_FIELD_DELETE_COMPAT_CONTEXT_KEY]: compatContext, + } as never; + + const result = await projection.handle(executionContext, { + tableId: { toString: () => 'tblCompatTable0001' }, + fieldId: { toString: () => 'fldCompatA00000001' }, + } as never); expect(result._unsafeUnwrap()).toBeUndefined(); expect(compatContext.completed).toBe(true); expect(v2ViewCompatService.batchUpdateViewByOps).toHaveBeenCalledWith( 'tblCompatTable0001', - compatContext.frozenFieldOps + compatContext.frozenFieldOps, + executionContext ); expect(db.insertInto).toHaveBeenCalledWith('table_trash'); expect(query.values).toHaveBeenCalledWith({ diff --git a/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.ts b/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.ts index 141dc99798..ce135446d8 100644 --- a/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.ts +++ b/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.ts @@ -76,11 +76,12 @@ export class V2FieldDeletedCompatProjection implements IEventHandler 0) { await this.v2ViewCompatService.batchUpdateViewByOps( compatContext.tableId, - compatContext.frozenFieldOps + compatContext.frozenFieldOps, + context ); } - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(compatContext.tableId); const db = container.resolve>(v2DataDbTokens.db); await db diff --git a/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts b/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts index 98dc827a8a..577aac3b0e 100644 --- a/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts +++ b/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts @@ -64,9 +64,10 @@ type IRecordHistoryDb = V1TeableDatabase & { }; const getRecordHistoryDb = async ( - v2ContainerService: V2ContainerService + v2ContainerService: V2ContainerService, + tableId: string ): Promise> => { - const container = await v2ContainerService.getContainer(); + const container = await v2ContainerService.getContainerForTable(tableId); return container.resolve>(v2DataDbTokens.db); }; @@ -315,7 +316,7 @@ export class V2RecordUpdatedHistoryProjection implements IEventHandler ({ + captureException: vi.fn(), + withScope: vi.fn((callback: (scope: typeof sentryScope) => void) => callback(sentryScope)), +})); + const operation = (id: string): SchemaOperationRecord => ({ id, type: 'table.create', status: 'running', phase: 'running', - target: { resourceType: 'table', resourceId: 'tblSchemaOpRunner' }, + target: { + resourceType: 'table', + resourceId: 'tblSchemaOpRunner', + baseId: 'bseSchemaOpRunner', + tableId: 'tblSchemaOpRunner', + }, idempotencyKey: `schema-op:${id}`, attempts: 0, maxAttempts: 8, @@ -80,6 +97,11 @@ const createService = ({ describe('V2SchemaOperationRunnerService', () => { beforeEach(() => { vi.useFakeTimers(); + vi.mocked(Sentry.captureException).mockClear(); + vi.mocked(Sentry.withScope).mockClear(); + sentryScope.setContext.mockClear(); + sentryScope.setLevel.mockClear(); + sentryScope.setTag.mockClear(); }); afterEach(() => { @@ -122,6 +144,50 @@ describe('V2SchemaOperationRunnerService', () => { service.onModuleDestroy(); }); + it('captures terminal schema operation failures to Sentry with operation context', async () => { + const failure = domainError.notImplemented({ + code: 'schema_operation.repair_not_supported', + message: 'Only missing-column table updates can be repaired automatically', + }); + const { service } = createService({ + results: [ + okResult({ + status: 'failed', + operation: { + ...operation('sgoTerminal'), + status: 'dead', + phase: 'error', + attempts: 2, + lastError: 'Only missing-column table updates can be repaired automatically', + }, + terminal: true, + retryable: false, + error: failure, + originalLastError: 'Unexpected unit of work error: error: too many range table entries', + }), + okResult({ status: 'idle', reason: 'empty' }), + ], + }); + + await service.onApplicationBootstrap(); + await vi.advanceTimersByTimeAsync(0); + + expect(Sentry.captureException).toHaveBeenCalledTimes(1); + expect(sentryScope.setTag).toHaveBeenCalledWith('feature', 'v2-schema-operation-runner'); + expect(sentryScope.setTag).toHaveBeenCalledWith('table.id', 'tblSchemaOpRunner'); + expect(sentryScope.setTag).toHaveBeenCalledWith('schema_operation.id', 'sgoTerminal'); + expect(sentryScope.setContext).toHaveBeenCalledWith( + 'schema_operation', + expect.objectContaining({ + id: 'sgoTerminal', + originalLastError: 'Unexpected unit of work error: error: too many range table entries', + runnerError: 'Only missing-column table updates can be repaired automatically', + }) + ); + + service.onModuleDestroy(); + }); + it('does not resolve the v2 container when disabled', async () => { const { service, v2ContainerService, runner } = createService({ config: { V2_SCHEMA_OPERATION_RUNNER_ENABLED: 'false' }, diff --git a/apps/nestjs-backend/src/features/v2/v2-schema-operation-runner.service.ts b/apps/nestjs-backend/src/features/v2/v2-schema-operation-runner.service.ts index 490b4a0c52..0edc410956 100644 --- a/apps/nestjs-backend/src/features/v2/v2-schema-operation-runner.service.ts +++ b/apps/nestjs-backend/src/features/v2/v2-schema-operation-runner.service.ts @@ -1,6 +1,7 @@ import type { OnApplicationBootstrap, OnModuleDestroy } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import * as Sentry from '@sentry/nestjs'; import { ActorId, type IExecutionContext, @@ -189,5 +190,49 @@ export class V2SchemaOperationRunnerService implements OnApplicationBootstrap, O this.logger[level]( `V2 schema operation failed: operationId=${result.operation.id}, terminal=${result.terminal}, retryable=${result.retryable}, error=${result.error.message}` ); + this.captureTerminalFailure(result); + } + + private captureTerminalFailure( + result: Extract + ) { + if (!result.terminal) return; + + const operation = result.operation; + const target = operation.target; + Sentry.withScope((scope) => { + scope.setLevel('error'); + scope.setTag('feature', 'v2-schema-operation-runner'); + scope.setTag('teable.version', 'v2'); + scope.setTag('schema_operation.id', operation.id); + scope.setTag('schema_operation.type', operation.type); + scope.setTag('schema_operation.status', operation.status); + scope.setTag('schema_operation.phase', operation.phase); + scope.setTag('schema_operation.terminal', String(result.terminal)); + scope.setTag('schema_operation.retryable', String(result.retryable)); + if (target.baseId) { + scope.setTag('base.id', target.baseId); + } + if (target.tableId) { + scope.setTag('table.id', target.tableId); + } + scope.setContext('schema_operation', { + id: operation.id, + type: operation.type, + status: operation.status, + phase: operation.phase, + attempts: operation.attempts, + maxAttempts: operation.maxAttempts, + idempotencyKey: operation.idempotencyKey, + target, + lastError: operation.lastError, + originalLastError: result.originalLastError ?? null, + runnerError: result.error.message, + }); + + const error = new Error(result.error.message); + error.name = 'V2SchemaOperationFailure'; + Sentry.captureException(error); + }); } } diff --git a/apps/nestjs-backend/src/features/v2/v2-view-compat.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-view-compat.service.spec.ts index ee59d8f1dc..c3bde87f36 100644 --- a/apps/nestjs-backend/src/features/v2/v2-view-compat.service.spec.ts +++ b/apps/nestjs-backend/src/features/v2/v2-view-compat.service.spec.ts @@ -1,20 +1,74 @@ import { IdPrefix, ViewOpBuilder } from '@teable/core'; -import { v2MetaDbTokens } from '@teable/v2-adapter-db-postgres-pg'; import { describe, expect, it, vi } from 'vitest'; + +const mockV2Tokens = vi.hoisted(() => ({ + v2MetaDbTokens: { + db: Symbol('v2.meta.db'), + }, + v2CoreTokens: { + viewOperationPluginRunner: Symbol('v2.core.viewOperationPluginRunner'), + }, +})); + +vi.mock('@teable/v2-adapter-db-postgres-pg', () => ({ + v2MetaDbTokens: mockV2Tokens.v2MetaDbTokens, +})); + +vi.mock('@teable/v2-core', () => ({ + v2CoreTokens: mockV2Tokens.v2CoreTokens, + ViewOperationKind: { + update: 'update', + }, +})); + +vi.mock('./v2-container.service', () => ({ + V2ContainerService: class V2ContainerService {}, +})); + +vi.mock('./v2-execution-context.factory', () => ({ + V2ExecutionContextFactory: class V2ExecutionContextFactory {}, +})); + import { V2ViewCompatService } from './v2-view-compat.service'; -const createV2ContainerService = (db: unknown) => ({ +const createV2ContainerService = (db: unknown, viewOperationPluginRunner: unknown) => ({ getContainer: vi.fn().mockResolvedValue({ resolve: vi.fn((token: symbol) => { - if (token !== v2MetaDbTokens.db) { - throw new Error(`Unexpected token ${String(token)}`); + if (token === mockV2Tokens.v2MetaDbTokens.db) { + return db; } - return db; + if (token === mockV2Tokens.v2CoreTokens.viewOperationPluginRunner) { + return viewOperationPluginRunner; + } + + throw new Error(`Unexpected token ${String(token)}`); }), }), }); +const okResult = (value: T) => ({ + value, + isErr: () => false, +}); + +const errResult = (error: T) => ({ + error, + isErr: () => true, +}); + +const createViewOperationPluginRunner = (guardResult = okResult(undefined)) => { + const guard = vi.fn().mockResolvedValue(guardResult); + const prepare = vi.fn().mockResolvedValue(okResult({ guard })); + return { guard, prepare }; +}; + +const createV2ContextFactory = () => ({ + createContext: vi.fn().mockResolvedValue({ + actorId: { toString: () => 'usrCompatWriter00001' }, + }), +}); + describe('V2ViewCompatService', () => { it('updates matching views through the v2 db and stores raw ops in cls state', async () => { const executeSelect = vi.fn().mockResolvedValue([{ id: 'viwCompat000000001', version: 3 }]); @@ -33,7 +87,9 @@ describe('V2ViewCompatService', () => { selectFrom: vi.fn().mockReturnValue(selectQuery), updateTable: vi.fn().mockReturnValue(updateQuery), }; - const v2ContainerService = createV2ContainerService(db); + const viewOperationPluginRunner = createViewOperationPluginRunner(); + const v2ContainerService = createV2ContainerService(db, viewOperationPluginRunner); + const v2ContextFactory = createV2ContextFactory(); const clsState = new Map(); const cls = { getId: vi.fn().mockReturnValue('cls-request-id'), @@ -48,7 +104,11 @@ describe('V2ViewCompatService', () => { clsState.set(key, value); }), }; - const service = new V2ViewCompatService(v2ContainerService as never, cls as never); + const service = new V2ViewCompatService( + v2ContainerService as never, + cls as never, + v2ContextFactory as never + ); const ops = [ ViewOpBuilder.editor.setViewProperty.build({ key: 'options', @@ -61,6 +121,19 @@ describe('V2ViewCompatService', () => { viwCompat000000001: ops, }); + expect(viewOperationPluginRunner.prepare).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'update', + payload: { + tableId: 'tblCompatTable0001', + viewId: 'viwCompat000000001', + patch: { options: { frozenFieldId: 'fldNewFrozen00001' } }, + }, + }) + ); + expect(viewOperationPluginRunner.guard).toHaveBeenCalledWith( + expect.objectContaining({ actorId: expect.anything() }) + ); expect(db.selectFrom).toHaveBeenCalledWith('view'); expect(db.updateTable).toHaveBeenCalledWith('view'); expect(updateQuery.set).toHaveBeenCalledWith({ @@ -80,4 +153,58 @@ describe('V2ViewCompatService', () => { v: 3, }); }); + + it('rejects view updates when the v2 view operation plugin reports a limit error', async () => { + const executeSelect = vi.fn().mockResolvedValue([{ id: 'viwCompat000000001', version: 3 }]); + const selectQuery = { + where: vi.fn().mockReturnThis(), + select: vi.fn().mockReturnThis(), + execute: executeSelect, + }; + const updateQuery = { + set: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + execute: vi.fn().mockResolvedValue(undefined), + }; + const db = { + selectFrom: vi.fn().mockReturnValue(selectQuery), + updateTable: vi.fn().mockReturnValue(updateQuery), + }; + const limitError = { + code: 'validation.limit.view_options_max_bytes', + message: 'Table data safety limit exceeded: validation.limit.view_options_max_bytes', + details: { attempted: 16, max: 4 }, + }; + const viewOperationPluginRunner = createViewOperationPluginRunner(errResult(limitError)); + const v2ContainerService = createV2ContainerService(db, viewOperationPluginRunner); + const cls = { + getId: vi.fn().mockReturnValue('cls-request-id'), + get: vi.fn().mockReturnValue(undefined), + set: vi.fn(), + }; + const service = new V2ViewCompatService( + v2ContainerService as never, + cls as never, + createV2ContextFactory() as never + ); + const ops = [ + ViewOpBuilder.editor.setViewProperty.build({ + key: 'options', + oldValue: {}, + newValue: { frozenFieldId: 'fldNewFrozen00001' }, + }), + ]; + + await expect( + service.batchUpdateViewByOps('tblCompatTable0001', { + viwCompat000000001: ops, + }) + ).rejects.toMatchObject({ + data: { + domainCode: 'validation.limit.view_options_max_bytes', + details: { attempted: 16, max: 4 }, + }, + }); + expect(db.updateTable).not.toHaveBeenCalled(); + }); }); diff --git a/apps/nestjs-backend/src/features/v2/v2-view-compat.service.ts b/apps/nestjs-backend/src/features/v2/v2-view-compat.service.ts index 10abcda9ec..12adcf0e1d 100644 --- a/apps/nestjs-backend/src/features/v2/v2-view-compat.service.ts +++ b/apps/nestjs-backend/src/features/v2/v2-view-compat.service.ts @@ -9,6 +9,15 @@ import { type ISetViewPropertyOpContext, } from '@teable/core'; import { v2MetaDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { + v2CoreTokens, + ViewOperationKind, + type DomainError, + type IExecutionContext, + type ViewOperationPayloadViewConfig, + type ViewOperationPluginContext, + type ViewOperationPluginRunner, +} from '@teable/v2-core'; import type { V1TeableDatabase } from '@teable/v2-postgres-schema'; import type { Kysely } from 'kysely'; import { snakeCase } from 'lodash'; @@ -18,6 +27,7 @@ import { CustomHttpException } from '../../custom.exception'; import type { IRawOp, IRawOpMap } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; import { V2ContainerService } from './v2-container.service'; +import { V2ExecutionContextFactory } from './v2-execution-context.factory'; /* eslint-disable @typescript-eslint/naming-convention */ type IV2ViewCompatDb = V1TeableDatabase & { @@ -43,16 +53,20 @@ type IV2ViewCompatDb = V1TeableDatabase & { export class V2ViewCompatService { constructor( private readonly v2ContainerService: V2ContainerService, - private readonly cls: ClsService + private readonly cls: ClsService, + private readonly v2ContextFactory: V2ExecutionContextFactory ) {} - private async getDb(): Promise> { - const container = await this.v2ContainerService.getContainer(); - return container.resolve>(v2MetaDbTokens.db); + private throwDomainError(error: DomainError): never { + throw new CustomHttpException(error.message, HttpErrorCode.VALIDATION_ERROR, { + domainCode: error.code, + domainTags: error.tags, + details: error.details, + }); } private mergeSetViewPropertyByOpContexts(opContexts: ISetViewPropertyOpContext[]) { - const result: Record = {}; + const result: Record = {}; for (const opContext of opContexts) { const { key, newValue } = opContext; const parseResult = viewVoSchema.partial().safeParse({ [key]: newValue }); @@ -69,12 +83,7 @@ export class V2ViewCompatService { } const parsedValue = parseResult.data[key]; - result[key] = - parsedValue == null - ? null - : typeof parsedValue === 'object' - ? JSON.stringify(parsedValue) - : parsedValue; + result[key] = parsedValue == null ? null : parsedValue; } return result; @@ -129,13 +138,38 @@ export class V2ViewCompatService { return rawOpMap; } - async batchUpdateViewByOps(tableId: string, opsMap: { [viewId: string]: IOtOperation[] }) { + private async ensureViewOperation( + runner: ViewOperationPluginRunner, + executionContext: IExecutionContext, + context: ViewOperationPluginContext + ): Promise { + const preparedResult = await runner.prepare(context); + if (preparedResult.isErr()) { + this.throwDomainError(preparedResult.error); + } + + const guardResult = await preparedResult.value.guard(executionContext); + if (guardResult.isErr()) { + this.throwDomainError(guardResult.error); + } + } + + async batchUpdateViewByOps( + tableId: string, + opsMap: { [viewId: string]: IOtOperation[] }, + context?: IExecutionContext + ) { const updatedViewIds = Object.keys(opsMap); if (!updatedViewIds.length) { return; } - const db = await this.getDb(); + const container = await this.v2ContainerService.getContainer(); + const db = container.resolve>(v2MetaDbTokens.db); + const viewOperationPluginRunner = container.resolve( + v2CoreTokens.viewOperationPluginRunner + ); + const executionContext = context ?? (await this.v2ContextFactory.createContext()); const views = await db .selectFrom('view') .where('id', 'in', updatedViewIds) @@ -153,8 +187,22 @@ export class V2ViewCompatService { continue; } + await this.ensureViewOperation(viewOperationPluginRunner, executionContext, { + kind: ViewOperationKind.update, + executionContext, + payload: { + tableId, + viewId: view.id, + patch: properties as ViewOperationPayloadViewConfig, + }, + isTransactionBound: false, + }); + const dbValues = Object.fromEntries( - Object.entries(properties).map(([key, value]) => [snakeCase(key), value]) + Object.entries(properties).map(([key, value]) => [ + snakeCase(key), + value == null ? null : typeof value === 'object' ? JSON.stringify(value) : value, + ]) ); await db diff --git a/apps/nestjs-backend/src/features/v2/v2.controller.ts b/apps/nestjs-backend/src/features/v2/v2.controller.ts index bb4ce75ea0..28bb61d70f 100644 --- a/apps/nestjs-backend/src/features/v2/v2.controller.ts +++ b/apps/nestjs-backend/src/features/v2/v2.controller.ts @@ -45,9 +45,9 @@ export class V2Controller { tables() { return { create: implement(v2Contract.tables.create).handler(async ({ input }) => { - const container = await this.v2Container.getContainer(); + const container = await this.v2Container.getContainerForBase(input.baseId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const result = await executeCreateTableEndpoint(context, input, commandBus); @@ -56,9 +56,9 @@ export class V2Controller { throwOrpcErrorByStatus(result.status, result.body.error); }), getById: implement(v2Contract.tables.getById).handler(async ({ input }) => { - const container = await this.v2Container.getContainer(); + const container = await this.v2Container.getContainerForTable(input.tableId); const queryBus = container.resolve(v2CoreTokens.queryBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const result = await executeGetTableByIdEndpoint(context, input, queryBus); if (result.status === 200) return result.body; @@ -66,9 +66,9 @@ export class V2Controller { throwOrpcErrorByStatus(result.status, result.body.error); }), deleteRecords: implement(v2Contract.tables.deleteRecords).handler(async ({ input }) => { - const container = await this.v2Container.getContainer(); + const container = await this.v2Container.getContainerForTable(input.tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const result = await executeDeleteRecordsEndpoint(context, input, commandBus); @@ -77,9 +77,9 @@ export class V2Controller { throwOrpcErrorByStatus(result.status, result.body.error); }), updateRecords: implement(v2Contract.tables.updateRecords).handler(async ({ input }) => { - const container = await this.v2Container.getContainer(); + const container = await this.v2Container.getContainerForTable(input.tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const result = await executeUpdateRecordsEndpoint(context, input, commandBus); diff --git a/apps/nestjs-backend/src/features/view/constant.ts b/apps/nestjs-backend/src/features/view/constant.ts index 67725402ac..e399c46898 100644 --- a/apps/nestjs-backend/src/features/view/constant.ts +++ b/apps/nestjs-backend/src/features/view/constant.ts @@ -4,11 +4,7 @@ import type { IShareViewMeta } from '@teable/core'; export const ROW_ORDER_FIELD_PREFIX = '__row'; export const defaultShareMetaMap: Record = { - [ViewType.Form]: { - submit: { - allow: true, - }, - }, + [ViewType.Form]: {}, [ViewType.Kanban]: { includeRecords: true, }, diff --git a/apps/nestjs-backend/src/features/view/model/form-view.dto.ts b/apps/nestjs-backend/src/features/view/model/form-view.dto.ts index 9d8e95cfe6..3866186c7e 100644 --- a/apps/nestjs-backend/src/features/view/model/form-view.dto.ts +++ b/apps/nestjs-backend/src/features/view/model/form-view.dto.ts @@ -2,9 +2,5 @@ import type { IShareViewMeta } from '@teable/core'; import { FormViewCore } from '@teable/core'; export class FormViewDto extends FormViewCore { - defaultShareMeta: IShareViewMeta = { - submit: { - allow: true, - }, - }; + defaultShareMeta: IShareViewMeta = {}; } diff --git a/apps/nestjs-backend/src/features/view/open-api/view-open-api-v2.service.ts b/apps/nestjs-backend/src/features/view/open-api/view-open-api-v2.service.ts index 7b90f9f141..c233b9e631 100644 --- a/apps/nestjs-backend/src/features/view/open-api/view-open-api-v2.service.ts +++ b/apps/nestjs-backend/src/features/view/open-api/view-open-api-v2.service.ts @@ -38,9 +38,9 @@ export class ViewOpenApiV2Service { viewId: string, updateRecordOrdersRo: IUpdateRecordOrdersRo ): Promise { - const container = await this.v2ContainerService.getContainer(); + const container = await this.v2ContainerService.getContainerForTable(tableId); const commandBus = container.resolve(v2CoreTokens.commandBus); - const context = await this.v2ContextFactory.createContext(); + const context = await this.v2ContextFactory.createContext(container); const v2Input = { tableId, diff --git a/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts b/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts index f231c3107d..6dc9fff327 100644 --- a/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts +++ b/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts @@ -33,7 +33,6 @@ import { HttpErrorCode, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { PluginPosition, PluginStatus } from '@teable/openapi'; import type { IViewPluginUpdateStorageRo, @@ -53,6 +52,7 @@ import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; import { Events } from '../../../event-emitter/events'; +import { DatabaseRouter } from '../../../global/database-router.service'; import { DATA_KNEX } from '../../../global/knex/knex.module'; import type { IClsStore } from '../../../types/cls'; import { Timing } from '../../../utils/timing'; @@ -71,7 +71,7 @@ export class ViewOpenApiService { constructor( private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, private readonly recordService: RecordService, private readonly viewService: ViewService, private readonly fieldService: FieldService, @@ -148,7 +148,11 @@ export class ViewOpenApiService { const { sortObjs } = viewOrderRo; const dbTableName = await this.recordService.getDbTableName(tableId); const fields = await this.fieldService.getFieldsByQuery(tableId, { viewId }); - const indexField = await this.viewService.getOrCreateViewIndexField(dbTableName, viewId); + const indexField = await this.viewService.getOrCreateViewIndexFieldForTable( + tableId, + dbTableName, + viewId + ); const queryBuilder = this.knex(dbTableName); @@ -170,7 +174,8 @@ export class ViewOpenApiService { manualSort: true, }; - await this.dataPrismaService.$tx( + await this.databaseRouter.dataPrismaTransactionForTable( + tableId, async (prisma) => { await prisma.$executeRawUnsafe( this.updateRecordOrderSql(orderRawSql, dbTableName, indexField) @@ -627,8 +632,8 @@ export class ViewOpenApiService { /** * shuffle record order */ - async shuffleRecords(dbTableName: string, indexField: string) { - const recordCount = await this.recordService.getAllRecordCount(dbTableName); + async shuffleRecords(tableId: string, dbTableName: string, indexField: string) { + const recordCount = await this.recordService.getAllRecordCount(dbTableName, tableId); if (recordCount > 100_000) { throw new CustomHttpException( `Not enough gap to shuffle the row here, record count: ${recordCount}`, @@ -647,7 +652,7 @@ export class ViewOpenApiService { indexField ); - await this.dataPrismaService.$executeRawUnsafe(sql); + await this.databaseRouter.executeDataPrismaForTable(tableId, sql); } @Timing() @@ -673,9 +678,8 @@ export class ViewOpenApiService { .where('__id', anchorId) .toQuery(); - const anchorRecord = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id: string; order: number }[]>(anchorRecordSql) + const anchorRecord = await this.databaseRouter + .queryDataPrismaForTable<{ id: string; order: number }[]>(tableId, anchorRecordSql) .then((res) => { return res[0]; }); @@ -711,16 +715,15 @@ export class ViewOpenApiService { .orderBy(indexField, align) .limit(1) .toQuery(); - return this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id: string; order: number }[]>(nextRecordSql) + return this.databaseRouter + .queryDataPrismaForTable<{ id: string; order: number }[]>(tableId, nextRecordSql) .then((res) => { return res[0]; }); }, update, shuffle: async () => { - await this.shuffleRecords(dbTableName, indexField); + await this.shuffleRecords(tableId, dbTableName, indexField); }, }); } @@ -753,7 +756,11 @@ export class ViewOpenApiService { ? await this.recordService.getRecordIndexes(table, recordIds, viewId) : undefined; - const indexField = await this.viewService.getOrCreateViewIndexField(dbTableName, viewId); + const indexField = await this.viewService.getOrCreateViewIndexFieldForTable( + table.id, + dbTableName, + viewId + ); await this.updateRecordOrdersInner({ tableId: table.id, @@ -762,7 +769,7 @@ export class ViewOpenApiService { indexField, orderRo, update: async (indexes) => { - await this.dataPrismaService.$tx(async (prisma) => { + await this.databaseRouter.dataPrismaTransactionForTable(table.id, async (prisma) => { for (let i = 0; i < recordIds.length; i++) { const recordId = recordIds[i]; const updateRecordSql = this.knex(dbTableName) @@ -1032,9 +1039,9 @@ export class ViewOpenApiService { .whereIn('__id', Array.from(recordSet)) .toQuery(); - const list = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ id: string; title: string | null }[]>(nativeQuery); + const list = await this.databaseRouter.queryDataPrismaForTable< + { id: string; title: string | null }[] + >(foreignTableId, nativeQuery); const fieldInstances = createFieldInstanceByRaw(lookupedFieldRaw); res.push({ tableId: foreignTableId, diff --git a/apps/nestjs-backend/src/features/view/view-data-safety-limit.service.spec.ts b/apps/nestjs-backend/src/features/view/view-data-safety-limit.service.spec.ts new file mode 100644 index 0000000000..70d1a874ad --- /dev/null +++ b/apps/nestjs-backend/src/features/view/view-data-safety-limit.service.spec.ts @@ -0,0 +1,159 @@ +import type { ConfigService } from '@nestjs/config'; +import { HttpErrorCode, type IFilter } from '@teable/core'; +import type { PrismaService } from '@teable/db-main-prisma'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; + +vi.mock('@teable/db-main-prisma', () => ({ PrismaService: class PrismaService {} })); + +let ViewDataSafetyLimitService: typeof import('./view-data-safety-limit.service').ViewDataSafetyLimitService; + +const filterItem = { + fieldId: 'fldTest', + operator: 'is', + value: 'x', + isSymbol: false, +}; + +const createService = ( + env: Record, + options: { currentViewCount?: number } = {} +) => { + const count = vi.fn().mockResolvedValue(options.currentViewCount ?? 0); + const configService = { + get: vi.fn((key: string) => env[key]), + } as unknown as ConfigService; + const prismaService = { + txClient: () => ({ + view: { count }, + }), + } as unknown as PrismaService; + + return { + service: new ViewDataSafetyLimitService(configService, prismaService), + count, + }; +}; + +const expectLimitError = (error: unknown, domainCode: string) => { + expect(error).toMatchObject({ + code: HttpErrorCode.VALIDATION_ERROR, + data: { + domainCode, + }, + }); +}; + +describe('ViewDataSafetyLimitService', () => { + beforeAll(async () => { + ({ ViewDataSafetyLimitService } = await import('./view-data-safety-limit.service')); + }, 30_000); + + it('rejects creating a view when the table has reached the views-per-table limit', async () => { + const { service, count } = createService( + { TABLE_LIMIT_VIEWS_PER_TABLE_MAX: '2' }, + { currentViewCount: 2 } + ); + + await expect(service.ensureCanCreateView('tblTest')).rejects.toSatisfy((error: unknown) => { + expectLimitError(error, 'validation.limit.views_per_table_max'); + return true; + }); + expect(count).toHaveBeenCalledWith({ where: { tableId: 'tblTest', deletedTime: null } }); + }); + + it('allows creating a view at the views-per-table boundary', async () => { + const { service } = createService( + { TABLE_LIMIT_VIEWS_PER_TABLE_MAX: '2' }, + { currentViewCount: 1 } + ); + + await expect(service.ensureCanCreateView('tblTest')).resolves.toBeUndefined(); + }); + + it.each([ + [ + 'validation.limit.name_max_length', + { TABLE_LIMIT_NAME_MAX_LENGTH: '3' }, + () => ({ name: 'Long' }), + ], + [ + 'validation.limit.description_max_length', + { TABLE_LIMIT_DESCRIPTION_MAX_LENGTH: '3' }, + () => ({ description: 'Long' }), + ], + [ + 'validation.limit.view_filter_items_max', + { TABLE_LIMIT_VIEW_FILTER_ITEMS_MAX: '1' }, + () => ({ + filter: { + conjunction: 'and', + filterSet: [filterItem, filterItem], + } as unknown as IFilter, + }), + ], + [ + 'validation.limit.view_filter_depth_max', + { TABLE_LIMIT_VIEW_FILTER_DEPTH_MAX: '1' }, + () => ({ + filter: { + conjunction: 'and', + filterSet: [{ conjunction: 'and', filterSet: [filterItem] }], + } as unknown as IFilter, + }), + ], + [ + 'validation.limit.view_sort_items_max', + { TABLE_LIMIT_VIEW_SORT_ITEMS_MAX: '1' }, + () => ({ + sort: { + sortObjs: [ + { fieldId: 'fldA', order: 'asc' }, + { fieldId: 'fldB', order: 'desc' }, + ], + }, + }), + ], + [ + 'validation.limit.view_group_items_max', + { TABLE_LIMIT_VIEW_GROUP_ITEMS_MAX: '1' }, + () => ({ + group: [ + { fieldId: 'fldA', order: 'asc' }, + { fieldId: 'fldB', order: 'desc' }, + ], + }), + ], + [ + 'validation.limit.view_options_max_bytes', + { TABLE_LIMIT_VIEW_OPTIONS_MAX_BYTES: '4' }, + () => ({ options: { rowHeight: 1 } }), + ], + ])('rejects %s for view payloads', (expectedCode, env, payloadFactory) => { + const { service } = createService(env); + + try { + service.ensureViewPayload(payloadFactory()); + throw new Error('Expected limit error'); + } catch (error) { + expectLimitError(error, expectedCode); + } + }); + + it('validates serialized view property updates', () => { + const { service } = createService({ TABLE_LIMIT_VIEW_SORT_ITEMS_MAX: '1' }); + + try { + service.ensureSerializedProperties({ + sort: JSON.stringify({ + sortObjs: [ + { fieldId: 'fldA', order: 'asc' }, + { fieldId: 'fldB', order: 'desc' }, + ], + }), + }); + throw new Error('Expected limit error'); + } catch (error) { + expectLimitError(error, 'validation.limit.view_sort_items_max'); + } + }); +}); diff --git a/apps/nestjs-backend/src/features/view/view-data-safety-limit.service.ts b/apps/nestjs-backend/src/features/view/view-data-safety-limit.service.ts new file mode 100644 index 0000000000..6a294f2044 --- /dev/null +++ b/apps/nestjs-backend/src/features/view/view-data-safety-limit.service.ts @@ -0,0 +1,178 @@ +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { + HttpErrorCode, + type IFilter, + type IGroup, + type ISort, + type IViewOptions, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { + ensureTableDataSafetyViewOperationLimits, + resolveTableDataSafetyLimits, + type ResolvedTableDataSafetyLimitConfig, + type TableDataSafetyLimitConfig, + ViewOperationKind, + type ViewOperationPayloadViewConfig, + type ViewOperationPluginContext, +} from '@teable/v2-core'; +import { CustomHttpException } from '../../custom.exception'; + +type SerializedViewProperties = { + name?: string | null; + description?: string | null; + filter?: string | null; + sort?: string | null; + group?: string | null; + options?: string | null; +}; + +type ViewPayload = ViewOperationPayloadViewConfig & { + name?: string | null; + description?: string | null; + filter?: IFilter; + sort?: ISort; + group?: IGroup; + options?: IViewOptions; +}; + +const TABLE_LIMIT_ENV_KEYS = { + tableSchema: { + maxViewsPerTable: 'TABLE_LIMIT_VIEWS_PER_TABLE_MAX', + }, + viewConfig: { + maxFilterItems: 'TABLE_LIMIT_VIEW_FILTER_ITEMS_MAX', + maxFilterDepth: 'TABLE_LIMIT_VIEW_FILTER_DEPTH_MAX', + maxSortItems: 'TABLE_LIMIT_VIEW_SORT_ITEMS_MAX', + maxGroupItems: 'TABLE_LIMIT_VIEW_GROUP_ITEMS_MAX', + maxOptionsBytes: 'TABLE_LIMIT_VIEW_OPTIONS_MAX_BYTES', + }, + displayText: { + maxNameLength: 'TABLE_LIMIT_NAME_MAX_LENGTH', + maxDescriptionLength: 'TABLE_LIMIT_DESCRIPTION_MAX_LENGTH', + }, +} as const; + +const parseJsonProperty = (value: string | null | undefined): T | undefined => { + if (value == null) return undefined; + return JSON.parse(value) as T; +}; + +@Injectable() +export class ViewDataSafetyLimitService { + constructor( + private readonly configService: ConfigService, + private readonly prismaService: PrismaService + ) {} + + private getPositiveInteger(key: string): number | undefined { + const value = this.configService.get(key); + const parsed = + typeof value === 'number' ? value : typeof value === 'string' ? Number(value) : NaN; + return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; + } + + private getLimits(): ResolvedTableDataSafetyLimitConfig { + const config: TableDataSafetyLimitConfig = { + tableSchema: { + maxViewsPerTable: this.getPositiveInteger( + TABLE_LIMIT_ENV_KEYS.tableSchema.maxViewsPerTable + ), + }, + viewConfig: { + maxFilterItems: this.getPositiveInteger(TABLE_LIMIT_ENV_KEYS.viewConfig.maxFilterItems), + maxFilterDepth: this.getPositiveInteger(TABLE_LIMIT_ENV_KEYS.viewConfig.maxFilterDepth), + maxSortItems: this.getPositiveInteger(TABLE_LIMIT_ENV_KEYS.viewConfig.maxSortItems), + maxGroupItems: this.getPositiveInteger(TABLE_LIMIT_ENV_KEYS.viewConfig.maxGroupItems), + maxOptionsBytes: this.getPositiveInteger(TABLE_LIMIT_ENV_KEYS.viewConfig.maxOptionsBytes), + }, + displayText: { + maxNameLength: this.getPositiveInteger(TABLE_LIMIT_ENV_KEYS.displayText.maxNameLength), + maxDescriptionLength: this.getPositiveInteger( + TABLE_LIMIT_ENV_KEYS.displayText.maxDescriptionLength + ), + }, + }; + + return resolveTableDataSafetyLimits(config); + } + + private ensureViewOperation(context: ViewOperationPluginContext): void { + const result = ensureTableDataSafetyViewOperationLimits(context, this.getLimits()); + if (result.isOk()) return; + + const error = result.error; + throw new CustomHttpException(error.message, HttpErrorCode.VALIDATION_ERROR, { + domainCode: error.code, + domainTags: error.tags, + details: error.details, + }); + } + + async ensureCanCreateView(tableId: string): Promise { + const currentViewCount = await this.prismaService.txClient().view.count({ + where: { tableId, deletedTime: null }, + }); + + this.ensureViewOperation({ + kind: ViewOperationKind.create, + executionContext: {} as ViewOperationPluginContext['executionContext'], + payload: { + tableId, + currentViewCount, + view: {}, + }, + isTransactionBound: false, + }); + } + + ensureViewPayload(payload: ViewPayload): void { + this.ensureViewOperation({ + kind: ViewOperationKind.update, + executionContext: {} as ViewOperationPluginContext['executionContext'], + payload: { + tableId: '', + viewId: '', + patch: payload, + }, + isTransactionBound: false, + }); + } + + ensureName(name: string | null | undefined): void { + this.ensureViewPayload({ name }); + } + + ensureDescription(description: string | null | undefined): void { + this.ensureViewPayload({ description }); + } + + ensureFilter(filter: IFilter | undefined): void { + this.ensureViewPayload({ filter }); + } + + ensureSort(sort: ISort | undefined): void { + this.ensureViewPayload({ sort }); + } + + ensureGroup(group: IGroup | undefined): void { + this.ensureViewPayload({ group }); + } + + ensureOptions(options: IViewOptions | undefined): void { + this.ensureViewPayload({ options }); + } + + ensureSerializedProperties(properties: SerializedViewProperties | undefined): void { + if (!properties) return; + this.ensureViewPayload({ + name: properties.name, + description: properties.description, + filter: parseJsonProperty(properties.filter), + sort: parseJsonProperty(properties.sort), + group: parseJsonProperty(properties.group), + options: parseJsonProperty(properties.options), + }); + } +} diff --git a/apps/nestjs-backend/src/features/view/view.module.ts b/apps/nestjs-backend/src/features/view/view.module.ts index 6a678ee315..7f8db3ee08 100644 --- a/apps/nestjs-backend/src/features/view/view.module.ts +++ b/apps/nestjs-backend/src/features/view/view.module.ts @@ -1,11 +1,12 @@ import { Module } from '@nestjs/common'; import { DbProvider } from '../../db-provider/db.provider'; import { CalculationModule } from '../calculation/calculation.module'; +import { ViewDataSafetyLimitService } from './view-data-safety-limit.service'; import { ViewService } from './view.service'; @Module({ imports: [CalculationModule], - providers: [ViewService, DbProvider], + providers: [ViewService, ViewDataSafetyLimitService, DbProvider], exports: [ViewService], }) export class ViewModule {} diff --git a/apps/nestjs-backend/src/features/view/view.service.ts b/apps/nestjs-backend/src/features/view/view.service.ts index 116076338e..6517832f37 100644 --- a/apps/nestjs-backend/src/features/view/view.service.ts +++ b/apps/nestjs-backend/src/features/view/view.service.ts @@ -34,7 +34,6 @@ import { } from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import { Knex } from 'knex'; import { isEmpty, isNull, isString, merge, snakeCase, uniq } from 'lodash'; import { InjectModel } from 'nest-knexjs'; @@ -43,6 +42,8 @@ import { fromZodError } from 'zod-validation-error'; import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import type { IDataPrismaQueryExecutor } from '../../global/database-router.service'; +import { DatabaseRouter } from '../../global/database-router.service'; import { CUSTOM_KNEX, DATA_KNEX } from '../../global/knex/knex.module'; import type { IReadonlyAdapterService } from '../../share-db/interface'; import { RawOpType } from '../../share-db/interface'; @@ -52,6 +53,7 @@ import { BatchService } from '../calculation/batch.service'; import { ROW_ORDER_FIELD_PREFIX } from './constant'; import { createViewInstanceByRaw, createViewVoByRaw } from './model/factory'; import { adjustFrozenField } from './utils/derive-frozen-fields'; +import { ViewDataSafetyLimitService } from './view-data-safety-limit.service'; type IViewOpContext = IUpdateViewColumnMetaOpContext | ISetViewPropertyOpContext; @@ -61,10 +63,11 @@ export class ViewService implements IReadonlyAdapterService { private readonly cls: ClsService, private readonly batchService: BatchService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @InjectModel(CUSTOM_KNEX) private readonly knex: Knex, @InjectModel(DATA_KNEX) private readonly dataKnex: Knex, - @InjectDbProvider() private readonly dbProvider: IDbProvider + @InjectDbProvider() private readonly dbProvider: IDbProvider, + private readonly viewDataSafetyLimitService: ViewDataSafetyLimitService ) {} getRowIndexFieldName(viewId: string) { @@ -93,26 +96,25 @@ export class ViewService implements IReadonlyAdapterService { return { name, order }; } - async existIndex(dbTableName: string, viewId: string) { + async existIndex(dbTableName: string, viewId: string, prisma: IDataPrismaQueryExecutor) { const columnName = this.getRowIndexFieldName(viewId); - const exists = await this.dbProvider.checkColumnExist( - dbTableName, - columnName, - this.dataPrismaService.txClient() - ); + const exists = await this.dbProvider.checkColumnExist(dbTableName, columnName, prisma); if (exists) { return columnName; } } - async createViewIndexField(dbTableName: string, viewId: string) { - const prisma = this.dataPrismaService.txClient(); - + async createViewIndexField( + dbTableName: string, + viewId: string, + prisma: IDataPrismaQueryExecutor, + knex: Knex = this.dataKnex + ) { const rowIndexFieldName = this.getRowIndexFieldName(viewId); // add a field for maintain row order number - const addRowIndexColumnSql = this.dataKnex.schema + const addRowIndexColumnSql = knex.schema .alterTable(dbTableName, (table) => { table.double(rowIndexFieldName); }) @@ -120,15 +122,15 @@ export class ViewService implements IReadonlyAdapterService { await prisma.$executeRawUnsafe(addRowIndexColumnSql); // fill initial order for every record, with auto increment integer - const updateRowIndexSql = this.dataKnex(dbTableName) + const updateRowIndexSql = knex(dbTableName) .update({ - [rowIndexFieldName]: this.dataKnex.ref('__auto_number'), + [rowIndexFieldName]: knex.ref('__auto_number'), }) .toQuery(); await prisma.$executeRawUnsafe(updateRowIndexSql); // create index - const createRowIndexSQL = this.dataKnex.schema + const createRowIndexSQL = knex.schema .alterTable(dbTableName, (table) => { table.index(rowIndexFieldName, this.getRowIndexFieldIndexName(viewId)); }) @@ -138,11 +140,27 @@ export class ViewService implements IReadonlyAdapterService { } async getOrCreateViewIndexField(dbTableName: string, viewId: string) { - const indexFieldName = await this.existIndex(dbTableName, viewId); - if (indexFieldName) { - return indexFieldName; - } - return this.createViewIndexField(dbTableName, viewId); + const view = await this.prismaService.txClient().view.findUniqueOrThrow({ + where: { id: viewId }, + select: { tableId: true }, + }); + return this.getOrCreateViewIndexFieldForTable(view.tableId, dbTableName, viewId); + } + + async getOrCreateViewIndexFieldForTable(tableId: string, dbTableName: string, viewId: string) { + return await this.databaseRouter.dataPrismaTransactionForTable(tableId, async (prisma) => { + const indexFieldName = await this.existIndex(dbTableName, viewId, prisma); + if (indexFieldName) { + return indexFieldName; + } + + return this.createViewIndexField( + dbTableName, + viewId, + prisma, + await this.databaseRouter.dataKnexForTable(tableId) + ); + }); } // eslint-disable-next-line sonarjs/cognitive-complexity @@ -252,6 +270,7 @@ export class ViewService implements IReadonlyAdapterService { async createDbView(tableId: string, viewRo: IViewRo) { const userId = this.cls.get('user.id'); + await this.viewDataSafetyLimitService.ensureCanCreateView(tableId); const createViewRo = await this.viewDataCompensation(tableId, viewRo); const { @@ -269,6 +288,14 @@ export class ViewService implements IReadonlyAdapterService { } = createViewRo; const { name, order } = await this.polishOrderAndName(tableId, createViewRo); + this.viewDataSafetyLimitService.ensureViewPayload({ + name, + description, + filter, + sort, + group, + options, + }); const viewId = generateViewId(); const prisma = this.prismaService.txClient(); @@ -381,6 +408,7 @@ export class ViewService implements IReadonlyAdapterService { } async updateViewSort(tableId: string, viewId: string, sort: ISort) { + this.viewDataSafetyLimitService.ensureSort(sort); const viewRaw = await this.prismaService .txClient() .view.findFirstOrThrow({ @@ -506,6 +534,9 @@ export class ViewService implements IReadonlyAdapterService { values, }; }); + for (const viewId of updatedViewIds) { + this.viewDataSafetyLimitService.ensureSerializedProperties(updateViewMap[viewId]?.property); + } if (data.length === 1) { const { id, values } = data[0]; diff --git a/apps/nestjs-backend/src/filter/global-exception.filter.spec.ts b/apps/nestjs-backend/src/filter/global-exception.filter.spec.ts index 3c232702d7..0f126c83b9 100644 --- a/apps/nestjs-backend/src/filter/global-exception.filter.spec.ts +++ b/apps/nestjs-backend/src/filter/global-exception.filter.spec.ts @@ -2,23 +2,63 @@ import { BadRequestException, Logger } from '@nestjs/common'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { GlobalExceptionFilter } from './global-exception.filter'; -const { sentryScope, captureException, withScope } = vi.hoisted(() => { - const sentryScope = { - setTag: vi.fn(), - setUser: vi.fn(), - }; - return { - sentryScope, - captureException: vi.fn(), - withScope: vi.fn((callback: (scope: typeof sentryScope) => void) => callback(sentryScope)), - }; -}); +const { activeSpan, runtimeErrorCounter, sentryScope, captureException, withScope } = vi.hoisted( + () => { + const activeSpan = { + setAttributes: vi.fn(), + setStatus: vi.fn(), + }; + const runtimeErrorCounter = { + add: vi.fn(), + }; + const sentryScope = { + setContext: vi.fn(), + setTag: vi.fn(), + setUser: vi.fn(), + }; + return { + activeSpan, + runtimeErrorCounter, + sentryScope, + captureException: vi.fn(), + withScope: vi.fn((callback: (scope: typeof sentryScope) => void) => callback(sentryScope)), + }; + } +); + +vi.mock('@opentelemetry/api', () => ({ + metrics: { + getMeter: vi.fn(() => ({ + createCounter: vi.fn(() => runtimeErrorCounter), + })), + }, + SpanStatusCode: { + ERROR: 2, + }, + trace: { + getActiveSpan: vi.fn(() => activeSpan), + }, +})); vi.mock('@sentry/nestjs', () => ({ captureException, withScope, })); +const userId = 'usr123'; +const userEmail = 'user@example.com'; +const spaceId = 'spc123'; +const dataDbConnectionId = 'dcn123'; +const dataDbUrlFingerprint = 'fp123'; +const dataDbErrorCode = 'data_db.database_missing'; +const dataDbOtelAttribute = { + errorCode: 'teable.data_db.error_code', + connectionId: 'teable.data_db.connection_id', + urlFingerprint: 'teable.data_db.url_fingerprint', + retryable: 'teable.data_db.retryable', + userActionable: 'teable.data_db.user_actionable', +} as const; + describe('GlobalExceptionFilter', () => { const configService = { getOrThrow: vi.fn(() => ({ enableGlobalErrorLogging: false })), @@ -49,9 +89,9 @@ describe('GlobalExceptionFilter', () => { const cls = { get: vi.fn((key: string) => { const values = new Map([ - ['user.id', 'usr123'], - ['user.email', 'user@example.com'], - ['spaceId', 'spc123'], + ['user.id', userId], + ['user.email', userEmail], + ['spaceId', spaceId], ]); return values.get(key); }), @@ -64,10 +104,10 @@ describe('GlobalExceptionFilter', () => { expect(withScope).toHaveBeenCalledTimes(1); expect(sentryScope.setUser).toHaveBeenNthCalledWith(1, null); expect(sentryScope.setUser).toHaveBeenNthCalledWith(2, { - id: 'usr123', - email: 'user@example.com', + id: userId, + email: userEmail, }); - expect(sentryScope.setTag).toHaveBeenCalledWith('space.id', 'spc123'); + expect(sentryScope.setTag).toHaveBeenCalledWith('space.id', spaceId); expect(captureException).toHaveBeenCalledWith(exception, { mechanism: { handled: false, type: 'auto.function.nestjs.exception_captured' }, }); @@ -81,4 +121,85 @@ describe('GlobalExceptionFilter', () => { expect(withScope).not.toHaveBeenCalled(); expect(captureException).not.toHaveBeenCalled(); }); + + it('returns a classified BYODB runtime error and annotates Sentry plus OTel', () => { + const cls = { + get: vi.fn((key: string) => { + const values = new Map([ + ['user.id', userId], + ['user.email', userEmail], + ['spaceId', spaceId], + [ + 'dataDb', + { + mode: 'byodb', + spaceId, + connectionId: dataDbConnectionId, + urlFingerprint: dataDbUrlFingerprint, + displayHost: 'db.example.com', + displayDatabase: 'customer_data', + internalSchema: 'teable_internal', + }, + ], + ]); + return values.get(key); + }), + }; + const exception = Object.assign(new Error('database "secret_customer_db" does not exist'), { + code: '3D000', + }); + const filter = new GlobalExceptionFilter(configService as never, cls as never); + + filter.catch(exception, host as never); + + expect(response.status).toHaveBeenCalledWith(503); + expect(response.json).toHaveBeenCalledWith({ + message: + 'The data database bound to this space is currently unavailable. Please check the external database connection and try again.', + status: 503, + code: 'database_connection_unavailable', + data: { + dataDb: { + code: dataDbErrorCode, + retryable: false, + userActionable: true, + connectionId: dataDbConnectionId, + urlFingerprint: dataDbUrlFingerprint, + displayHost: 'db.example.com', + displayDatabase: 'customer_data', + internalSchema: 'teable_internal', + }, + }, + }); + expect(JSON.stringify(response.json.mock.calls.at(-1)?.[0])).not.toContain( + 'secret_customer_db' + ); + expect(sentryScope.setTag).toHaveBeenCalledWith('data_db.error_code', dataDbErrorCode); + expect(sentryScope.setTag).toHaveBeenCalledWith('data_db.connection_id', dataDbConnectionId); + expect(sentryScope.setContext).toHaveBeenCalledWith( + 'data_db', + expect.objectContaining({ + errorCode: dataDbErrorCode, + driverCode: '3D000', + connectionId: dataDbConnectionId, + urlFingerprint: dataDbUrlFingerprint, + }) + ); + expect(activeSpan.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + [dataDbOtelAttribute.errorCode]: dataDbErrorCode, + [dataDbOtelAttribute.connectionId]: dataDbConnectionId, + [dataDbOtelAttribute.urlFingerprint]: dataDbUrlFingerprint, + }) + ); + expect(activeSpan.setStatus).toHaveBeenCalledWith({ + code: 2, + message: dataDbErrorCode, + }); + expect(runtimeErrorCounter.add).toHaveBeenCalledWith(1, { + [dataDbOtelAttribute.errorCode]: dataDbErrorCode, + [dataDbOtelAttribute.retryable]: false, + [dataDbOtelAttribute.userActionable]: true, + }); + }); }); diff --git a/apps/nestjs-backend/src/filter/global-exception.filter.ts b/apps/nestjs-backend/src/filter/global-exception.filter.ts index b7dd1c39fb..fdbf4e5520 100644 --- a/apps/nestjs-backend/src/filter/global-exception.filter.ts +++ b/apps/nestjs-backend/src/filter/global-exception.filter.ts @@ -11,14 +11,35 @@ import { UnauthorizedException, } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { metrics, trace, SpanStatusCode } from '@opentelemetry/api'; import * as Sentry from '@sentry/nestjs'; +import { HttpErrorCode } from '@teable/core'; import type { Request, Response } from 'express'; import { ClsService } from 'nestjs-cls'; import type { ILoggerConfig } from '../configs/logger.config'; import { TemplateAppTokenNotAllowedException } from '../custom.exception'; +import { classifyDataDbRuntimeError } from '../global/data-db-runtime-error'; +import type { IDataDbRuntimeErrorClassification } from '../global/data-db-runtime-error'; import type { IClsStore } from '../types/cls'; import { exceptionParse } from '../utils/exception-parse'; +const dataDbRuntimeErrorCounter = metrics + .getMeter('teable-observability') + .createCounter('teable.data_db.runtime_errors', { + description: 'Runtime errors from an external data database bound to a space', + }); +const dataDbOtelAttribute = { + mode: 'teable.data_db.mode', + errorCode: 'teable.data_db.error_code', + connectionId: 'teable.data_db.connection_id', + urlFingerprint: 'teable.data_db.url_fingerprint', + host: 'teable.data_db.host', + database: 'teable.data_db.database', + internalSchema: 'teable.data_db.internal_schema', + retryable: 'teable.data_db.retryable', + userActionable: 'teable.data_db.user_actionable', +} as const; + @Catch() export class GlobalExceptionFilter implements ExceptionFilter { private logger = new Logger(GlobalExceptionFilter.name); @@ -34,8 +55,12 @@ export class GlobalExceptionFilter implements ExceptionFilter { const ctx = host.switchToHttp(); const response = ctx.getResponse(); const request = ctx.getRequest(); + const dataDbContext = this.getDataDbContext(); + const dataDbError = dataDbContext ? classifyDataDbRuntimeError(exception) : null; - this.captureException(exception); + this.annotateActiveSpan(dataDbError); + this.recordDataDbMetric(dataDbError); + this.captureException(exception, dataDbError); if ( enableGlobalErrorLogging || @@ -54,6 +79,26 @@ export class GlobalExceptionFilter implements ExceptionFilter { message: exception.message, }); } + if (dataDbError) { + return response.status(503).json({ + message: + 'The data database bound to this space is currently unavailable. Please check the external database connection and try again.', + status: 503, + code: HttpErrorCode.DATABASE_CONNECTION_UNAVAILABLE, + data: { + dataDb: { + code: dataDbError.code, + retryable: dataDbError.retryable, + userActionable: dataDbError.userActionable, + connectionId: dataDbContext?.connectionId, + urlFingerprint: dataDbContext?.urlFingerprint, + displayHost: dataDbContext?.displayHost, + displayDatabase: dataDbContext?.displayDatabase, + internalSchema: dataDbContext?.internalSchema, + }, + }, + }); + } const customHttpException = exceptionParse(exception); const status = customHttpException.getStatus(); return response.status(status).json({ @@ -64,11 +109,15 @@ export class GlobalExceptionFilter implements ExceptionFilter { }); } - private captureException(exception: Error | HttpException) { + private captureException( + exception: Error | HttpException, + dataDbError?: IDataDbRuntimeErrorClassification | null + ) { if (this.isExpectedError(exception)) return; Sentry.withScope((scope) => { this.setSentryContext(scope); + this.setSentryDataDbContext(scope, dataDbError); Sentry.captureException(exception, { mechanism: { handled: false, type: 'auto.function.nestjs.exception_captured' }, }); @@ -95,6 +144,67 @@ export class GlobalExceptionFilter implements ExceptionFilter { } } + private setSentryDataDbContext( + scope: Sentry.Scope, + dataDbError?: IDataDbRuntimeErrorClassification | null + ) { + const dataDbContext = this.getDataDbContext(); + if (!dataDbContext || !dataDbError) return; + + scope.setTag('data_db.mode', dataDbContext.mode); + scope.setTag('data_db.error_code', dataDbError.code); + scope.setTag('data_db.connection_id', dataDbContext.connectionId); + scope.setTag('data_db.host', dataDbContext.displayHost ?? 'unknown'); + scope.setTag('data_db.database', dataDbContext.displayDatabase ?? 'unknown'); + scope.setTag('data_db.internal_schema', dataDbContext.internalSchema ?? 'unknown'); + scope.setContext('data_db', { + ...dataDbContext, + errorCode: dataDbError.code, + driverCode: dataDbError.driverCode, + retryable: dataDbError.retryable, + userActionable: dataDbError.userActionable, + }); + } + + private annotateActiveSpan(dataDbError?: IDataDbRuntimeErrorClassification | null) { + const dataDbContext = this.getDataDbContext(); + if (!dataDbContext || !dataDbError) return; + + const span = trace.getActiveSpan(); + if (!span) return; + + span.setAttributes({ + [dataDbOtelAttribute.mode]: dataDbContext.mode, + [dataDbOtelAttribute.errorCode]: dataDbError.code, + [dataDbOtelAttribute.connectionId]: dataDbContext.connectionId, + [dataDbOtelAttribute.urlFingerprint]: dataDbContext.urlFingerprint ?? '', + [dataDbOtelAttribute.host]: dataDbContext.displayHost ?? '', + [dataDbOtelAttribute.database]: dataDbContext.displayDatabase ?? '', + [dataDbOtelAttribute.internalSchema]: dataDbContext.internalSchema ?? '', + [dataDbOtelAttribute.retryable]: dataDbError.retryable, + [dataDbOtelAttribute.userActionable]: dataDbError.userActionable, + }); + span.setStatus({ code: SpanStatusCode.ERROR, message: dataDbError.code }); + } + + private recordDataDbMetric(dataDbError?: IDataDbRuntimeErrorClassification | null) { + if (!dataDbError) return; + + dataDbRuntimeErrorCounter.add(1, { + [dataDbOtelAttribute.errorCode]: dataDbError.code, + [dataDbOtelAttribute.retryable]: dataDbError.retryable, + [dataDbOtelAttribute.userActionable]: dataDbError.userActionable, + }); + } + + private getDataDbContext() { + try { + return this.cls?.get('dataDb'); + } catch { + return undefined; + } + } + private isExpectedError(exception: unknown) { return ( typeof exception === 'object' && @@ -104,10 +214,22 @@ export class GlobalExceptionFilter implements ExceptionFilter { } protected logError(exception: Error, request: Request) { + const dataDbContext = this.getDataDbContext(); + const dataDbError = dataDbContext ? classifyDataDbRuntimeError(exception) : null; this.logger.error( { url: request?.url, message: exception.message, + dataDb: dataDbError + ? { + code: dataDbError.code, + connectionId: dataDbContext?.connectionId, + urlFingerprint: dataDbContext?.urlFingerprint, + displayHost: dataDbContext?.displayHost, + displayDatabase: dataDbContext?.displayDatabase, + internalSchema: dataDbContext?.internalSchema, + } + : undefined, }, exception.stack ); diff --git a/apps/nestjs-backend/src/global/byodb-routing.guard.spec.ts b/apps/nestjs-backend/src/global/byodb-routing.guard.spec.ts new file mode 100644 index 0000000000..d92ec39ce0 --- /dev/null +++ b/apps/nestjs-backend/src/global/byodb-routing.guard.spec.ts @@ -0,0 +1,55 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { describe, expect, it } from 'vitest'; + +const backendRoot = join(__dirname, '../..'); + +const tableScopedDataPlaneFiles = [ + 'src/event-emitter/listeners/record-history.listener.ts', + 'src/features/aggregation/aggregation.service.ts', + 'src/features/base/base-query/base-query.service.ts', + 'src/features/base/base.service.ts', + 'src/features/base/base-export.service.ts', + 'src/features/base/db-connection.service.ts', + 'src/features/base-sql-executor/base-sql-executor.service.ts', + 'src/features/calculation/batch.service.ts', + 'src/features/calculation/field-calculation.service.ts', + 'src/features/calculation/link.service.ts', + 'src/features/calculation/system-field.service.ts', + 'src/features/database-view/database-view.service.ts', + 'src/features/field/field-calculate/field-converting.service.ts', + 'src/features/field/field-calculate/field-converting-link.service.ts', + 'src/features/field/field-calculate/field-supplement.service.ts', + 'src/features/field/field-duplicate/field-duplicate.service.ts', + 'src/features/field/field.service.ts', + 'src/features/field/open-api/field-open-api.service.ts', + 'src/features/graph/graph.service.ts', + 'src/features/integrity/foreign-key.service.ts', + 'src/features/integrity/link-field.service.ts', + 'src/features/integrity/link-integrity.service.ts', + 'src/features/integrity/unique-index.service.ts', + 'src/features/record/computed/services/computed-dependency-collector.service.ts', + 'src/features/record/computed/services/computed-orchestrator.service.ts', + 'src/features/record/computed/services/link-cascade-resolver.ts', + 'src/features/record/computed/services/record-computed-update.service.ts', + 'src/features/record/open-api/record-open-api.service.ts', + 'src/features/record/record-query.service.ts', + 'src/features/record/record.service.ts', + 'src/features/share/share.service.ts', + 'src/features/table/table-index.service.ts', + 'src/features/table/open-api/table-open-api.service.ts', + 'src/features/view/open-api/view-open-api.service.ts', + 'src/share-db/readonly/record-readonly.service.ts', +]; + +describe('BYODB data-plane routing guard', () => { + it('keeps migrated table-scoped data-plane services off the default data Prisma client', () => { + for (const file of tableScopedDataPlaneFiles) { + const content = readFileSync(join(backendRoot, file), 'utf8'); + + expect(content, file).not.toContain("from '@teable/db-data-prisma'"); + expect(content, file).not.toContain('private readonly dataPrismaService'); + expect(content, file).not.toContain('this.dataPrismaService'); + } + }); +}); diff --git a/apps/nestjs-backend/src/global/data-db-client-manager.service.spec.ts b/apps/nestjs-backend/src/global/data-db-client-manager.service.spec.ts index 685dfe819c..585d1fb929 100644 --- a/apps/nestjs-backend/src/global/data-db-client-manager.service.spec.ts +++ b/apps/nestjs-backend/src/global/data-db-client-manager.service.spec.ts @@ -1,54 +1,168 @@ import { describe, expect, it, vi } from 'vitest'; import { encryptDataDbUrl } from '../features/space/data-db-url-secret'; import { DataDbClientManager } from './data-db-client-manager.service'; +import { DataDbRuntimeCacheService } from './data-db-runtime-cache.service'; + +const withTxClient = (txClient: T) => ({ + ...txClient, + txClient: vi.fn(() => txClient), +}); describe('DataDbClientManager', () => { - it('uses the default data DB clients when a space has no BYODB binding', async () => { - const prismaService = { + it('falls back to the meta DB clients when a space has no BYODB binding', async () => { + const prismaService = withTxClient({ spaceDataDbBinding: { findUnique: vi.fn().mockResolvedValue(null), }, - }; - const defaultDataPrisma = {}; - const defaultDataKnex = {}; + }); + const metaFallbackDataPrisma = {}; + const metaFallbackDataKnex = {}; const manager = new DataDbClientManager( prismaService as never, - defaultDataPrisma as never, - defaultDataKnex as never + metaFallbackDataPrisma as never, + metaFallbackDataKnex as never, + new DataDbRuntimeCacheService() ); - await expect(manager.dataPrismaForSpace('spcxxx')).resolves.toBe(defaultDataPrisma); - await expect(manager.dataKnexForSpace('spcxxx')).resolves.toBe(defaultDataKnex); + await expect(manager.dataPrismaForSpace('spcxxx')).resolves.toBe(metaFallbackDataPrisma); + await expect(manager.dataKnexForSpace('spcxxx')).resolves.toBe(metaFallbackDataKnex); }); it('resolves base scoped clients through the base space', async () => { - const prismaService = { + const prismaService = withTxClient({ base: { findUnique: vi.fn().mockResolvedValue({ spaceId: 'spcxxx' }), }, spaceDataDbBinding: { findUnique: vi.fn().mockResolvedValue(null), }, - }; - const defaultDataPrisma = {}; - const defaultDataKnex = {}; + }); + const metaFallbackDataPrisma = {}; + const metaFallbackDataKnex = {}; const manager = new DataDbClientManager( prismaService as never, - defaultDataPrisma as never, - defaultDataKnex as never + metaFallbackDataPrisma as never, + metaFallbackDataKnex as never, + new DataDbRuntimeCacheService() ); - await expect(manager.dataPrismaForBase('bsexxx')).resolves.toBe(defaultDataPrisma); - await expect(manager.dataKnexForBase('bsexxx')).resolves.toBe(defaultDataKnex); + await expect(manager.dataPrismaForBase('bsexxx')).resolves.toBe(metaFallbackDataPrisma); + await expect(manager.dataKnexForBase('bsexxx')).resolves.toBe(metaFallbackDataKnex); expect(prismaService.base.findUnique).toHaveBeenCalledWith({ where: { id: 'bsexxx' }, select: { spaceId: true }, }); + expect(prismaService.txClient).not.toHaveBeenCalled(); + }); + + it('resolves table scoped clients through the table base space', async () => { + const prismaService = withTxClient({ + tableMeta: { + findUnique: vi.fn().mockResolvedValue({ base: { spaceId: 'spcxxx' } }), + }, + spaceDataDbBinding: { + findUnique: vi.fn().mockResolvedValue(null), + }, + }); + const metaFallbackDataPrisma = {}; + const metaFallbackDataKnex = {}; + const manager = new DataDbClientManager( + prismaService as never, + metaFallbackDataPrisma as never, + metaFallbackDataKnex as never, + new DataDbRuntimeCacheService() + ); + + await expect(manager.dataPrismaForTable('tblxxx')).resolves.toBe(metaFallbackDataPrisma); + await expect(manager.dataKnexForTable('tblxxx')).resolves.toBe(metaFallbackDataKnex); + expect(prismaService.tableMeta.findUnique).toHaveBeenCalledWith({ + where: { id: 'tblxxx' }, + select: { base: { select: { spaceId: true } } }, + }); + expect(prismaService.txClient).not.toHaveBeenCalled(); + }); + + it('uses the active transaction when explicitly requested', async () => { + const txClient = { + tableMeta: { + findUnique: vi.fn().mockResolvedValue({ base: { spaceId: 'spc_in_tx' } }), + }, + spaceDataDbBinding: { + findUnique: vi.fn().mockResolvedValue(null), + }, + }; + const prismaService = { + ...withTxClient(txClient), + tableMeta: { + findUnique: vi.fn().mockResolvedValue(null), + }, + }; + const metaFallbackDataPrisma = {}; + const metaFallbackDataKnex = {}; + const manager = new DataDbClientManager( + prismaService as never, + metaFallbackDataPrisma as never, + metaFallbackDataKnex as never, + new DataDbRuntimeCacheService() + ); + + await expect( + manager.dataPrismaForTable('tbl_new_in_tx', { useTransaction: true }) + ).resolves.toBe(metaFallbackDataPrisma); + expect(txClient.tableMeta.findUnique).toHaveBeenCalledWith({ + where: { id: 'tbl_new_in_tx' }, + select: { base: { select: { spaceId: true } } }, + }); + expect(prismaService.tableMeta.findUnique).not.toHaveBeenCalled(); + }); + + it('uses the root meta client by default even when transaction context exists', async () => { + const txClient = { + tableMeta: { + findUnique: vi.fn().mockResolvedValue({ base: { spaceId: 'spc_in_tx' } }), + }, + spaceDataDbBinding: { + findUnique: vi.fn().mockResolvedValue(null), + }, + }; + const prismaService = { + ...withTxClient(txClient), + tableMeta: { + findUnique: vi.fn().mockResolvedValue({ base: { spaceId: 'spc_after_tx' } }), + }, + spaceDataDbBinding: { + findUnique: vi.fn().mockResolvedValue(null), + }, + }; + const metaFallbackDataPrisma = {}; + const metaFallbackDataKnex = {}; + const manager = new DataDbClientManager( + prismaService as never, + metaFallbackDataPrisma as never, + metaFallbackDataKnex as never, + new DataDbRuntimeCacheService() + ); + + await expect(manager.dataPrismaForTable('tbl_after_tx')).resolves.toBe(metaFallbackDataPrisma); + expect(txClient.tableMeta.findUnique).not.toHaveBeenCalled(); + expect(prismaService.tableMeta.findUnique).toHaveBeenCalledWith({ + where: { id: 'tbl_after_tx' }, + select: { base: { select: { spaceId: true } } }, + }); + expect(prismaService.spaceDataDbBinding.findUnique).toHaveBeenCalledWith({ + where: { spaceId: 'spc_after_tx' }, + include: { dataDbConnection: true }, + }); }); it('resolves BYODB connection details from a ready space binding', async () => { const dataUrl = 'postgresql://teable:secret@example.com:5432/teable_data'; - const prismaService = { + const internalSchema = 'teable_meta_test'; + const cls = { + isActive: vi.fn().mockReturnValue(true), + set: vi.fn(), + }; + const prismaService = withTxClient({ spaceDataDbBinding: { findUnique: vi.fn().mockResolvedValue({ mode: 'byodb', @@ -56,21 +170,127 @@ describe('DataDbClientManager', () => { dataDbConnection: { id: 'dcnxxx', status: 'ready', + internalSchema, + displayHost: 'example.com', + displayDatabase: 'teable_data', + urlFingerprint: 'fp_xxx', encryptedUrl: encryptDataDbUrl(dataUrl), }, }), }, - }; - const defaultDataPrisma = {}; - const defaultDataKnex = {}; + }); + const metaFallbackDataPrisma = {}; + const metaFallbackDataKnex = {}; const manager = new DataDbClientManager( prismaService as never, - defaultDataPrisma as never, - defaultDataKnex as never + metaFallbackDataPrisma as never, + metaFallbackDataKnex as never, + new DataDbRuntimeCacheService(), + undefined, + cls as never ); - await expect(manager.getDataDatabaseUrlForSpace('spcxxx')).resolves.toBe(dataUrl); - await expect(manager.dataKnexForSpace('spcxxx')).resolves.not.toBe(defaultDataKnex); + await expect(manager.getDataDatabaseUrlForSpace('spcxxx')).resolves.toBe( + `${dataUrl}?schema=${internalSchema}&options=-c+search_path%3D${internalSchema}` + ); + await expect(manager.getDataDatabaseForSpace('spcxxx')).resolves.toMatchObject({ + cacheKey: 'dcnxxx', + connectionId: 'dcnxxx', + isMetaFallback: false, + url: `${dataUrl}?schema=${internalSchema}&options=-c+search_path%3D${internalSchema}`, + }); + expect(cls.set).toHaveBeenCalledWith('dataDb', { + mode: 'byodb', + spaceId: 'spcxxx', + connectionId: 'dcnxxx', + urlFingerprint: 'fp_xxx', + displayHost: 'example.com', + displayDatabase: 'teable_data', + internalSchema, + }); + await expect(manager.dataKnexForSpace('spcxxx')).resolves.not.toBe(metaFallbackDataKnex); await manager.onModuleDestroy(); }); + + it('resolves BYODB connection details when no CLS context is active', async () => { + const dataUrl = 'postgresql://teable:secret@example.com:5432/teable_data'; + const internalSchema = 'teable_meta_test'; + const cls = { + isActive: vi.fn().mockReturnValue(false), + set: vi.fn(() => { + throw new Error('No CLS context available'); + }), + }; + const prismaService = withTxClient({ + spaceDataDbBinding: { + findUnique: vi.fn().mockResolvedValue({ + mode: 'byodb', + state: 'ready', + dataDbConnection: { + id: 'dcnxxx', + status: 'ready', + internalSchema, + displayHost: 'example.com', + displayDatabase: 'teable_data', + urlFingerprint: 'fp_xxx', + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }), + }, + }); + const manager = new DataDbClientManager( + prismaService as never, + {} as never, + {} as never, + new DataDbRuntimeCacheService(), + undefined, + cls as never + ); + + await expect(manager.getDataDatabaseUrlForSpace('spcxxx')).resolves.toBe( + `${dataUrl}?schema=${internalSchema}&options=-c+search_path%3D${internalSchema}` + ); + expect(cls.set).not.toHaveBeenCalled(); + }); + + it('ensures the BYODB internal schema is migrated before returning a scoped URL', async () => { + const dataUrl = 'postgresql://teable:secret@example.com:5432/teable_data'; + const internalSchema = 'teable_meta_test'; + const dataDbMigrationService = { + ensureConnectionMigrated: vi.fn().mockResolvedValue([]), + }; + const prismaService = withTxClient({ + spaceDataDbBinding: { + findUnique: vi.fn().mockResolvedValue({ + mode: 'byodb', + state: 'migrating', + dataDbConnection: { + id: 'dcnxxx', + status: 'migrating', + internalSchema, + encryptedUrl: encryptDataDbUrl(dataUrl), + }, + }), + }, + }); + const manager = new DataDbClientManager( + prismaService as never, + {} as never, + {} as never, + new DataDbRuntimeCacheService(), + dataDbMigrationService as never + ); + + await expect(manager.getDataDatabaseForSpace('spcxxx')).resolves.toMatchObject({ + cacheKey: 'dcnxxx', + connectionId: 'dcnxxx', + internalSchema, + isMetaFallback: false, + }); + expect(dataDbMigrationService.ensureConnectionMigrated).toHaveBeenCalledWith({ + connectionId: 'dcnxxx', + internalSchema, + url: dataUrl, + }); + }); }); diff --git a/apps/nestjs-backend/src/global/data-db-client-manager.service.ts b/apps/nestjs-backend/src/global/data-db-client-manager.service.ts index 71a4690a67..d569b0db5d 100644 --- a/apps/nestjs-backend/src/global/data-db-client-manager.service.ts +++ b/apps/nestjs-backend/src/global/data-db-client-manager.service.ts @@ -1,153 +1,275 @@ -import type { OnModuleDestroy } from '@nestjs/common'; -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable, Optional } from '@nestjs/common'; import { DataPrismaService, PrismaClient as DataPrismaClient, - getDataDatabaseUrl, + getMetaDatabaseUrl, } from '@teable/db-data-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import createKnex, { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import { withDataDbInternalSchemaParam } from '../features/space/data-db-internal-schema'; +import { DataDbMigrationService } from '../features/space/data-db-migration.service'; import { decryptDataDbUrl } from '../features/space/data-db-url-secret'; +import type { IClsStore } from '../types/cls'; +import { + DATA_DB_KNEX_CACHE_NAMESPACE, + DATA_DB_PRISMA_CACHE_NAMESPACE, + DataDbRuntimeCacheService, +} from './data-db-runtime-cache.service'; import { DATA_KNEX } from './knex'; -@Injectable() -export class DataDbClientManager implements OnModuleDestroy { - private readonly knexClients = new Map(); - private readonly prismaClients = new Map(); +export interface IResolvedDataDatabase { + cacheKey: string; + url: string; + isMetaFallback: boolean; + connectionId?: string; + internalSchema?: string; +} + +export interface IDataDbRoutingOptions { + useTransaction?: boolean; +} +type IMetaRoutingClient = PrismaService | NonNullable; + +@Injectable() +export class DataDbClientManager { constructor( private readonly prismaService: PrismaService, - private readonly defaultDataPrismaService: DataPrismaService, - @InjectModel(DATA_KNEX) private readonly defaultDataKnex: Knex + private readonly metaFallbackDataPrismaService: DataPrismaService, + @InjectModel(DATA_KNEX) private readonly metaFallbackDataKnex: Knex, + private readonly runtimeCache: DataDbRuntimeCacheService, + @Optional() + private readonly dataDbMigrationService?: DataDbMigrationService, + @Optional() + @Inject(ClsService) + private readonly cls?: ClsService ) {} - async getDataDatabaseUrlForSpace(spaceId: string) { - const binding = await this.prismaService.spaceDataDbBinding.findUnique({ - where: { spaceId }, - include: { dataDbConnection: true }, - }); + private getMetaRoutingClient(options?: IDataDbRoutingOptions): IMetaRoutingClient { + return options?.useTransaction ? this.prismaService.txClient() : this.prismaService; + } - if (!binding || binding.mode === 'default') { - return getDataDatabaseUrl(); - } + async getDataDatabaseForSpace( + spaceId: string, + options?: IDataDbRoutingOptions + ): Promise { + const resolved = await this.resolveSpaceDataDb(spaceId, options); - if (binding.state !== 'ready' || binding.dataDbConnection?.status !== 'ready') { - throw new Error(`Data database binding for space ${spaceId} is not ready`); + if (resolved.isMetaFallback) { + return { + cacheKey: 'meta-fallback', + url: getMetaDatabaseUrl(), + isMetaFallback: true, + }; } - if (!binding.dataDbConnection.encryptedUrl) { - throw new Error(`Data database connection for space ${spaceId} has no encrypted URL`); - } + return { + cacheKey: resolved.connectionId, + connectionId: resolved.connectionId, + internalSchema: resolved.internalSchema, + url: withDataDbInternalSchemaParam(resolved.url, resolved.internalSchema), + isMetaFallback: false, + }; + } - return decryptDataDbUrl(binding.dataDbConnection.encryptedUrl); + async getDataDatabaseUrlForSpace(spaceId: string, options?: IDataDbRoutingOptions) { + return (await this.getDataDatabaseForSpace(spaceId, options)).url; } - async dataKnexForSpace(spaceId: string) { - const binding = await this.prismaService.spaceDataDbBinding.findUnique({ - where: { spaceId }, - include: { dataDbConnection: true }, + async getDataDatabaseForBase(baseId: string, options?: IDataDbRoutingOptions) { + const base = await this.getMetaRoutingClient(options).base.findUnique({ + where: { id: baseId }, + select: { spaceId: true }, }); - - if (!binding || binding.mode === 'default') { - return this.defaultDataKnex; + if (!base) { + throw new Error(`Base ${baseId} not found`); } + return await this.getDataDatabaseForSpace(base.spaceId, options); + } - if (binding.state !== 'ready' || binding.dataDbConnection?.status !== 'ready') { - throw new Error(`Data database binding for space ${spaceId} is not ready`); - } + async getDataDatabaseUrlForBase(baseId: string, options?: IDataDbRoutingOptions) { + return (await this.getDataDatabaseForBase(baseId, options)).url; + } - const connectionId = binding.dataDbConnection.id; - const existing = this.knexClients.get(connectionId); - if (existing) { - return existing; + async getDataDatabaseForTable(tableId: string, options?: IDataDbRoutingOptions) { + const table = await this.getMetaRoutingClient(options).tableMeta.findUnique({ + where: { id: tableId }, + select: { base: { select: { spaceId: true } } }, + }); + if (!table) { + throw new Error(`Table ${tableId} not found`); } + return await this.getDataDatabaseForSpace(table.base.spaceId, options); + } - const client = createKnex({ - client: 'pg', - connection: decryptDataDbUrl(binding.dataDbConnection.encryptedUrl), - pool: { - min: 0, - max: Number(process.env.BYODB_DATA_DB_POOL_MAX ?? 5), - }, - }); - this.knexClients.set(connectionId, client); - return client; + async getDataDatabaseUrlForTable(tableId: string, options?: IDataDbRoutingOptions) { + return (await this.getDataDatabaseForTable(tableId, options)).url; } - async dataPrismaForSpace(spaceId: string) { - const binding = await this.prismaService.spaceDataDbBinding.findUnique({ - where: { spaceId }, - include: { dataDbConnection: true }, - }); + async dataKnexForSpace(spaceId: string, options?: IDataDbRoutingOptions) { + const resolved = await this.resolveSpaceDataDb(spaceId, options); - if (!binding || binding.mode === 'default') { - return this.defaultDataPrismaService; + if (resolved.isMetaFallback) { + return this.metaFallbackDataKnex; } - if (binding.state !== 'ready' || binding.dataDbConnection?.status !== 'ready') { - throw new Error(`Data database binding for space ${spaceId} is not ready`); - } + return await this.runtimeCache.getOrCreate( + DATA_DB_KNEX_CACHE_NAMESPACE, + resolved.connectionId, + () => + createKnex({ + client: 'pg', + connection: resolved.url, + searchPath: [resolved.internalSchema], + pool: { + min: 0, + max: Number(process.env.BYODB_DATA_DB_POOL_MAX ?? 5), + }, + }), + (client) => client.destroy() + ); + } - const connectionId = binding.dataDbConnection.id; - const existing = this.prismaClients.get(connectionId); - if (existing) { - return existing; + async dataPrismaForSpace(spaceId: string, options?: IDataDbRoutingOptions) { + const resolved = await this.resolveSpaceDataDb(spaceId, options); + + if (resolved.isMetaFallback) { + return this.metaFallbackDataPrismaService; } - const client = new DataPrismaClient({ - datasources: { - db: { - url: decryptDataDbUrl(binding.dataDbConnection.encryptedUrl), - }, - }, - }); - this.prismaClients.set(connectionId, client); - return client; + return await this.runtimeCache.getOrCreate( + DATA_DB_PRISMA_CACHE_NAMESPACE, + resolved.connectionId, + () => + new DataPrismaClient({ + datasources: { + db: { + url: withDataDbInternalSchemaParam(resolved.url, resolved.internalSchema), + }, + }, + }), + (client) => client.$disconnect() + ); } - async dataKnexForBase(baseId: string) { - const base = await this.prismaService.base.findUnique({ + async dataKnexForBase(baseId: string, options?: IDataDbRoutingOptions) { + const base = await this.getMetaRoutingClient(options).base.findUnique({ where: { id: baseId }, select: { spaceId: true }, }); if (!base) { throw new Error(`Base ${baseId} not found`); } - return await this.dataKnexForSpace(base.spaceId); + return await this.dataKnexForSpace(base.spaceId, options); + } + + async dataKnexForTable(tableId: string, options?: IDataDbRoutingOptions) { + const table = await this.getMetaRoutingClient(options).tableMeta.findUnique({ + where: { id: tableId }, + select: { base: { select: { spaceId: true } } }, + }); + if (!table) { + throw new Error(`Table ${tableId} not found`); + } + return await this.dataKnexForSpace(table.base.spaceId, options); } - async dataPrismaForBase(baseId: string) { - const base = await this.prismaService.base.findUnique({ + async dataPrismaForTable(tableId: string, options?: IDataDbRoutingOptions) { + const table = await this.getMetaRoutingClient(options).tableMeta.findUnique({ + where: { id: tableId }, + select: { base: { select: { spaceId: true } } }, + }); + if (!table) { + throw new Error(`Table ${tableId} not found`); + } + return await this.dataPrismaForSpace(table.base.spaceId, options); + } + + async dataPrismaForBase(baseId: string, options?: IDataDbRoutingOptions) { + const base = await this.getMetaRoutingClient(options).base.findUnique({ where: { id: baseId }, select: { spaceId: true }, }); if (!base) { throw new Error(`Base ${baseId} not found`); } - return await this.dataPrismaForSpace(base.spaceId); + return await this.dataPrismaForSpace(base.spaceId, options); } - invalidateConnection(connectionId: string) { - const knex = this.knexClients.get(connectionId); - if (knex) { - void knex.destroy(); - this.knexClients.delete(connectionId); + async invalidateConnection(connectionId: string) { + await this.runtimeCache.deleteByKey(connectionId); + } + + private async resolveSpaceDataDb( + spaceId: string, + options?: IDataDbRoutingOptions + ): Promise< + | { isMetaFallback: true } + | { connectionId: string; internalSchema: string; isMetaFallback: false; url: string } + > { + const binding = await this.getMetaRoutingClient(options).spaceDataDbBinding.findUnique({ + where: { spaceId }, + include: { dataDbConnection: true }, + }); + + if (!binding || binding.mode === 'default') { + return { isMetaFallback: true }; } - const prisma = this.prismaClients.get(connectionId); - if (prisma) { - void prisma.$disconnect(); - this.prismaClients.delete(connectionId); + const connection = binding.dataDbConnection; + if (!connection) { + throw new Error(`Data database connection for space ${spaceId} was not found`); + } + + const migratableStates = this.dataDbMigrationService + ? ['ready', 'migrating', 'error'] + : ['ready']; + + if (!migratableStates.includes(binding.state)) { + throw new Error(`Data database binding for space ${spaceId} is not ready`); + } + + if (!migratableStates.includes(connection.status)) { + throw new Error(`Data database binding for space ${spaceId} is not ready`); } + + if (!connection.encryptedUrl) { + throw new Error(`Data database connection for space ${spaceId} has no encrypted URL`); + } + + if (this.cls?.isActive()) { + this.cls.set('dataDb', { + mode: 'byodb', + spaceId, + connectionId: connection.id, + urlFingerprint: connection.urlFingerprint, + displayHost: connection.displayHost, + displayDatabase: connection.displayDatabase, + internalSchema: connection.internalSchema, + }); + } + + const url = decryptDataDbUrl(connection.encryptedUrl); + await this.dataDbMigrationService?.ensureConnectionMigrated({ + connectionId: connection.id, + internalSchema: connection.internalSchema, + url, + }); + + return { + connectionId: connection.id, + internalSchema: connection.internalSchema, + isMetaFallback: false, + url, + }; } async onModuleDestroy() { await Promise.all([ - ...Array.from(this.knexClients.values()).map((client) => client.destroy()), - ...Array.from(this.prismaClients.values()).map((client) => client.$disconnect()), + this.runtimeCache.deleteByNamespace(DATA_DB_KNEX_CACHE_NAMESPACE), + this.runtimeCache.deleteByNamespace(DATA_DB_PRISMA_CACHE_NAMESPACE), ]); - this.knexClients.clear(); - this.prismaClients.clear(); } } diff --git a/apps/nestjs-backend/src/global/data-db-runtime-cache.service.spec.ts b/apps/nestjs-backend/src/global/data-db-runtime-cache.service.spec.ts new file mode 100644 index 0000000000..e9385bac02 --- /dev/null +++ b/apps/nestjs-backend/src/global/data-db-runtime-cache.service.spec.ts @@ -0,0 +1,55 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { DataDbRuntimeCacheService } from './data-db-runtime-cache.service'; + +describe('DataDbRuntimeCacheService', () => { + afterEach(() => { + delete process.env.BYODB_RUNTIME_CACHE_MAX; + }); + + it('reuses entries by namespace and key', async () => { + const cache = new DataDbRuntimeCacheService(); + const create = vi.fn().mockResolvedValue({ id: 'client' }); + const destroy = vi.fn(); + + await expect(cache.getOrCreate('ns', 'key', create, destroy)).resolves.toEqual({ + id: 'client', + }); + await expect(cache.getOrCreate('ns', 'key', create, destroy)).resolves.toEqual({ + id: 'client', + }); + + expect(create).toHaveBeenCalledTimes(1); + expect(destroy).not.toHaveBeenCalled(); + await cache.onModuleDestroy(); + expect(destroy).toHaveBeenCalledTimes(1); + }); + + it('evicts the least recently used entry and destroys it', async () => { + process.env.BYODB_RUNTIME_CACHE_MAX = '2'; + const cache = new DataDbRuntimeCacheService(); + const destroy = vi.fn(); + + await cache.getOrCreate('ns', 'a', () => Promise.resolve('a'), destroy); + await cache.getOrCreate('ns', 'b', () => Promise.resolve('b'), destroy); + await cache.getOrCreate('ns', 'a', () => Promise.resolve('new-a'), destroy); + await cache.getOrCreate('ns', 'c', () => Promise.resolve('c'), destroy); + + expect(destroy).toHaveBeenCalledWith('b'); + expect(cache.size).toBe(2); + await cache.onModuleDestroy(); + }); + + it('can actively invalidate all runtimes for a connection key', async () => { + const cache = new DataDbRuntimeCacheService(); + const destroy = vi.fn(); + + await cache.getOrCreate('prisma', 'dcnxxx', () => Promise.resolve('prisma'), destroy); + await cache.getOrCreate('knex', 'dcnxxx', () => Promise.resolve('knex'), destroy); + await cache.getOrCreate('v2', 'dcnxxx', () => Promise.resolve('container'), destroy); + + await cache.deleteByKey('dcnxxx'); + + expect(destroy).toHaveBeenCalledTimes(3); + expect(cache.size).toBe(0); + }); +}); diff --git a/apps/nestjs-backend/src/global/data-db-runtime-cache.service.ts b/apps/nestjs-backend/src/global/data-db-runtime-cache.service.ts new file mode 100644 index 0000000000..0cffd5668c --- /dev/null +++ b/apps/nestjs-backend/src/global/data-db-runtime-cache.service.ts @@ -0,0 +1,137 @@ +import type { OnModuleDestroy } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; + +export const DATA_DB_KNEX_CACHE_NAMESPACE = 'data-db:knex'; +export const DATA_DB_PRISMA_CACHE_NAMESPACE = 'data-db:prisma'; +export const V2_CONTAINER_CACHE_NAMESPACE = 'v2:container'; + +type DestroyFn = (value: T) => Promise | void; +type UnknownDestroyFn = (value: unknown) => Promise | void; + +interface ICacheEntry { + namespace: string; + key: string; + promise: Promise; + value?: unknown; + destroy: UnknownDestroyFn; +} + +const resolveMaxEntries = () => { + const raw = Number(process.env.BYODB_RUNTIME_CACHE_MAX ?? 50); + return Number.isInteger(raw) && raw > 0 ? raw : 50; +}; + +@Injectable() +export class DataDbRuntimeCacheService implements OnModuleDestroy { + private readonly logger = new Logger(DataDbRuntimeCacheService.name); + private readonly entries = new Map(); + private readonly maxEntries = resolveMaxEntries(); + + async getOrCreate( + namespace: string, + key: string, + create: () => Promise | T, + destroy: DestroyFn + ): Promise { + const cacheKey = this.getCacheKey(namespace, key); + const existing = this.entries.get(cacheKey); + if (existing) { + this.entries.delete(cacheKey); + this.entries.set(cacheKey, existing); + return (await existing.promise) as T; + } + + const entry: ICacheEntry = { + namespace, + key, + destroy: (value) => destroy(value as T), + promise: Promise.resolve(undefined), + }; + + entry.promise = Promise.resolve() + .then(create) + .then((value) => { + entry.value = value; + return value; + }) + .catch((error) => { + this.entries.delete(cacheKey); + throw error; + }); + + this.entries.set(cacheKey, entry); + await this.evictIfNeeded(); + return (await entry.promise) as T; + } + + async delete(namespace: string, key: string) { + const cacheKey = this.getCacheKey(namespace, key); + const entry = this.entries.get(cacheKey); + if (!entry) return; + + this.entries.delete(cacheKey); + await this.destroyEntry(entry); + } + + async deleteByNamespace(namespace: string) { + const entries = Array.from(this.entries.entries()).filter( + ([, entry]) => entry.namespace === namespace + ); + await Promise.all( + entries.map(async ([cacheKey, entry]) => { + this.entries.delete(cacheKey); + await this.destroyEntry(entry); + }) + ); + } + + async deleteByKey(key: string) { + const entries = Array.from(this.entries.entries()).filter(([, entry]) => entry.key === key); + await Promise.all( + entries.map(async ([cacheKey, entry]) => { + this.entries.delete(cacheKey); + await this.destroyEntry(entry); + }) + ); + } + + get size() { + return this.entries.size; + } + + private async evictIfNeeded() { + while (this.entries.size > this.maxEntries) { + const oldest = this.entries.entries().next().value as [string, ICacheEntry] | undefined; + if (!oldest) return; + + const [cacheKey, entry] = oldest; + this.entries.delete(cacheKey); + await this.destroyEntry(entry); + } + } + + private async destroyEntry(entry: ICacheEntry) { + try { + const value = entry.value ?? (await entry.promise.catch(() => undefined)); + if (value != null) { + await entry.destroy(value); + } + } catch (error) { + this.logger.warn( + `Failed to destroy cached data DB runtime ${entry.namespace}:${entry.key}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + private getCacheKey(namespace: string, key: string) { + return `${namespace}:${key}`; + } + + async onModuleDestroy() { + const entries = Array.from(this.entries.entries()); + this.entries.clear(); + await Promise.all(entries.map(([, entry]) => this.destroyEntry(entry))); + } +} diff --git a/apps/nestjs-backend/src/global/data-db-runtime-error.spec.ts b/apps/nestjs-backend/src/global/data-db-runtime-error.spec.ts new file mode 100644 index 0000000000..59414aa6fb --- /dev/null +++ b/apps/nestjs-backend/src/global/data-db-runtime-error.spec.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'vitest'; +import { classifyDataDbRuntimeError } from './data-db-runtime-error'; + +describe('classifyDataDbRuntimeError', () => { + it('classifies a missing external database without echoing the raw driver message', () => { + const error = Object.assign(new Error('database "customer_deleted_db" does not exist'), { + code: '3D000', + }); + + expect(classifyDataDbRuntimeError(error)).toMatchObject({ + code: 'data_db.database_missing', + message: 'The bound data database no longer exists or cannot be selected.', + retryable: false, + userActionable: true, + pgCode: '3D000', + driverCode: '3D000', + }); + }); + + it('classifies common auth, missing relation, timeout, and pool errors', () => { + expect(classifyDataDbRuntimeError({ code: '28P01', message: 'password failed' })).toMatchObject( + { + code: 'data_db.auth_failed', + retryable: false, + userActionable: true, + } + ); + expect( + classifyDataDbRuntimeError({ code: '42P01', message: 'relation missing' }) + ).toMatchObject({ + code: 'data_db.relation_missing', + retryable: false, + userActionable: true, + }); + expect( + classifyDataDbRuntimeError({ code: 'ETIMEDOUT', message: 'connect timed out' }) + ).toMatchObject({ + code: 'data_db.timeout', + retryable: true, + userActionable: true, + driverCode: 'ETIMEDOUT', + }); + expect( + classifyDataDbRuntimeError({ code: 'P2024', message: 'Timed out fetching a new connection' }) + ).toMatchObject({ + code: 'data_db.pool_exhausted', + retryable: true, + userActionable: true, + driverCode: 'P2024', + }); + }); + + it('classifies Prisma messages even when the code is missing', () => { + expect( + classifyDataDbRuntimeError(new Error("Can't reach database server at `db.example.com:5432`")) + ).toMatchObject({ + code: 'data_db.timeout', + retryable: true, + userActionable: true, + }); + }); + + it('returns null for unrelated application errors', () => { + expect(classifyDataDbRuntimeError(new Error('field validation failed'))).toBeNull(); + }); +}); diff --git a/apps/nestjs-backend/src/global/data-db-runtime-error.ts b/apps/nestjs-backend/src/global/data-db-runtime-error.ts new file mode 100644 index 0000000000..12874403c0 --- /dev/null +++ b/apps/nestjs-backend/src/global/data-db-runtime-error.ts @@ -0,0 +1,207 @@ +export type IDataDbRuntimeErrorCode = + | 'data_db.database_missing' + | 'data_db.auth_failed' + | 'data_db.connection_refused' + | 'data_db.timeout' + | 'data_db.network_unreachable' + | 'data_db.connection_lost' + | 'data_db.schema_missing' + | 'data_db.relation_missing' + | 'data_db.permission_denied' + | 'data_db.pool_exhausted'; + +export type IDataDbRuntimeErrorClassification = { + code: IDataDbRuntimeErrorCode; + message: string; + retryable: boolean; + userActionable: boolean; + pgCode?: string; + driverCode?: string; +}; + +const getErrorCode = (error: unknown): string | undefined => { + if (!error || typeof error !== 'object') { + return undefined; + } + + const candidate = error as { code?: unknown; errorCode?: unknown }; + return typeof candidate.code === 'string' + ? candidate.code + : typeof candidate.errorCode === 'string' + ? candidate.errorCode + : undefined; +}; + +const getErrorMessage = (error: unknown): string => { + if (error instanceof Error) { + return error.message; + } + return String(error); +}; + +const buildClassification = ( + error: unknown, + code: IDataDbRuntimeErrorCode, + message: string, + options: Pick +): IDataDbRuntimeErrorClassification => { + const driverCode = getErrorCode(error); + const isPgCode = driverCode + ? /^[0-9A-Z]{5}$/.test(driverCode) && !/^P\d{4}$/.test(driverCode) + : false; + return { + code, + message, + ...options, + ...(driverCode ? { driverCode } : {}), + ...(isPgCode ? { pgCode: driverCode } : {}), + }; +}; + +export const classifyDataDbRuntimeError = ( + error: unknown +): IDataDbRuntimeErrorClassification | null => { + const driverCode = getErrorCode(error); + const message = getErrorMessage(error); + + switch (driverCode) { + case '3D000': + case 'P1003': + return buildClassification( + error, + 'data_db.database_missing', + 'The bound data database no longer exists or cannot be selected.', + { retryable: false, userActionable: true } + ); + case '28P01': + case 'P1000': + return buildClassification( + error, + 'data_db.auth_failed', + 'The bound data database rejected the configured credentials.', + { retryable: false, userActionable: true } + ); + case 'ECONNREFUSED': + return buildClassification( + error, + 'data_db.connection_refused', + 'The bound data database refused the connection.', + { retryable: true, userActionable: true } + ); + case 'ETIMEDOUT': + case 'P1008': + case '57014': + return buildClassification(error, 'data_db.timeout', 'The bound data database timed out.', { + retryable: true, + userActionable: true, + }); + case 'ENOTFOUND': + case 'ENETUNREACH': + case 'EHOSTUNREACH': + case 'EAI_AGAIN': + case 'P1001': + return buildClassification( + error, + 'data_db.network_unreachable', + 'The bound data database host is not reachable.', + { retryable: true, userActionable: true } + ); + case 'ECONNRESET': + case '08000': + case '08001': + case '08003': + case '08004': + case '08006': + case '08007': + case '57P01': + case 'P1017': + return buildClassification( + error, + 'data_db.connection_lost', + 'The bound data database connection was interrupted.', + { retryable: true, userActionable: true } + ); + case '3F000': + return buildClassification( + error, + 'data_db.schema_missing', + 'The bound data database internal schema is missing.', + { retryable: false, userActionable: true } + ); + case '42P01': + case 'P2021': + return buildClassification( + error, + 'data_db.relation_missing', + 'A required table or relation is missing from the bound data database.', + { retryable: false, userActionable: true } + ); + case '42501': + return buildClassification( + error, + 'data_db.permission_denied', + 'The bound data database user does not have the required permissions.', + { retryable: false, userActionable: true } + ); + case '53300': + case '53400': + case 'P2024': + return buildClassification( + error, + 'data_db.pool_exhausted', + 'The bound data database does not have enough available connections.', + { retryable: true, userActionable: true } + ); + default: + break; + } + + if (/database ".+" does not exist/i.test(message)) { + return buildClassification( + error, + 'data_db.database_missing', + 'The bound data database no longer exists or cannot be selected.', + { retryable: false, userActionable: true } + ); + } + if (/password authentication failed/i.test(message)) { + return buildClassification( + error, + 'data_db.auth_failed', + 'The bound data database rejected the configured credentials.', + { retryable: false, userActionable: true } + ); + } + if (/relation ".+" does not exist/i.test(message)) { + return buildClassification( + error, + 'data_db.relation_missing', + 'A required table or relation is missing from the bound data database.', + { retryable: false, userActionable: true } + ); + } + if (/permission denied/i.test(message)) { + return buildClassification( + error, + 'data_db.permission_denied', + 'The bound data database user does not have the required permissions.', + { retryable: false, userActionable: true } + ); + } + if (/Unable to start a transaction|Timed out fetching a new connection/i.test(message)) { + return buildClassification( + error, + 'data_db.pool_exhausted', + 'The bound data database does not have enough available connections.', + { retryable: true, userActionable: true } + ); + } + if (/Can't reach database server|connect ETIMEDOUT|connection timed out/i.test(message)) { + return buildClassification(error, 'data_db.timeout', 'The bound data database timed out.', { + retryable: true, + userActionable: true, + }); + } + + return null; +}; diff --git a/apps/nestjs-backend/src/global/database-router.service.spec.ts b/apps/nestjs-backend/src/global/database-router.service.spec.ts new file mode 100644 index 0000000000..788cddab35 --- /dev/null +++ b/apps/nestjs-backend/src/global/database-router.service.spec.ts @@ -0,0 +1,89 @@ +import { describe, expect, it, vi } from 'vitest'; +import { DatabaseRouter } from './database-router.service'; + +describe('DatabaseRouter', () => { + it('executes table scoped raw queries through the scoped table client', async () => { + const queryRaw = vi.fn().mockResolvedValue([{ count: 1 }]); + const executeRaw = vi.fn().mockResolvedValue(1); + const dataDbClientManager = { + dataPrismaForTable: vi.fn().mockResolvedValue({ + txClient: () => ({ + $queryRawUnsafe: queryRaw, + $executeRawUnsafe: executeRaw, + }), + }), + }; + const router = new DatabaseRouter( + {} as never, + {} as never, + {} as never, + {} as never, + dataDbClientManager as never + ); + + await expect(router.queryDataPrismaForTable('tblxxx', 'select 1')).resolves.toEqual([ + { count: 1 }, + ]); + await expect(router.executeDataPrismaForTable('tblxxx', 'update x')).resolves.toBe(1); + expect(dataDbClientManager.dataPrismaForTable).toHaveBeenCalledWith('tblxxx', undefined); + expect(queryRaw).toHaveBeenCalledWith('select 1'); + expect(executeRaw).toHaveBeenCalledWith('update x'); + }); + + it('passes explicit routing options separately from raw query values', async () => { + const queryRaw = vi.fn().mockResolvedValue([{ id: 'recxxx' }]); + const dataDbClientManager = { + dataPrismaForTable: vi.fn().mockResolvedValue({ + txClient: () => ({ + $queryRawUnsafe: queryRaw, + $executeRawUnsafe: vi.fn(), + }), + }), + }; + const router = new DatabaseRouter( + {} as never, + {} as never, + {} as never, + {} as never, + dataDbClientManager as never + ); + + await router.queryDataPrismaForTable( + 'tblxxx', + 'select * from x where id = $1', + { useTransaction: true }, + 'recxxx' + ); + + expect(dataDbClientManager.dataPrismaForTable).toHaveBeenCalledWith('tblxxx', { + useTransaction: true, + }); + expect(queryRaw).toHaveBeenCalledWith('select * from x where id = $1', 'recxxx'); + }); + + it('executes table scoped transactions with PrismaClient transaction fallback', async () => { + const executeRaw = vi.fn().mockResolvedValue(1); + const transaction = vi.fn(async (fn) => await fn({ $executeRawUnsafe: executeRaw })); + const dataDbClientManager = { + dataPrismaForTable: vi.fn().mockResolvedValue({ + $executeRawUnsafe: vi.fn(), + $queryRawUnsafe: vi.fn(), + $transaction: transaction, + }), + }; + const router = new DatabaseRouter( + {} as never, + {} as never, + {} as never, + {} as never, + dataDbClientManager as never + ); + + await router.dataPrismaTransactionForTable('tblxxx', async (prisma) => { + await prisma.$executeRawUnsafe('alter table x'); + }); + + expect(transaction).toHaveBeenCalledTimes(1); + expect(executeRaw).toHaveBeenCalledWith('alter table x'); + }); +}); diff --git a/apps/nestjs-backend/src/global/database-router.service.ts b/apps/nestjs-backend/src/global/database-router.service.ts index 627286dd24..3ffacbb7e6 100644 --- a/apps/nestjs-backend/src/global/database-router.service.ts +++ b/apps/nestjs-backend/src/global/database-router.service.ts @@ -4,8 +4,34 @@ import { getDatabaseUrl, MetaPrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; import { DataDbClientManager } from './data-db-client-manager.service'; +import type { IDataDbRoutingOptions } from './data-db-client-manager.service'; import { DATA_KNEX, META_KNEX } from './knex'; +export type IDataPrismaQueryExecutor = { + $queryRawUnsafe(query: string, ...values: unknown[]): Promise; + $executeRawUnsafe(query: string, ...values: unknown[]): Promise; +}; + +type IDataPrismaScopedClient = IDataPrismaQueryExecutor & { + txClient?: () => IDataPrismaQueryExecutor; + $tx?: ( + fn: (prisma: IDataPrismaQueryExecutor) => Promise, + options?: { + maxWait?: number; + timeout?: number; + isolationLevel?: unknown; + } + ) => Promise; + $transaction?: ( + fn: (prisma: IDataPrismaQueryExecutor) => Promise, + options?: { + maxWait?: number; + timeout?: number; + isolationLevel?: unknown; + } + ) => Promise; +}; + @Injectable() export class DatabaseRouter { constructor( @@ -36,23 +62,177 @@ export class DatabaseRouter { return getDatabaseUrl(target); } - async getDataDatabaseUrlForSpace(spaceId: string) { - return await this.dataDbClientManager.getDataDatabaseUrlForSpace(spaceId); + async getDataDatabaseUrlForSpace(spaceId: string, options?: IDataDbRoutingOptions) { + return await this.dataDbClientManager.getDataDatabaseUrlForSpace(spaceId, options); + } + + async getDataDatabaseUrlForTable(tableId: string, options?: IDataDbRoutingOptions) { + return await this.dataDbClientManager.getDataDatabaseUrlForTable(tableId, options); + } + + async getDataDatabaseUrlForBase(baseId: string, options?: IDataDbRoutingOptions) { + return await this.dataDbClientManager.getDataDatabaseUrlForBase(baseId, options); + } + + async dataKnexForSpace(spaceId: string, options?: IDataDbRoutingOptions) { + return await this.dataDbClientManager.dataKnexForSpace(spaceId, options); + } + + async dataPrismaForSpace(spaceId: string, options?: IDataDbRoutingOptions) { + return await this.dataDbClientManager.dataPrismaForSpace(spaceId, options); + } + + async dataKnexForBase(baseId: string, options?: IDataDbRoutingOptions) { + return await this.dataDbClientManager.dataKnexForBase(baseId, options); + } + + async dataPrismaForBase(baseId: string, options?: IDataDbRoutingOptions) { + return await this.dataDbClientManager.dataPrismaForBase(baseId, options); + } + + async dataKnexForTable(tableId: string, options?: IDataDbRoutingOptions) { + return await this.dataDbClientManager.dataKnexForTable(tableId, options); + } + + async dataPrismaForTable(tableId: string, options?: IDataDbRoutingOptions) { + return await this.dataDbClientManager.dataPrismaForTable(tableId, options); + } + + private getDataPrismaExecutor(prisma: IDataPrismaScopedClient): IDataPrismaQueryExecutor { + return prisma.txClient?.() ?? prisma; } - async dataKnexForSpace(spaceId: string) { - return await this.dataDbClientManager.dataKnexForSpace(spaceId); + async dataPrismaExecutorForTable( + tableId: string, + options?: IDataDbRoutingOptions + ): Promise { + const prisma = (await this.dataPrismaForTable(tableId, options)) as IDataPrismaScopedClient; + return this.getDataPrismaExecutor(prisma); } - async dataPrismaForSpace(spaceId: string) { - return await this.dataDbClientManager.dataPrismaForSpace(spaceId); + async dataPrismaExecutorForBase( + baseId: string, + options?: IDataDbRoutingOptions + ): Promise { + const prisma = (await this.dataPrismaForBase(baseId, options)) as IDataPrismaScopedClient; + return this.getDataPrismaExecutor(prisma); } - async dataKnexForBase(baseId: string) { - return await this.dataDbClientManager.dataKnexForBase(baseId); + async queryDataPrismaForTable( + tableId: string, + query: string, + optionsOrFirstValue?: IDataDbRoutingOptions | unknown, + ...values: unknown[] + ): Promise { + const { options, queryValues } = this.normalizeRoutingOptions(optionsOrFirstValue, values); + const prisma = await this.dataPrismaExecutorForTable(tableId, options); + return await prisma.$queryRawUnsafe(query, ...queryValues); } - async dataPrismaForBase(baseId: string) { - return await this.dataDbClientManager.dataPrismaForBase(baseId); + async executeDataPrismaForTable( + tableId: string, + query: string, + optionsOrFirstValue?: IDataDbRoutingOptions | unknown, + ...values: unknown[] + ): Promise { + const { options, queryValues } = this.normalizeRoutingOptions(optionsOrFirstValue, values); + const prisma = await this.dataPrismaExecutorForTable(tableId, options); + return await prisma.$executeRawUnsafe(query, ...queryValues); + } + + async queryDataPrismaForBase( + baseId: string, + query: string, + optionsOrFirstValue?: IDataDbRoutingOptions | unknown, + ...values: unknown[] + ): Promise { + const { options, queryValues } = this.normalizeRoutingOptions(optionsOrFirstValue, values); + const prisma = await this.dataPrismaExecutorForBase(baseId, options); + return await prisma.$queryRawUnsafe(query, ...queryValues); + } + + async executeDataPrismaForBase( + baseId: string, + query: string, + optionsOrFirstValue?: IDataDbRoutingOptions | unknown, + ...values: unknown[] + ): Promise { + const { options, queryValues } = this.normalizeRoutingOptions(optionsOrFirstValue, values); + const prisma = await this.dataPrismaExecutorForBase(baseId, options); + return await prisma.$executeRawUnsafe(query, ...queryValues); + } + + async dataPrismaTransactionForTable( + tableId: string, + fn: (prisma: IDataPrismaQueryExecutor) => Promise, + options?: { + maxWait?: number; + timeout?: number; + isolationLevel?: unknown; + }, + routingOptions?: IDataDbRoutingOptions + ): Promise { + const prisma = (await this.dataPrismaForTable( + tableId, + routingOptions + )) as IDataPrismaScopedClient; + + if (prisma.$tx) { + return await prisma.$tx(fn, options); + } + + if (prisma.$transaction) { + return await prisma.$transaction(fn, options); + } + + return await fn(this.getDataPrismaExecutor(prisma)); + } + + async dataPrismaTransactionForBase( + baseId: string, + fn: (prisma: IDataPrismaQueryExecutor) => Promise, + options?: { + maxWait?: number; + timeout?: number; + isolationLevel?: unknown; + }, + routingOptions?: IDataDbRoutingOptions + ): Promise { + const prisma = (await this.dataPrismaForBase( + baseId, + routingOptions + )) as IDataPrismaScopedClient; + + if (prisma.$tx) { + return await prisma.$tx(fn, options); + } + + if (prisma.$transaction) { + return await prisma.$transaction(fn, options); + } + + return await fn(this.getDataPrismaExecutor(prisma)); + } + + private isRoutingOptions(value: unknown): value is IDataDbRoutingOptions { + return ( + Boolean(value) && + typeof value === 'object' && + Object.keys(value as Record).length > 0 && + Object.keys(value as Record).every((key) => key === 'useTransaction') + ); + } + + private normalizeRoutingOptions( + optionsOrFirstValue: IDataDbRoutingOptions | unknown, + values: unknown[] + ): { options?: IDataDbRoutingOptions; queryValues: unknown[] } { + if (this.isRoutingOptions(optionsOrFirstValue)) { + return { options: optionsOrFirstValue, queryValues: values }; + } + + return { + queryValues: optionsOrFirstValue === undefined ? values : [optionsOrFirstValue, ...values], + }; } } diff --git a/apps/nestjs-backend/src/global/global.module.ts b/apps/nestjs-backend/src/global/global.module.ts index 5003a92c43..9bbcb526cc 100644 --- a/apps/nestjs-backend/src/global/global.module.ts +++ b/apps/nestjs-backend/src/global/global.module.ts @@ -24,12 +24,14 @@ import { PermissionGuard } from '../features/auth/guard/permission.guard'; import { PermissionModule } from '../features/auth/permission.module'; import { DataLoaderModule } from '../features/data-loader/data-loader.module'; import { ModelModule } from '../features/model/model.module'; +import { DataDbMigrationService } from '../features/space/data-db-migration.service'; import { RequestInfoMiddleware } from '../middleware/request-info.middleware'; import { SessionCsrfMiddleware } from '../middleware/session-csrf.middleware'; import { PerformanceCacheModule } from '../performance-cache'; import { RouteTracingInterceptor } from '../tracing/route-tracing.interceptor'; import { getI18nPath, getI18nTypesOutputPath } from '../utils/i18n'; import { DataDbClientManager } from './data-db-client-manager.service'; +import { DataDbRuntimeCacheService } from './data-db-runtime-cache.service'; import { DatabaseRouter } from './database-router.service'; import { KnexModule } from './knex'; @@ -93,7 +95,9 @@ const globalModules = { // for overriding the default TablePermissionService, FieldPermissionService, RecordPermissionService, and ViewPermissionService providers: [ DbProvider, + DataDbRuntimeCacheService, DataDbClientManager, + DataDbMigrationService, DatabaseRouter, RequestInfoMiddleware, SessionCsrfMiddleware, @@ -112,7 +116,9 @@ const globalModules = { ], exports: [ DbProvider, + DataDbRuntimeCacheService, DataDbClientManager, + DataDbMigrationService, DatabaseRouter, KnexModule, PrismaModule, diff --git a/apps/nestjs-backend/src/share-db/readonly/record-readonly.service.ts b/apps/nestjs-backend/src/share-db/readonly/record-readonly.service.ts index 286c6b28f0..9286f2ddb2 100644 --- a/apps/nestjs-backend/src/share-db/readonly/record-readonly.service.ts +++ b/apps/nestjs-backend/src/share-db/readonly/record-readonly.service.ts @@ -1,11 +1,11 @@ import { Injectable, NotFoundException } from '@nestjs/common'; import { PrismaService } from '@teable/db-main-prisma'; -import { DataPrismaService } from '@teable/db-data-prisma'; import type { IGetRecordsRo } from '@teable/openapi'; import { IS_TEMPLATE_HEADER, BASE_SHARE_ID_HEADER } from '@teable/openapi'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; +import { DatabaseRouter } from '../../global/database-router.service'; import { DATA_KNEX } from '../../global/knex/knex.module'; import type { IShareDbReadonlyAdapterService, RawOpType } from '../interface'; import { ReadonlyService } from './readonly.service'; @@ -19,7 +19,7 @@ export class RecordReadonlyServiceAdapter constructor( private readonly cls: ClsService, private readonly prismaService: PrismaService, - private readonly dataPrismaService: DataPrismaService, + private readonly databaseRouter: DatabaseRouter, @InjectModel(DATA_KNEX) private readonly knex: Knex ) { super(cls); @@ -99,17 +99,11 @@ export class RecordReadonlyServiceAdapter async getVersionAndType(tableId: string, recordId: string) { const table = await this.validateTable(tableId); - return this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ version: number; deletedTime: Date | null }[]>( - this.knex(table.dbTableName) - .select('__version as version') - .where('__id', recordId) - .toQuery() - ) - .then((res) => { - return this.formatVersionAndType(res[0]); - }); + return this.databaseRouter + .queryDataPrismaForTable< + { version: number; deletedTime: Date | null }[] + >(tableId, this.knex(table.dbTableName).select('__version as version').where('__id', recordId).toQuery()) + .then((res) => this.formatVersionAndType(res[0])); } async getVersionAndTypeMap(tableId: string, recordIds: string[]) { @@ -118,9 +112,9 @@ export class RecordReadonlyServiceAdapter .select('__version as version', '__id') .whereIn('__id', recordIds) .toQuery(); - const recordRaw = await this.dataPrismaService - .txClient() - .$queryRawUnsafe<{ version: number; deletedTime: Date | null; __id: string }[]>(nativeQuery); + const recordRaw = await this.databaseRouter.queryDataPrismaForTable< + { version: number; deletedTime: Date | null; __id: string }[] + >(tableId, nativeQuery); return recordRaw.reduce( (acc, record) => { acc[record.__id] = this.formatVersionAndType(record); diff --git a/apps/nestjs-backend/src/tracing-db-context.spec.ts b/apps/nestjs-backend/src/tracing-db-context.spec.ts new file mode 100644 index 0000000000..a91796132b --- /dev/null +++ b/apps/nestjs-backend/src/tracing-db-context.spec.ts @@ -0,0 +1,94 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + resolveTeableDbTraceContext, + setTeableDbSpanAttributes, + setTeableDbSpanAttributesFromSpan, +} from './tracing-db-context'; + +describe('tracing db context', () => { + const env = { + PRISMA_DATABASE_URL: 'postgresql://postgres:secret@meta.example.test:5433/teable', + }; + + it('marks connections matching the meta URL as meta and redacts the password', () => { + expect( + resolveTeableDbTraceContext( + { + database: 'teable', + host: 'meta.example.test', + port: 5433, + user: 'postgres', + }, + env + ) + ).toEqual({ + role: 'meta', + source: 'PRISMA_DATABASE_URL', + url: 'postgresql://postgres@meta.example.test:5433/teable', + }); + }); + + it('marks non-meta postgres connections as dynamic data DB connections', () => { + expect( + resolveTeableDbTraceContext( + { + database: 'postgres', + host: 'byodb.example.test', + port: 5544, + user: 'postgres', + }, + env + ) + ).toEqual({ + role: 'data', + source: 'inferred.non_meta_postgres', + url: 'postgresql://postgres@byodb.example.test:5544/postgres', + }); + }); + + it('writes teable db attributes to query spans', () => { + const span = { setAttribute: vi.fn() }; + + setTeableDbSpanAttributes( + span, + { + database: 'teable', + host: 'meta.example.test', + port: 5433, + user: 'postgres', + }, + env + ); + + expect(span.setAttribute).toHaveBeenCalledWith('teable.db.role', 'meta'); + expect(span.setAttribute).toHaveBeenCalledWith( + 'teable.db.url', + 'postgresql://postgres@meta.example.test:5433/teable' + ); + expect(span.setAttribute).toHaveBeenCalledWith('teable.db.source', 'PRISMA_DATABASE_URL'); + }); + + it('writes teable db attributes to connection spans from existing span attributes', () => { + const span = { + attributes: { + 'db.name': 'teable_data', + 'db.user': 'postgres', + 'net.peer.name': 'data.example.test', + 'net.peer.port': 5434, + }, + setAttribute: vi.fn(), + }; + + setTeableDbSpanAttributesFromSpan(span, env); + + expect(span.setAttribute).toHaveBeenCalledWith('teable.db.role', 'data'); + expect(span.setAttribute).toHaveBeenCalledWith( + 'teable.db.url', + 'postgresql://postgres@data.example.test:5434/teable_data' + ); + expect(span.setAttribute).toHaveBeenCalledWith( + 'teable.db.source', + 'inferred.non_meta_postgres' + ); + }); +}); diff --git a/apps/nestjs-backend/src/tracing-db-context.ts b/apps/nestjs-backend/src/tracing-db-context.ts new file mode 100644 index 0000000000..f12c8699ec --- /dev/null +++ b/apps/nestjs-backend/src/tracing-db-context.ts @@ -0,0 +1,149 @@ +type IEnv = Record; + +export type ITraceDbRole = 'meta' | 'data'; + +export type ITraceDbConnection = { + database?: string; + host?: string; + port?: number | string; + user?: string; +}; + +export type ITraceDbContext = { + role: ITraceDbRole; + url: string; + source: string; +}; + +export type ITraceDbSpan = { + setAttribute(key: string, value: string): void; +}; + +const META_DATABASE_URL_KEYS = ['PRISMA_META_DATABASE_URL', 'PRISMA_DATABASE_URL', 'DATABASE_URL']; + +const normalizeDatabaseName = (value?: string) => value?.replace(/^\//, '') || undefined; + +const normalizePort = (value?: string | number) => { + if (value == null || value === '') { + return 5432; + } + + const port = Number(value); + return Number.isFinite(port) ? port : 5432; +}; + +const normalizeConnection = (connection: ITraceDbConnection) => ({ + database: normalizeDatabaseName(connection.database), + host: connection.host?.toLowerCase(), + port: normalizePort(connection.port), + user: connection.user, +}); + +const parseDatabaseUrl = (url: string | undefined) => { + if (!url) { + return; + } + + try { + const parsed = new URL(url); + const userPart = parsed.username ? `${decodeURIComponent(parsed.username)}@` : ''; + const port = normalizePort(parsed.port); + const database = normalizeDatabaseName(parsed.pathname); + + return { + database, + host: parsed.hostname.toLowerCase(), + port, + user: parsed.username ? decodeURIComponent(parsed.username) : undefined, + url: `${parsed.protocol}//${userPart}${parsed.hostname}${parsed.port ? `:${parsed.port}` : ''}${database ? `/${database}` : ''}`, + }; + } catch { + return; + } +}; + +const findMatchingDatabaseUrl = ( + connection: ReturnType, + keys: string[], + env: IEnv +) => { + for (const key of keys) { + const candidate = parseDatabaseUrl(env[key]); + if (!candidate) { + continue; + } + + if ( + candidate.host === connection.host && + candidate.port === connection.port && + candidate.database === connection.database + ) { + return { key, url: candidate.url }; + } + } +}; + +const buildConnectionUrl = (connection: ReturnType) => { + const userPart = connection.user ? `${connection.user}@` : ''; + const host = connection.host || 'unknown-host'; + const databasePart = connection.database ? `/${connection.database}` : ''; + return `postgresql://${userPart}${host}:${connection.port}${databasePart}`; +}; + +export const resolveTeableDbTraceContext = ( + connection: ITraceDbConnection, + env: IEnv = process.env +): ITraceDbContext => { + const normalized = normalizeConnection(connection); + const metaMatch = findMatchingDatabaseUrl(normalized, META_DATABASE_URL_KEYS, env); + if (metaMatch) { + return { + role: 'meta', + url: metaMatch.url, + source: metaMatch.key, + }; + } + + return { + role: 'data', + url: buildConnectionUrl(normalized), + source: 'inferred.non_meta_postgres', + }; +}; + +export const setTeableDbSpanAttributes = ( + span: ITraceDbSpan, + connection: ITraceDbConnection, + env: IEnv = process.env +) => { + const context = resolveTeableDbTraceContext(connection, env); + span.setAttribute('teable.db.role', context.role); + span.setAttribute('teable.db.url', context.url); + span.setAttribute('teable.db.source', context.source); +}; + +export const setTeableDbSpanAttributesFromSpan = ( + span: ITraceDbSpan & { attributes?: Record }, + env: IEnv = process.env +) => { + const attributes = span.attributes ?? {}; + const host = attributes['net.peer.name'] ?? attributes['server.address']; + const port = attributes['net.peer.port'] ?? attributes['server.port']; + const database = attributes['db.name'] ?? attributes['db.namespace']; + const user = attributes['db.user'] ?? attributes['db.user.name']; + + if (!host && !database) { + return; + } + + setTeableDbSpanAttributes( + span, + { + database: typeof database === 'string' ? database : undefined, + host: typeof host === 'string' ? host : undefined, + port: typeof port === 'string' || typeof port === 'number' ? port : undefined, + user: typeof user === 'string' ? user : undefined, + }, + env + ); +}; diff --git a/apps/nestjs-backend/src/tracing.ts b/apps/nestjs-backend/src/tracing.ts index c390745869..c323c5431e 100644 --- a/apps/nestjs-backend/src/tracing.ts +++ b/apps/nestjs-backend/src/tracing.ts @@ -55,6 +55,7 @@ import { SentrySpanProcessor, wrapContextManagerClass, } from '@sentry/opentelemetry'; +import { setTeableDbSpanAttributes, setTeableDbSpanAttributesFromSpan } from './tracing-db-context'; // Use webpack's special require that bypasses bundling, falling back to standard require // This is needed because webpack transforms import.meta.url and createRequire in ways @@ -267,12 +268,30 @@ const httpClientActiveRequestsProcessor: SpanProcessor = { forceFlush: () => Promise.resolve(), }; +const teableDbSpanAttributeProcessor: SpanProcessor = { + onStart(span): void { + const attributes = (span as unknown as { attributes?: Record }).attributes; + const dbSystem = attributes?.['db.system']; + if (dbSystem !== 'postgresql' && dbSystem !== 'postgres') { + return; + } + + setTeableDbSpanAttributesFromSpan( + span as unknown as Parameters[0] + ); + }, + onEnd: () => undefined, + shutdown: () => Promise.resolve(), + forceFlush: () => Promise.resolve(), +}; + // Span processors - NoopSpanProcessor ensures trace context is always generated // even when no exporter is configured (needed for trace ID in logs) const spanProcessors = [ ...(hasSentry ? [new SentrySpanProcessor()] : []), ...(traceExporter ? [createSmartBatchProcessor(traceExporter)] : [new NoopSpanProcessor()]), httpClientActiveRequestsProcessor, + teableDbSpanAttributeProcessor, ]; // When Sentry is enabled, use SentryPropagator and SentryContextManager to ensure @@ -351,6 +370,9 @@ const otelSDK = new opentelemetry.NodeSDK({ new PgInstrumentation({ enhancedDatabaseReporting: true, // Records SQL; ensure sensitive data is scrubbed. requireParentSpan: false, // Create spans even without parent, ensures v2 Kysely queries are traced + requestHook: (span, queryInfo) => { + setTeableDbSpanAttributes(span, queryInfo.connection); + }, }), new PinoInstrumentation(), new RuntimeNodeInstrumentation(), diff --git a/apps/nestjs-backend/src/types/cls.ts b/apps/nestjs-backend/src/types/cls.ts index 06a2edd6f8..68a71fea53 100644 --- a/apps/nestjs-backend/src/types/cls.ts +++ b/apps/nestjs-backend/src/types/cls.ts @@ -7,7 +7,7 @@ import type { IPerformanceCacheStore } from '../performance-cache'; import type { IRawOpMap } from '../share-db/interface'; import type { IDataLoaderCache } from './data-loader'; -export type V2Reason = +export type IV2Reason = | 'env_force_v2_all' | 'config_force_v2_all' | 'new_base' @@ -62,6 +62,15 @@ export interface IClsStore extends ClsStore { permissions: Action[]; // this is used to check if the user is in the space when the user operate in a space spaceId?: string; + dataDb?: { + mode: 'byodb'; + spaceId: string; + connectionId: string; + urlFingerprint?: string | null; + displayHost?: string | null; + displayDatabase?: string | null; + internalSchema?: string | null; + }; // for share db adapter cookie?: string; oldField?: IFieldVo; @@ -82,7 +91,7 @@ export interface IClsStore extends ClsStore { clearCacheKeys?: (keyof IPerformanceCacheStore)[]; canaryHeader?: string; // x-canary header value for canary release override useV2?: boolean; // Flag to indicate if V2 implementation should be used (set by V2FeatureGuard) - v2Reason?: V2Reason; // Reason why V2 was enabled or disabled + v2Reason?: IV2Reason; // Reason why V2 was enabled or disabled v2Feature?: V2Feature; // The feature name that triggered V2 check windowId?: string; // Window ID from x-window-id header for undo/redo tracking skipFieldComputation?: boolean; // Skip computed field evaluation during bulk structure creation (import/duplicate) diff --git a/apps/nestjs-backend/src/types/i18n.generated.ts b/apps/nestjs-backend/src/types/i18n.generated.ts index c7eda51e38..9595f711d8 100644 --- a/apps/nestjs-backend/src/types/i18n.generated.ts +++ b/apps/nestjs-backend/src/types/i18n.generated.ts @@ -1715,6 +1715,7 @@ export type I18nTranslations = { "modelServiceError": string; "imageProcessingFailed": string; "imageProcessingFailedDescription": string; + "sandboxSameChatBusy": string; "sandboxBusy": string; "sandboxCapacityFull": string; "sandboxTransient": string; @@ -2792,6 +2793,8 @@ export type I18nTranslations = { }; "selectionStatistic": { "tip": string; + "copyTip": string; + "copied": string; }; "baseQuery": { "add": string; @@ -2903,6 +2906,31 @@ export type I18nTranslations = { "automationNodeNeedTest": string; "automationNodeTestOutdated": string; "invalidToken": string; + "limit": { + "fieldOptionsMaxBytes": string; + "selectChoicesMax": string; + "selectChoiceNameMaxLength": string; + "selectDefaultValuesMax": string; + "cellValueMaxBytes": string; + "recordFieldsMaxBytes": string; + "recordsPerMutationMax": string; + "computedCellValueMaxBytes": string; + "formulaMaxLength": string; + "tablesPerBaseMax": string; + "fieldsPerTableMax": string; + "rowsPerTableMax": string; + "viewsPerTableMax": string; + "createTableFieldsMax": string; + "createTableViewsMax": string; + "createTableRecordsMax": string; + "viewFilterItemsMax": string; + "viewFilterDepthMax": string; + "viewSortItemsMax": string; + "viewGroupItemsMax": string; + "viewOptionsMaxBytes": string; + "nameMaxLength": string; + "descriptionMaxLength": string; + }; "custom": { "fieldValueNotNull": string; "fieldValueDuplicate": string; @@ -3025,8 +3053,8 @@ export type I18nTranslations = { "cannotOperate": string; "notBelongToOrg": string; "invalidSpaceIds": string; - "ownedSpaceLimitExceeded": string; - "ownedSpaceLimitExceededOther": string; + "freeOwnedSpaceLimitExceeded": string; + "freeOwnedSpaceLimitExceededOther": string; }; "base": { "notFound": string; @@ -3190,6 +3218,7 @@ export type I18nTranslations = { "recordMapNotFound": string; "forbidDeletePrimaryField": string; "foreignTableIdInvalid": string; + "crossSpaceLinkForbidden": string; "relationshipInvalid": string; "linkFieldIdInvalid": string; "lookupFieldIdInvalid": string; @@ -3486,6 +3515,12 @@ export type I18nTranslations = { }; }; "space": { + "crossSpace": { + "duplicateBaseTitle": string; + "duplicateBaseDescription": string; + "affectedTableSuffix": string; + "convertAndDuplicate": string; + }; "initialSpaceName": string; "action": { "createBase": string; @@ -3548,6 +3583,8 @@ export type I18nTranslations = { "exportReadyDescription": string; "moveBaseSuccessTitle": string; "moveBaseSuccessDescription": string; + "moveBaseCrossSpaceTitle": string; + "moveBaseCrossSpaceDataLossWarning": string; }; "deleteSpaceModal": { "title": string; @@ -3722,8 +3759,97 @@ export type I18nTranslations = { }; "collaborators": string; "more": string; + "export": { + "phase": { + "preparing": string; + "exportingArchive": string; + "exportingStructure": string; + "exportingAttachments": string; + "exportingAttachmentMetadata": string; + "exportingTableData": string; + "tableDataStarted": string; + "tableDataProgress": string; + "tableDataDone": string; + "exportingExtraFiles": string; + "exportingAppFiles": string; + "uploadingArchive": string; + "generatingDownloadUrl": string; + "rowsProgress": string; + "done": string; + }; + }; + "dataDb": { + "create": { + "title": string; + "description": string; + "defaultOption": string; + "defaultHint": string; + "byodbOption": string; + "byodbHint": string; + "urlLabel": string; + "sslHint": string; + "testConnection": string; + "testing": string; + "retestRequired": string; + "databaseLabel": string; + "databasePlaceholder": string; + "databaseHint": string; + "preflightPassed": string; + "preflightFailed": string; + "missingCapabilities": string; + "testFailed": string; + "errors": { + "INVALID_DATABASE_URL": { + "message": string; + }; + "PRIVATE_NETWORK_BLOCKED": { + "message": string; + "remediation": string; + }; + "CONNECTION_FAILED": { + "message": string; + "remediation": string; + }; + "IPV6_NETWORK_UNREACHABLE": { + "message": string; + "remediation": string; + }; + "PRIVILEGE_CHECK_FAILED": { + "message": string; + }; + "DDL_PRIVILEGE_CHECK_FAILED": { + "message": string; + "remediation": string; + }; + "NON_EMPTY_UNKNOWN_DATABASE": { + "message": string; + "remediation": string; + }; + "INCOMPATIBLE_TEABLE_DATABASE": { + "message": string; + "remediation": string; + }; + }; + }; + "fields": { + "host": string; + "database": string; + "internalSchema": string; + "version": string; + "classification": string; + }; + }; }; "table": { + "crossSpace": { + "duplicateFieldTitle": string; + "duplicateFieldDescription": string; + "duplicateTableTitle": string; + "duplicateTableDescription": string; + "duplicateBaseTitle": string; + "duplicateBaseDescription": string; + "convertAndDuplicate": string; + }; "toolbar": { "comingSoon": string; "viewFilterInShare": string; @@ -4281,6 +4407,7 @@ export type I18nTranslations = { "noFocus": string; "noPermission": string; }; + "pasteNoEditableFields": string; "clearFailed": string; "clearConfirmTitle": string; "clearConfirmDescription": string; @@ -4528,6 +4655,8 @@ export type I18nTranslations = { "foreignKeyOrphanRows": string; "junctionForeignKeyTargetTableMissing": string; "junctionForeignKeyOrphanRows": string; + "symmetricFieldMissing": string; + "symmetricFieldBroken": string; }; "description": { "symmetricFieldConflict": string; @@ -4536,6 +4665,8 @@ export type I18nTranslations = { "junctionForeignKeyTargetTableMissing": string; "junctionForeignKeyOrphanRows": string; "autoRule": string; + "symmetricFieldMissing": string; + "symmetricFieldBroken": string; }; "manual": { "apply": string; @@ -4550,6 +4681,42 @@ export type I18nTranslations = { "convertDuplicate": string; }; }; + "symmetricFieldMissing": { + "title": string; + "description": string; + "resolutionLabel": string; + "resolutionDescription": string; + "option": { + "convertCurrent": string; + }; + }; + "symmetricFieldBroken": { + "title": string; + "description": string; + "resolutionLabel": string; + "resolutionDescription": string; + "option": { + "convertCurrent": string; + }; + }; + "foreignKeyOrphanRows": { + "title": string; + "description": string; + "resolutionLabel": string; + "resolutionDescription": string; + "option": { + "clearOrphanValues": string; + }; + }; + "junctionForeignKeyOrphanRows": { + "title": string; + "description": string; + "resolutionLabel": string; + "resolutionDescription": string; + "option": { + "deleteOrphanRows": string; + }; + }; }; }; "manualRepairPreview": string; @@ -4935,6 +5102,10 @@ export type I18nTranslations = { "contextTipNewChat": string; "contextTipMemory": string; }; + "contextCompaction": { + "auto": string; + "manual": string; + }; "taskProgress": { "title": string; }; diff --git a/apps/nestjs-backend/test/attachment.e2e-spec.ts b/apps/nestjs-backend/test/attachment.e2e-spec.ts index 860f6de6f8..1817b7081f 100644 --- a/apps/nestjs-backend/test/attachment.e2e-spec.ts +++ b/apps/nestjs-backend/test/attachment.e2e-spec.ts @@ -142,6 +142,41 @@ describe('OpenAPI AttachmentController (e2e)', () => { expect(attachment.smThumbnailUrl).not.toBe(attachment.presignedUrl); }); + it('should keep cross-origin headers on the 304 cache-hit read path', async () => { + const field = await createField(table.id, { type: FieldType.Attachment }); + const uploadResult = await uploadAttachment( + table.id, + table.records[0].id, + field.id, + fs.createReadStream(filePath) + ); + expect(uploadResult.status).toBe(201); + + const attachment = (uploadResult.data.fields[field.id] as IAttachmentCellValue)[0]!; + const presignedUrl = attachment.presignedUrl ?? ''; + const readUrl = presignedUrl.startsWith('http') ? presignedUrl : `${appUrl}${presignedUrl}`; + + const axios = createAxios(); + axios.defaults.validateStatus = (status) => status === 200 || status === 304; + + // The 200 read sets a non-`same-origin` CORP so the attachment can be + // embedded cross-origin. + const firstRes = await axios.get(readUrl, { responseType: 'arraybuffer' }); + expect(firstRes.status).toBe(200); + const corp = firstRes.headers['cross-origin-resource-policy']; + expect(corp).not.toBe('same-origin'); + + // Regression: revalidation returns 304 — it must carry the same CORP header + // as the 200 read, otherwise helmet's default `same-origin` leaks into the + // 304 and the browser blocks the cross-origin embedded attachment. + const cachedRes = await axios.get(readUrl, { + responseType: 'arraybuffer', + headers: { 'If-Modified-Since': firstRes.headers['last-modified'] }, + }); + expect(cachedRes.status).toBe(304); + expect(cachedRes.headers['cross-origin-resource-policy']).toBe(corp); + }); + it('should write attachment with simplified ro format without typecast', async () => { // Step 1: Upload attachment to get token const field = await createField(table.id, { type: FieldType.Attachment }); diff --git a/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts b/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts index b79d194882..ebaf7841ab 100644 --- a/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts +++ b/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts @@ -42,6 +42,7 @@ import { listPluginPanels, LLMProviderType, moveBaseNode, + SettingKey, updateSetting, urlBuilder, } from '@teable/openapi'; @@ -53,8 +54,9 @@ import { createTable, getRecords, initApp, - updateRecord, permanentDeleteBase, + permanentDeleteSpace, + updateRecord, } from './utils/init-app'; describe('OpenAPI Base Duplicate (e2e)', () => { @@ -103,42 +105,109 @@ describe('OpenAPI Base Duplicate (e2e)', () => { return; } - it('duplicate base with cross base link and lookup field', async () => { + it('duplicate base with cross-space link/lookup downgrades values to title text', async () => { + // Source base (`base`) and link target (`base2`) share spaceId, so the link + // is legal at create time (same-space cross-base). The duplicate destination + // is a SEPARATE space — from its perspective the preserved link would point + // back into another space, which the duplicate must downgrade to text. const base2 = (await createBase({ spaceId, name: 'test base 2' })).data; - const base2Table = await createTable(base2.id, { name: 'table1' }); + const destSpace = (await createSpace({ name: 'duplicate dest space' })).data; + try { + const base2Table = await createTable(base2.id, { name: 'table1', records: [] }); + const base2PrimaryId = base2Table.fields[0].id; + + // Peer record sets the title we expect to see flow into the duplicated + // base as plain text after the cross-space link/lookup are downgraded. + const peerRecord = ( + await createRecords(base2Table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [base2PrimaryId]: 'peer-A' } }], + }) + ).records[0]; - const table1 = await createTable(base.id, { name: 'table1' }); + const table1 = await createTable(base.id, { name: 'table1', records: [] }); + const table1Primary = table1.fields.find((f) => f.isPrimary)!; - const crossBaseLinkField = ( - await createField(table1.id, { - name: 'cross base link field', - type: FieldType.Link, - options: { - baseId: base2.id, - relationship: Relationship.ManyMany, - foreignTableId: base2Table.id, - }, - }) - ).data; + const crossBaseLinkField = ( + await createField(table1.id, { + name: 'cross base link field', + type: FieldType.Link, + options: { + baseId: base2.id, + relationship: Relationship.ManyMany, + foreignTableId: base2Table.id, + }, + }) + ).data; - await createField(table1.id, { - name: 'cross base lookup field', - type: FieldType.SingleLineText, - isLookup: true, - lookupOptions: { - foreignTableId: base2Table.id, - linkFieldId: crossBaseLinkField.id, - lookupFieldId: base2Table.fields[0].id, - }, - }); + const crossBaseLookupField = ( + await createField(table1.id, { + name: 'cross base lookup field', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: base2Table.id, + linkFieldId: crossBaseLinkField.id, + lookupFieldId: base2PrimaryId, + }, + }) + ).data; - const dupResult = await duplicateBase({ - fromBaseId: base.id, - spaceId: spaceId, - name: 'test base copy', - }); + await createRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [table1Primary.id]: 'src-1', + [crossBaseLinkField.id]: [{ id: peerRecord.id }], + }, + }, + ], + }); - expect(dupResult.status).toBe(201); + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: destSpace.id, + name: 'test base copy', + withRecords: true, + }); + expect(dupResult.status).toBe(201); + duplicateBaseId = dupResult.data.id; + + // Field downgrade is the easy half — the cellValue downgrade is what + // multiple recent fixes (0d441780e / ac5da54d0 / 175e2c59c / 9490800a9) + // were chasing. Without this assertion a regression would silently leave + // the new text columns null while still appearing structurally correct. + const dupTables = (await getTableList(duplicateBaseId)).data; + const dupTable1 = dupTables.find((t) => t.name === 'table1')!; + const dupFields = (await getFields(dupTable1.id)).data; + const dupLinkField = dupFields.find((f) => f.name === 'cross base link field')!; + const dupLookupField = dupFields.find((f) => f.name === 'cross base lookup field')!; + + expect(dupLinkField.type).toBe(FieldType.SingleLineText); + expect(dupLookupField.type).toBe(FieldType.SingleLineText); + expect(dupLookupField.isLookup).toBeFalsy(); + + const dupRecords = await getRecords(dupTable1.id); + const dupRow = dupRecords.records[0]; + // Link's source DB column stores the cached title text directly, so the + // SQL-direct copy lands on a clean "peer-A" in the new text column. + // Lookup's source DB column stores the multi-value JSON array (the + // lookup engine's storage contract), and duplicateBase uses SQL-direct + // row copy with no cellValue2String pass on downgraded fields — so the + // raw '["peer-A"]' is what survives the move. This documents the + // intentional split: downgrade preserves data verbatim, it doesn't + // re-stringify it. + expect(dupRow.fields[dupLinkField.name]).toBe('peer-A'); + expect(dupRow.fields[dupLookupField.name]).toBe('["peer-A"]'); + } finally { + await permanentDeleteBase(base2.id); + if (duplicateBaseId) { + await permanentDeleteBase(duplicateBaseId); + duplicateBaseId = undefined; + } + await permanentDeleteSpace(destSpace.id); + } }); it('duplicate within current space', async () => { @@ -586,6 +655,674 @@ describe('OpenAPI Base Duplicate (e2e)', () => { ); }); + it('should duplicate a complex base through v2 canary', async () => { + const previousCanaryEnv = process.env.ENABLE_CANARY_FEATURE; + process.env.ENABLE_CANARY_FEATURE = 'true'; + + const externalBase = (await createBase({ spaceId, name: 'external link base' })).data; + try { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + forceV2All: true, + spaceIds: [spaceId], + }, + }); + + const externalTable = await createTable(externalBase.id, { name: 'vendors' }); + const externalRecord = ( + await createRecords(externalTable.id, { + records: [{ fields: { [externalTable.fields[0].id]: 'Vendor A' } }], + }) + ).records[0]; + + const peopleFolder = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Folder, + name: 'People Folder', + }).then((res) => res.data); + const taskFolder = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Folder, + name: 'Task Folder', + }).then((res) => res.data); + + const peopleNode = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'People', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Score', type: FieldType.Number }, + ], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + const taskNode = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'Tasks', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + await moveBaseNode(base.id, peopleNode.id, { parentId: peopleFolder.id }); + await moveBaseNode(base.id, taskNode.id, { parentId: taskFolder.id }); + + const [peopleDefaultRecords, taskDefaultRecords] = await Promise.all([ + getRecords(peopleNode.resourceId).then(({ records }) => records), + getRecords(taskNode.resourceId).then(({ records }) => records), + ]); + if (peopleDefaultRecords.length) { + await deleteRecords( + peopleNode.resourceId, + peopleDefaultRecords.map(({ id }) => id) + ); + } + if (taskDefaultRecords.length) { + await deleteRecords( + taskNode.resourceId, + taskDefaultRecords.map(({ id }) => id) + ); + } + + const peopleFields = (await getFields(peopleNode.resourceId)).data; + const peopleNameField = peopleFields.find(({ name }) => name === 'Name')!; + const scoreField = peopleFields.find(({ name }) => name === 'Score')!; + const taskTitleField = (await getFields(taskNode.resourceId)).data.find( + ({ name }) => name === 'Title' + )!; + const doubledScoreField = ( + await createField(peopleNode.resourceId, { + name: 'Doubled Score', + type: FieldType.Formula, + options: { expression: `{${scoreField.id}} * 2`, timeZone: 'Asia/Shanghai' }, + }) + ).data; + const ownerLinkField = ( + await createField(taskNode.resourceId, { + name: 'Owner', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: peopleNode.resourceId, + }, + }) + ).data; + const externalLinkField = ( + await createField(taskNode.resourceId, { + name: 'External Vendor', + type: FieldType.Link, + options: { + baseId: externalBase.id, + relationship: Relationship.ManyMany, + foreignTableId: externalTable.id, + }, + }) + ).data; + + const peopleRecord = ( + await createRecords(peopleNode.resourceId, { + records: [ + { + fields: { + [peopleNameField.id]: 'Alice', + [scoreField.id]: 11, + }, + }, + ], + }) + ).records[0]; + await createRecords(taskNode.resourceId, { + records: [ + { + fields: { + [taskTitleField.id]: 'Task A', + [ownerLinkField.id]: [{ id: peopleRecord.id }], + [externalLinkField.id]: [{ id: externalRecord.id }], + }, + }, + ], + }); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId, + name: 'complex v2 base copy', + withRecords: true, + }); + expect(dupResult.status).toBe(201); + duplicateBaseId = dupResult.data.id; + + const duplicatedTables = await getTableList(duplicateBaseId).then((res) => res.data); + const duplicatedPeopleTable = duplicatedTables.find(({ name }) => name === 'People')!; + const duplicatedTaskTable = duplicatedTables.find(({ name }) => name === 'Tasks')!; + expect(duplicatedPeopleTable.id).not.toBe(peopleNode.resourceId); + expect(duplicatedTaskTable.id).not.toBe(taskNode.resourceId); + + const duplicatedPeopleFields = (await getFields(duplicatedPeopleTable.id)).data; + const duplicatedDoubledScoreField = duplicatedPeopleFields.find( + ({ name }) => name === doubledScoreField.name + ); + expect(duplicatedDoubledScoreField?.options).toMatchObject({ + expression: expect.stringContaining( + duplicatedPeopleFields.find(({ name }) => name === scoreField.name)!.id + ), + }); + + const duplicatedTasks = await getRecords(duplicatedTaskTable.id, { + fieldKeyType: FieldKeyType.Name, + }); + expect(duplicatedTasks.records).toHaveLength(1); + expect(duplicatedTasks.records[0].fields[ownerLinkField.name]).toMatchObject([ + { title: 'Alice' }, + ]); + expect(duplicatedTasks.records[0].fields[externalLinkField.name]).toMatchObject([ + { title: 'Vendor A' }, + ]); + + const duplicatedNodeTree = await getBaseNodeTree(duplicateBaseId).then((res) => res.data); + const duplicatedPeopleFolder = duplicatedNodeTree.nodes.find( + ({ resourceMeta, resourceType }) => + resourceMeta?.name === peopleFolder.resourceMeta?.name && + resourceType === BaseNodeResourceType.Folder + ); + const duplicatedPeopleNode = duplicatedNodeTree.nodes.find( + ({ resourceMeta, resourceType }) => + resourceMeta?.name === peopleNode.resourceMeta?.name && + resourceType === BaseNodeResourceType.Table + ); + expect(duplicatedPeopleNode?.parentId).toBe(duplicatedPeopleFolder?.id); + } finally { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + spaceIds: [], + }, + }); + process.env.ENABLE_CANARY_FEATURE = previousCanaryEnv; + await permanentDeleteBase(externalBase.id); + } + }); + + describe('V2 canary duplicate parity', () => { + let previousCanaryEnv: string | undefined; + + beforeEach(async () => { + previousCanaryEnv = process.env.ENABLE_CANARY_FEATURE; + process.env.ENABLE_CANARY_FEATURE = 'true'; + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + forceV2All: true, + spaceIds: [spaceId], + }, + }); + }); + + afterEach(async () => { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + forceV2All: false, + spaceIds: [], + }, + }); + process.env.ENABLE_CANARY_FEATURE = previousCanaryEnv; + }); + + it('duplicates schema, records, and auto number through v2', async () => { + const table = await createTable(base.id, { name: 'Basic Table' }); + + const structureOnlyResult = await duplicateBase({ + fromBaseId: base.id, + spaceId, + name: 'v2 structure only copy', + }); + duplicateBaseId = structureOnlyResult.data.id; + + const structureOnlyTables = await getTableList(duplicateBaseId).then((res) => res.data); + const structureOnlyRecords = await getRecords(structureOnlyTables[0].id); + expect(structureOnlyTables).toHaveLength(1); + expect(structureOnlyTables[0].name).toBe(table.name); + expect(structureOnlyTables[0].id).not.toBe(table.id); + expect(structureOnlyRecords.records).toHaveLength(0); + + await permanentDeleteBase(duplicateBaseId); + duplicateBaseId = undefined; + + const sourceRecords = await getRecords(table.id); + await updateRecord(table.id, sourceRecords.records[0].id, { + record: { fields: { [table.fields[0].name]: 'new value' } }, + }); + + const withRecordsResult = await duplicateBase({ + fromBaseId: base.id, + spaceId, + name: 'v2 records copy', + withRecords: true, + }); + duplicateBaseId = withRecordsResult.data.id; + + const duplicatedTable = (await getTableList(duplicateBaseId)).data[0]; + const duplicatedRecords = await getRecords(duplicatedTable.id); + expect(duplicatedRecords.records).toHaveLength(3); + expect(duplicatedRecords.records[0].lastModifiedBy).toBeFalsy(); + expect(duplicatedRecords.records[0].createdTime).toBeTruthy(); + expect(duplicatedRecords.records[0].fields[table.fields[0].name]).toEqual('new value'); + + await createRecords(duplicatedTable.id, { records: [{ fields: {} }] }); + const recordsAfterCreate = await getRecords(duplicatedTable.id); + expect(recordsAfterCreate.records[recordsAfterCreate.records.length - 1].autoNumber).toEqual( + recordsAfterCreate.records.length + ); + }); + + it('duplicates formula, link, lookup, rollup, bidirectional link, and ai field config through v2', async () => { + const peopleTable = await createTable(base.id, { name: 'People' }); + const taskTable = await createTable(base.id, { name: 'Tasks' }); + + const peopleFields = (await getFields(peopleTable.id)).data; + const peoplePrimaryField = peopleFields.find(({ isPrimary }) => isPrimary)!; + const scoreField = ( + await createField(peopleTable.id, { + name: 'Score', + type: FieldType.Number, + }) + ).data; + const doubledScoreField = ( + await createField(peopleTable.id, { + name: 'Doubled Score', + type: FieldType.Formula, + options: { expression: `{${scoreField.id}} * 2`, timeZone: 'Asia/Shanghai' }, + }) + ).data; + const ownerLinkField = ( + await createField(taskTable.id, { + name: 'Owner', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: peopleTable.id, + }, + }) + ).data; + const ownerSymmetricField = ( + await getField( + peopleTable.id, + (ownerLinkField.options as ILinkFieldOptions).symmetricFieldId as string + ) + ).data; + const ownerLookupField = ( + await createField(taskTable.id, { + name: 'Owner Name', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: peopleTable.id, + linkFieldId: ownerLinkField.id, + lookupFieldId: peoplePrimaryField.id, + }, + }) + ).data; + const ownerScoreRollupField = ( + await createField(taskTable.id, { + name: 'Owner Score Sum', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: peopleTable.id, + linkFieldId: ownerLinkField.id, + lookupFieldId: scoreField.id, + }, + }) + ).data; + + const aiSetting = ( + await updateSetting({ + aiConfig: { + enable: true, + llmProviders: [ + { + apiKey: 'test-ai-config', + baseUrl: 'localhost:3000/api/test', + models: 'test-e2e', + name: 'test', + type: LLMProviderType.ANTHROPIC, + }, + ], + }, + }) + ).data; + const aiField = ( + await createField(peopleTable.id, { + name: 'ai field', + type: FieldType.SingleLineText, + aiConfig: { + attachPrompt: 'test-attach-prompt', + modelKey: aiSetting.aiConfig?.llmProviders[0].models, + sourceFieldId: peoplePrimaryField.id, + type: FieldAIActionType.Summary, + }, + }) + ).data; + + const peopleRecords = await getRecords(peopleTable.id); + const taskRecords = await getRecords(taskTable.id); + await updateRecord(peopleTable.id, peopleRecords.records[0].id, { + record: { + fields: { + [peoplePrimaryField.name]: 'Alice', + [scoreField.name]: 11, + }, + }, + }); + await updateRecord(taskTable.id, taskRecords.records[0].id, { + record: { + fields: { + [taskTable.fields[0].name]: 'Task A', + [ownerLinkField.name]: [{ id: peopleRecords.records[0].id }], + }, + }, + }); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId, + name: 'v2 field parity copy', + withRecords: true, + }); + duplicateBaseId = dupResult.data.id; + + const duplicatedTables = await getTableList(duplicateBaseId).then((res) => res.data); + const duplicatedPeopleTable = duplicatedTables.find(({ name }) => name === peopleTable.name)!; + const duplicatedTaskTable = duplicatedTables.find(({ name }) => name === taskTable.name)!; + const duplicatedPeopleFields = (await getFields(duplicatedPeopleTable.id)).data; + const duplicatedTaskFields = (await getFields(duplicatedTaskTable.id)).data; + + const duplicatedScoreField = duplicatedPeopleFields.find( + ({ name }) => name === scoreField.name + )!; + const duplicatedOwnerLinkField = duplicatedTaskFields.find( + ({ name }) => name === ownerLinkField.name + )!; + const duplicatedOwnerScoreRollupField = duplicatedTaskFields.find( + ({ name }) => name === ownerScoreRollupField.name + ); + const duplicatedDoubledScoreField = duplicatedPeopleFields.find( + ({ name }) => name === doubledScoreField.name + ); + expect(duplicatedDoubledScoreField?.options).toMatchObject({ + expression: expect.stringContaining(duplicatedScoreField.id), + }); + + const duplicatedAiField = duplicatedPeopleFields.find(({ name }) => name === aiField.name); + expect(duplicatedAiField?.aiConfig).toEqual({ + ...aiField.aiConfig, + sourceFieldId: duplicatedPeopleFields.find(({ name }) => name === peoplePrimaryField.name)! + .id, + }); + + const duplicatedPeopleRecords = await getRecords(duplicatedPeopleTable.id); + const duplicatedTaskRecords = await getRecords(duplicatedTaskTable.id, { + fieldKeyType: FieldKeyType.Name, + }); + expect(duplicatedTaskRecords.records[0].fields[ownerLinkField.name]).toMatchObject([ + { id: duplicatedPeopleRecords.records[0].id, title: 'Alice' }, + ]); + expect(duplicatedTaskRecords.records[0].fields[ownerLookupField.name]).toEqual(['Alice']); + expect(duplicatedPeopleRecords.records[0].fields[ownerSymmetricField.name]).toMatchObject([ + { id: duplicatedTaskRecords.records[0].id }, + ]); + expect( + duplicatedTaskFields.find(({ name }) => name === ownerLookupField.name)?.isLookup + ).toBe(true); + expect(duplicatedOwnerScoreRollupField?.type).toBe(FieldType.Rollup); + expect(duplicatedOwnerScoreRollupField?.lookupOptions).toMatchObject({ + foreignTableId: duplicatedPeopleTable.id, + linkFieldId: duplicatedOwnerLinkField.id, + lookupFieldId: duplicatedScoreField.id, + }); + }); + + it('duplicates folders, dashboards, and plugins through v2', async () => { + const folderNode = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder 1', + }).then((res) => res.data); + const pluginTableNode = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'Plugin Table', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + await moveBaseNode(base.id, pluginTableNode.id, { parentId: folderNode.id }); + + const dashboard = (await createDashboard(base.id, { name: 'Dashboard 1' })).data; + await installPlugin(base.id, dashboard.id, { + name: 'dashboard plugin', + pluginId: 'plgchart', + }); + + const panel = (await createPluginPanel(pluginTableNode.resourceId, { name: 'panel1' })).data; + await installPluginPanel(pluginTableNode.resourceId, panel.id, { + name: 'panel plugin', + pluginId: 'plgchart', + }); + + const sheetView = ( + await installViewPlugin(pluginTableNode.resourceId, { + name: 'sheetView1', + pluginId: 'plgsheetform', + }) + ).data; + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId, + name: 'v2 extras parity copy', + }); + duplicateBaseId = dupResult.data.id; + + const duplicatedNodeTree = await getBaseNodeTree(duplicateBaseId).then((res) => res.data); + const duplicatedFolder = duplicatedNodeTree.nodes.find( + ({ resourceMeta, resourceType }) => + resourceMeta?.name === folderNode.resourceMeta?.name && + resourceType === BaseNodeResourceType.Folder + ); + const duplicatedPluginTableNode = duplicatedNodeTree.nodes.find( + ({ resourceMeta, resourceType }) => + resourceMeta?.name === pluginTableNode.resourceMeta?.name && + resourceType === BaseNodeResourceType.Table + ); + expect(duplicatedPluginTableNode?.parentId).toBe(duplicatedFolder?.id); + + const duplicatedDashboard = (await getDashboardList(duplicateBaseId)).data.find( + ({ name }) => name === dashboard.name + )!; + const duplicatedDashboardInfo = (await getDashboard(duplicateBaseId, duplicatedDashboard.id)) + .data; + expect(duplicatedDashboardInfo.layout).toHaveLength(1); + const duplicatedDashboardPlugin = ( + await getDashboardInstallPlugin( + duplicateBaseId, + duplicatedDashboard.id, + duplicatedDashboardInfo.layout![0].pluginInstallId + ) + ).data; + expect(duplicatedDashboardPlugin.name).toBe('dashboard plugin'); + + const duplicatedTable = (await getTableList(duplicateBaseId)).data.find( + ({ name }) => name === pluginTableNode.resourceMeta?.name + )!; + const duplicatedPanels = (await listPluginPanels(duplicatedTable.id)).data; + const duplicatedPanel = duplicatedPanels.find(({ name }) => name === panel.name)!; + const duplicatedPanelInfo = (await getPluginPanel(duplicatedTable.id, duplicatedPanel.id)) + .data; + expect(duplicatedPanelInfo.layout).toHaveLength(1); + const duplicatedPanelPlugin = ( + await getPluginPanelPlugin( + duplicatedTable.id, + duplicatedPanel.id, + duplicatedPanelInfo.layout![0].pluginInstallId + ) + ).data; + expect(duplicatedPanelPlugin.name).toBe('panel plugin'); + + const duplicatedPluginViews = (await getViewList(duplicatedTable.id)).data.filter( + ({ type }) => type === ViewType.Plugin + ); + expect(duplicatedPluginViews.find(({ name }) => name === sheetView.name)).toBeDefined(); + }); + + it('duplicates base to another space through v2', async () => { + const newSpace = (await createSpace({ name: 'v2 target space' })).data; + try { + await createTable(base.id, { name: 'Cross Space Table' }); + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: newSpace.id, + name: 'v2 cross space copy', + }); + const newSpaceDuplicateBaseId = dupResult.data.id; + + const baseResult = await getBaseList({ spaceId: newSpace.id }); + const tableResult = await getTableList(newSpaceDuplicateBaseId); + const records = await getRecords(tableResult.data[0].id); + expect(baseResult.data).toHaveLength(1); + expect(tableResult.data).toHaveLength(1); + expect(records.records).toHaveLength(0); + } finally { + await deleteSpace(newSpace.id); + } + }); + + it('duplicates partial nodes, disconnected links, lookup conversion, and parent folders through v2', async () => { + const folderNode = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Folder, + name: 'Orders Folder', + }).then((res) => res.data); + const ordersNode = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'Orders', + fields: [{ name: 'Order', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + const customersNode = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'Customers', + fields: [{ name: 'Customer', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + const productsNode = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'Products', + fields: [{ name: 'Product', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + await moveBaseNode(base.id, ordersNode.id, { parentId: folderNode.id }); + const productPrimaryField = (await getFields(productsNode.resourceId)).data.find( + ({ isPrimary }) => isPrimary + )!; + + const customerLinkField = ( + await createField(ordersNode.resourceId, { + name: 'customer', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: customersNode.resourceId, + }, + }) + ).data; + const productLinkField = ( + await createField(ordersNode.resourceId, { + name: 'product', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: productsNode.resourceId, + }, + }) + ).data; + const productLookupField = ( + await createField(ordersNode.resourceId, { + name: 'product lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: productsNode.resourceId, + linkFieldId: productLinkField.id, + lookupFieldId: productPrimaryField.id, + }, + }) + ).data; + + const orderRecords = await getRecords(ordersNode.resourceId); + const customerRecords = await getRecords(customersNode.resourceId); + const productRecords = await getRecords(productsNode.resourceId); + await updateRecord(ordersNode.resourceId, orderRecords.records[0].id, { + record: { + fields: { + [customerLinkField.name]: [{ id: customerRecords.records[0].id }], + [productLinkField.name]: [{ id: productRecords.records[0].id }], + }, + }, + }); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId, + name: 'v2 partial nodes copy', + withRecords: true, + nodes: [ordersNode.id, customersNode.id], + }); + duplicateBaseId = dupResult.data.id; + + const duplicatedNodeTree = await getBaseNodeTree(duplicateBaseId).then((res) => res.data); + const duplicatedFolders = duplicatedNodeTree.nodes.filter( + ({ resourceType }) => resourceType === BaseNodeResourceType.Folder + ); + const duplicatedTableNodes = duplicatedNodeTree.nodes.filter( + ({ resourceType }) => resourceType === BaseNodeResourceType.Table + ); + expect(duplicatedFolders).toHaveLength(1); + expect(duplicatedFolders[0].resourceMeta?.name).toBe(folderNode.resourceMeta?.name); + expect(duplicatedTableNodes.map(({ resourceMeta }) => resourceMeta?.name).sort()).toEqual( + ['Customers', 'Orders'].sort() + ); + expect( + duplicatedTableNodes.find(({ resourceMeta }) => resourceMeta?.name === 'Orders')?.parentId + ).toBe(duplicatedFolders[0].id); + + const duplicatedTables = await getTableList(duplicateBaseId).then((res) => res.data); + const duplicatedOrdersTable = duplicatedTables.find(({ name }) => name === 'Orders')!; + const duplicatedCustomersTable = duplicatedTables.find(({ name }) => name === 'Customers')!; + const duplicatedOrderFields = (await getFields(duplicatedOrdersTable.id)).data; + expect(duplicatedTables.map(({ name }) => name).sort()).toEqual(['Customers', 'Orders']); + expect(duplicatedOrderFields.find(({ name }) => name === customerLinkField.name)?.type).toBe( + FieldType.Link + ); + expect(duplicatedOrderFields.find(({ name }) => name === productLinkField.name)?.type).toBe( + FieldType.SingleLineText + ); + const duplicatedProductLookupField = duplicatedOrderFields.find( + ({ name }) => name === productLookupField.name + ); + expect(duplicatedProductLookupField?.type).toBe(FieldType.SingleLineText); + expect(duplicatedProductLookupField?.isLookup).toBeFalsy(); + + const duplicatedOrderRecords = await getRecords(duplicatedOrdersTable.id); + const duplicatedCustomerRecords = await getRecords(duplicatedCustomersTable.id); + expect(duplicatedOrderRecords.records[0].fields[customerLinkField.name]).toMatchObject([ + { id: duplicatedCustomerRecords.records[0].id }, + ]); + const duplicatedProductLinkValue = + duplicatedOrderRecords.records[0].fields[productLinkField.name]; + expect( + duplicatedProductLinkValue === null || + duplicatedProductLinkValue === undefined || + duplicatedProductLinkValue === '' + ).toBe(true); + }); + }); + describe('Duplicate cross space', () => { let newSpace: ICreateSpaceVo; beforeEach(async () => { diff --git a/apps/nestjs-backend/test/base-sql-executor.e2e-spec.ts b/apps/nestjs-backend/test/base-sql-executor.e2e-spec.ts index 83fb93f50c..fe22fb46c2 100644 --- a/apps/nestjs-backend/test/base-sql-executor.e2e-spec.ts +++ b/apps/nestjs-backend/test/base-sql-executor.e2e-spec.ts @@ -55,10 +55,10 @@ describe('BaseSqlExecutorService', () => { expect(result).toBeDefined(); }); - it('read only role can not execute sql to throw error', async () => { + it('read only role can not execute write sql to throw error', async () => { await expect( - baseSqlExecutorService['db']?.$queryRawUnsafe(`create table ${tableDbName} (id int)`) - ).rejects.toThrow('ERROR: permission denied for schema'); + baseSqlExecutorService.executeQuerySql(baseId, `create table ${tableDbName} (id int)`) + ).rejects.toThrow('An error occurred while checking table access'); }); it('read only role can read base', async () => { diff --git a/apps/nestjs-backend/test/base.e2e-spec.ts b/apps/nestjs-backend/test/base.e2e-spec.ts index 65d198828f..6045ad3e24 100644 --- a/apps/nestjs-backend/test/base.e2e-spec.ts +++ b/apps/nestjs-backend/test/base.e2e-spec.ts @@ -1,9 +1,10 @@ import type { INestApplication } from '@nestjs/common'; -import type { ILinkFieldOptions } from '@teable/core'; -import { FieldType, Relationship, Role } from '@teable/core'; +import type { IFieldRo, ILinkFieldOptions, ILinkFieldOptionsRo } from '@teable/core'; +import { DriverClient, FieldKeyType, FieldType, Relationship, Role } from '@teable/core'; import type { ICreateBaseVo, ICreateSpaceVo, + ITableFullVo, IUserMeVo, ListBaseInvitationLinkVo, UserCollaboratorItem, @@ -35,6 +36,7 @@ import { listBaseCollaboratorUserVoSchema, listBaseInvitationLink, MOVE_BASE, + moveBase, PrincipalType, UPDATE_BASE_COLLABORATE, UPDATE_BASE_INVITATION_LINK, @@ -49,9 +51,13 @@ import { getError } from './utils/get-error'; import { createBase, createField, + createRecords, createSpace, deleteSpace, + getFields, + getRecords, initApp, + permanentDeleteBase, permanentDeleteSpace, } from './utils/init-app'; @@ -774,4 +780,222 @@ describe('OpenAPI BaseController (e2e)', () => { expect(getRelationReference(base2Erd.edges).length).toEqual(2); }); }); + + // Contract: moveBase preserves values on every affected field: + // - Dependent lookup/rollup convert via the regular convertField path in + // deepest-first order, so each one's stored value is snapshotted by + // cellValue2String before the upstream Link is downgraded. + // - The Link itself converts via convertCrossSpaceLinkToText, which skips + // the destructive linkToOther steps so the symmetric partner on the + // other base survives and gets converted independently in its turn, + // preserving its own display values. + describe('moveBase cross-space value preservation', () => { + let sourceSpaceId: string; + let targetSpaceId: string; + let movingBaseId: string; + let peerBaseId: string; + + beforeAll(async () => { + sourceSpaceId = (await createSpace({ name: 'move-src' })).id; + targetSpaceId = (await createSpace({ name: 'move-dst' })).id; + }); + + afterAll(async () => { + await permanentDeleteSpace(sourceSpaceId); + await permanentDeleteSpace(targetSpaceId); + }); + + beforeEach(async () => { + movingBaseId = (await createBase({ spaceId: sourceSpaceId, name: 'moving' })).id; + peerBaseId = (await createBase({ spaceId: sourceSpaceId, name: 'peer' })).id; + }); + + afterEach(async () => { + await permanentDeleteBase(movingBaseId); + await permanentDeleteBase(peerBaseId); + }); + + it('preserves link, lookups, and the symmetric partner after move', async () => { + // Cross-base links require Postgres data sharding per base; the same code + // path is exercised — and the bug only surfaced — on PG. + if (globalThis.testConfig.driver !== DriverClient.Pg) { + return; + } + + const peerTitle = 'peer-title-1'; + + // Setup: peer base (target) gets a table with one record holding a known + // title; moving base gets a cross-base link to that table plus a lookup + // chain (link → lookupA → lookupB) so we exercise the multi-hop ordering. + // Both tables are created with empty records so that the single record + // we explicitly insert below is always at index 0 in getRecords. + const peerTable = (await createTable(peerBaseId, { name: 'peer', records: [] })).data; + const peerPrimary = peerTable.fields.find((f) => f.isPrimary)!; + const peerRecord = ( + await createRecords(peerTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [peerPrimary.id]: peerTitle } }], + }) + ).records[0]; + + const movingTable = (await createTable(movingBaseId, { name: 'moving', records: [] })).data; + const movingPrimary = movingTable.fields.find((f) => f.isPrimary)!; + + const linkField = await createField(movingTable.id, { + name: 'cross_base_link', + type: FieldType.Link, + options: { + baseId: peerBaseId, + relationship: Relationship.ManyOne, + foreignTableId: peerTable.id, + }, + }); + // ManyOne auto-creates a OneMany symmetric on the peer table. After the + // move it must survive (used to be cascade-deleted) and end up as text + // with the linked moving record's primary value. + const symmetricFieldId = (linkField.options as { symmetricFieldId?: string }) + .symmetricFieldId; + expect(symmetricFieldId).toBeTruthy(); + + const lookupA = await createField(movingTable.id, { + name: 'lookup_a', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: peerTable.id, + linkFieldId: linkField.id, + lookupFieldId: peerPrimary.id, + }, + }); + + const lookupB = await createField(movingTable.id, { + name: 'lookup_b', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: peerTable.id, + linkFieldId: linkField.id, + lookupFieldId: peerPrimary.id, + }, + }); + + await createRecords(movingTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [movingPrimary.id]: 'row-1', + [linkField.id]: { id: peerRecord.id }, + }, + }, + ], + }); + + // Sanity: before the move, lookups must have the title materialised so + // the post-move check is meaningful (otherwise null could mean "never + // computed" rather than "wiped by cascade"). + const beforeRecords = await getRecords(movingTable.id, { fieldKeyType: FieldKeyType.Id }); + const beforeRow = beforeRecords.records[0]; + expect(beforeRow.fields[lookupA.id]).toBe(peerTitle); + expect(beforeRow.fields[lookupB.id]).toBe(peerTitle); + + // Act: move the base — peer stays in sourceSpaceId, so the link becomes + // cross-space and must be downgraded along with both lookups. + const moveRes = await moveBase(movingBaseId, targetSpaceId); + expect(moveRes.status).toBe(200); + + const fieldsAfter = await getFields(movingTable.id); + const linkAfter = fieldsAfter.find((f) => f.id === linkField.id)!; + const lookupAAfter = fieldsAfter.find((f) => f.id === lookupA.id)!; + const lookupBAfter = fieldsAfter.find((f) => f.id === lookupB.id)!; + + expect(linkAfter.type).toBe(FieldType.SingleLineText); + expect(lookupAAfter.type).toBe(FieldType.SingleLineText); + expect(lookupAAfter.isLookup).toBeFalsy(); + expect(lookupBAfter.type).toBe(FieldType.SingleLineText); + expect(lookupBAfter.isLookup).toBeFalsy(); + + const afterRecords = await getRecords(movingTable.id, { fieldKeyType: FieldKeyType.Id }); + const afterRow = afterRecords.records[0]; + expect(afterRow.fields[linkField.id]).toBe(peerTitle); + expect(afterRow.fields[lookupA.id]).toBe(peerTitle); + expect(afterRow.fields[lookupB.id]).toBe(peerTitle); + + // Symmetric partner on the peer side: should still exist (NOT + // cascade-deleted) and be converted to text holding the linked moving + // record's primary value. + const peerFieldsAfter = await getFields(peerTable.id); + const symmetricAfter = peerFieldsAfter.find((f) => f.id === symmetricFieldId); + expect(symmetricAfter).toBeDefined(); + expect(symmetricAfter!.type).toBe(FieldType.SingleLineText); + const peerRecordsAfter = await getRecords(peerTable.id, { fieldKeyType: FieldKeyType.Id }); + const peerRowAfter = peerRecordsAfter.records[0]; + expect(peerRowAfter.fields[symmetricFieldId!]).toBe('row-1'); + }); + }); + + // Contract: assertNoNewCrossSpaceField rejects any new Link / conditional + // Lookup / conditional Rollup whose target table lives in another space. + // This is the gate enforcing the "no new cross-space refs" rule the PR is + // built around — without an e2e it can silently regress on any future + // createField refactor. + describe('cross-space field creation gate', () => { + let spaceA: string; + let spaceB: string; + let baseA: string; + let baseB: string; + let tableA: ITableFullVo; + let tableB: ITableFullVo; + + beforeAll(async () => { + spaceA = (await createSpace({ name: 'gate-a' })).id; + spaceB = (await createSpace({ name: 'gate-b' })).id; + baseA = (await createBase({ spaceId: spaceA, name: 'gate-base-a' })).id; + baseB = (await createBase({ spaceId: spaceB, name: 'gate-base-b' })).id; + tableA = (await createTable(baseA, { name: 'a' })).data; + tableB = (await createTable(baseB, { name: 'b' })).data; + }); + + afterAll(async () => { + await permanentDeleteSpace(spaceA); + await permanentDeleteSpace(spaceB); + }); + + it('rejects a new cross-space Link field with 400', async () => { + const fieldRo: IFieldRo = { + name: 'cross_space_link', + type: FieldType.Link, + options: { + baseId: baseB, + relationship: Relationship.ManyOne, + foreignTableId: tableB.id, + } as ILinkFieldOptionsRo, + }; + // createField helper returns {} when the response status matches the + // expected non-2xx — that's our "rejected" signal. A 201 here would + // make the helper throw, failing the test. + const result = await createField(tableA.id, fieldRo, 400); + expect(result).toEqual({}); + }); + + it('allows same-space cross-base Link creation (sanity for the gate)', async () => { + const baseA2 = (await createBase({ spaceId: spaceA, name: 'gate-base-a2' })).id; + const tableA2 = (await createTable(baseA2, { name: 'a2' })).data; + try { + const fieldRo: IFieldRo = { + name: 'same_space_link', + type: FieldType.Link, + options: { + baseId: baseA2, + relationship: Relationship.ManyOne, + foreignTableId: tableA2.id, + } as ILinkFieldOptionsRo, + }; + const created = await createField(tableA.id, fieldRo); + expect(created.type).toBe(FieldType.Link); + } finally { + await permanentDeleteBase(baseA2); + } + }); + }); }); diff --git a/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts b/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts new file mode 100644 index 0000000000..de51fd52c4 --- /dev/null +++ b/apps/nestjs-backend/test/byodb-space-storage-placement.e2e-spec.ts @@ -0,0 +1,1431 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import fs from 'fs'; +import path from 'path'; +import type { INestApplication } from '@nestjs/common'; +import type { ILinkFieldOptions } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship, SortFunc, StatisticsFunc } from '@teable/core'; +import { + analyzeFile as apiAnalyzeFile, + axios, + ensureUndoRedoWindowIdHeader, + exportBase, + getAggregation, + getFields, + getGroupPoints, + getImportStatus as apiGetImportStatus, + getRecordHistory, + getRowCount, + getSignature as apiGetSignature, + getTableList, + getTableActivatedIndex, + GroupPointType, + importBase, + importTableFromFile as apiImportTableFromFile, + type INotifyVo, + notify as apiNotify, + redo, + ResourceType, + SettingKey, + SUPPORTEDTYPE, + TableIndex, + toggleTableIndex, + undo, + updateSetting, + updateDbTableName, + updateRecordOrders, + UploadType, + uploadFile as apiUploadFile, + X_CANARY_HEADER, + type ITableFullVo, +} from '@teable/openapi'; +import Knex from 'knex'; +import type { Knex as KnexType } from 'knex'; +import type { ClsStore } from 'nestjs-cls'; +import { ClsService } from 'nestjs-cls'; +import type { IBaseConfig } from '../src/configs/base.config'; +import { baseConfig } from '../src/configs/base.config'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import StorageAdapter from '../src/features/attachments/plugins/adapter'; +import { + X_TEABLE_V2_HEADER, + X_TEABLE_V2_REASON_HEADER, +} from '../src/features/canary/interceptors/v2-indicator.interceptor'; +import { CsvImporter } from '../src/features/import/open-api/import.class'; +import { createAwaitWithEventWithResult } from './utils/event-promise'; +import { + createBase, + createField, + createRecords, + createSpace, + createTable, + deleteField, + deleteRecord, + deleteTable, + getRecords, + getTable, + initApp, + permanentDeleteBase, + permanentDeleteSpace, + permanentDeleteTable, + updateRecord, +} from './utils/init-app'; + +const databaseIdentity = (url?: string) => { + if (!url) { + return undefined; + } + + const parsed = new URL(url); + return `${parsed.protocol}//${parsed.host}${parsed.pathname}`; +}; + +const metaDatabaseUrl = + process.env.PRISMA_META_DATABASE_URL ?? + process.env.PRISMA_DATABASE_URL ?? + process.env.DATABASE_URL; +const byodbDataDatabaseUrl = process.env.BYODB_E2E_DATA_DATABASE_URL; +const isIndependentByodbDataDb = + databaseIdentity(metaDatabaseUrl) != null && + databaseIdentity(byodbDataDatabaseUrl) != null && + databaseIdentity(metaDatabaseUrl) !== databaseIdentity(byodbDataDatabaseUrl); +const describeByodbStorage = isIndependentByodbDataDb ? describe : describe.skip; + +const dataPlaneSystemTables = [ + '__teable_data_schema_migrations', + 'computed_update_outbox', + 'computed_update_outbox_seed', + 'computed_update_dead_letter', + 'computed_update_pause_scope', + 'record_history', + 'table_trash', + 'record_trash', + '__undo_log', +]; + +const metaPlaneTables = [ + 'space', + 'base', + 'base_node', + 'table_meta', + 'field', + 'view', + 'reference', + 'ops', + 'trash', + 'data_db_connection', + 'space_data_db_binding', +]; + +const quoteIdent = (value: string) => `"${value.replace(/"/g, '""')}"`; + +const parseDbTableName = (dbTableName: string) => { + const [schemaName, tableName] = dbTableName.split('.'); + + if (!schemaName || !tableName) { + throw new Error(`Invalid dbTableName: ${dbTableName}`); + } + + return { schemaName, tableName }; +}; + +const rawRows = async ( + client: KnexType, + query: string, + bindings: unknown[] = [] +): Promise => { + const result = await client.raw(query, bindings); + return result.rows as T[]; +}; + +const schemaExists = async (client: KnexType, schemaName: string) => { + const rows = await rawRows<{ exists: boolean }>( + client, + ` + SELECT EXISTS ( + SELECT 1 + FROM information_schema.schemata + WHERE schema_name = ? + ) AS exists + `, + [schemaName] + ); + + return Boolean(rows[0]?.exists); +}; + +const relationExists = async (client: KnexType, schemaName: string, tableName: string) => { + const rows = await rawRows<{ exists: boolean }>( + client, + ` + SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = ? AND table_name = ? + ) AS exists + `, + [schemaName, tableName] + ); + + return Boolean(rows[0]?.exists); +}; + +const tableExists = async (client: KnexType, dbTableName: string) => { + const { schemaName, tableName } = parseDbTableName(dbTableName); + return relationExists(client, schemaName, tableName); +}; + +const columnExists = async (client: KnexType, dbTableName: string, columnName: string) => { + const { schemaName, tableName } = parseDbTableName(dbTableName); + const rows = await rawRows<{ exists: boolean }>( + client, + ` + SELECT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = ? AND table_name = ? AND column_name = ? + ) AS exists + `, + [schemaName, tableName, columnName] + ); + + return Boolean(rows[0]?.exists); +}; + +const countRows = async ( + client: KnexType, + schemaName: string, + tableName: string, + whereSql?: string, + bindings: unknown[] = [] +) => { + if (!(await relationExists(client, schemaName, tableName))) { + return 0; + } + + const where = whereSql ? ` WHERE ${whereSql}` : ''; + const rows = await rawRows<{ count: number | string }>( + client, + `SELECT COUNT(*)::int AS count FROM ${quoteIdent(schemaName)}.${quoteIdent(tableName)}${where}`, + bindings + ); + + return Number(rows[0]?.count ?? 0); +}; + +const countDbTableRows = async (client: KnexType, dbTableName: string) => { + const { schemaName, tableName } = parseDbTableName(dbTableName); + return countRows(client, schemaName, tableName); +}; + +const dataDbMigrationVersions = async (client: KnexType, schemaName: string) => + rawRows<{ id: string }>( + client, + `SELECT ${quoteIdent('id')} FROM ${quoteIdent(schemaName)}.${quoteIdent( + '__teable_data_schema_migrations' + )} ORDER BY ${quoteIdent('id')}` + ); + +const dataDbConnectionVersionForSpace = async (client: KnexType, targetSpaceId: string) => + rawRows<{ schema_version: string | null }>( + client, + ` + SELECT c.${quoteIdent('schema_version')} + FROM ${quoteIdent('space_data_db_binding')} b + JOIN ${quoteIdent('data_db_connection')} c ON c.${quoteIdent('id')} = b.${quoteIdent( + 'data_db_connection_id' + )} + WHERE b.${quoteIdent('space_id')} = ? + `, + [targetSpaceId] + ); + +const constraintExists = async (client: KnexType, schemaName: string, constraintName: string) => { + const rows = await rawRows<{ exists: boolean }>( + client, + ` + SELECT EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_namespace n ON n.oid = c.connamespace + WHERE n.nspname = ? AND c.conname = ? + ) AS exists + `, + [schemaName, constraintName] + ); + + return Boolean(rows[0]?.exists); +}; + +const countDbTableRowsWhere = async ( + client: KnexType, + dbTableName: string, + whereSql: string, + bindings: unknown[] +) => { + const { schemaName, tableName } = parseDbTableName(dbTableName); + return countRows(client, schemaName, tableName, whereSql, bindings); +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const streamToBuffer = async (stream: NodeJS.ReadableStream) => { + const chunks: Buffer[] = []; + + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + + return Buffer.concat(chunks); +}; + +const waitForCount = async ( + getCount: () => Promise, + expectedCount: number, + maxRetries = 60 +) => { + for (let i = 0; i < maxRetries; i++) { + const count = await getCount(); + if (count === expectedCount) { + return count; + } + await sleep(100); + } + + return getCount(); +}; + +const waitForAtLeast = async ( + getCount: () => Promise, + expectedMinimum: number, + maxRetries = 60 +) => { + for (let i = 0; i < maxRetries; i++) { + const count = await getCount(); + if (count >= expectedMinimum) { + return count; + } + await sleep(100); + } + + return getCount(); +}; + +const waitForImportCompleted = async (tableId: string, expectedSuccessCount: number) => { + const maxRetries = 60; + + for (let i = 0; i < maxRetries; i++) { + const { data } = await apiGetImportStatus(tableId); + + if (data.status === 'completed' || data.status === 'failed') { + expect(data.status).toBe('completed'); + expect(data.successCount).toBe(expectedSuccessCount); + expect(data.failedCount ?? 0).toBe(0); + return; + } + + expect(data.status).not.toBe('not_found'); + await sleep(500); + } + + const { data } = await apiGetImportStatus(tableId); + throw new Error(`BYODB import timed out with latest status: ${data.status}`); +}; + +const createPgClient = (url: string) => + Knex({ + client: 'pg', + connection: url, + }); + +const importCsvData = `You_Xiang,Ming_Zi,order_count +ada@example.com,Ada,3 +bob@example.com,Bob,5 +`; + +const uploadImportCsv = async () => { + const tmpPath = path.resolve( + path.join(StorageAdapter.TEMPORARY_DIR, `byodb-import-${Date.now().toString(36)}.csv`) + ); + fs.writeFileSync(tmpPath, importCsvData); + + try { + const stats = fs.statSync(tmpPath); + const { token, requestHeaders } = ( + await apiGetSignature( + { + type: UploadType.Import, + contentLength: stats.size, + contentType: 'text/csv', + }, + undefined + ) + ).data; + + await apiUploadFile(token, fs.createReadStream(tmpPath), requestHeaders); + const { + data: { presignedUrl }, + } = await apiNotify(token, undefined, 'byodb-import.csv'); + + return presignedUrl; + } finally { + fs.unlinkSync(tmpPath); + } +}; + +const safeDropSchema = async (client: KnexType | undefined, schemaName: string | undefined) => { + if (!client || !schemaName) { + return; + } + + await client + .raw(`DROP SCHEMA IF EXISTS ${quoteIdent(schemaName)} CASCADE`) + .catch(() => undefined); +}; + +describeByodbStorage('BYODB space storage placement (e2e)', () => { + let app: INestApplication; + let metaDb: KnexType; + let dataDb: KnexType; + let baseConfigService: IBaseConfig; + let recordHistoryDisabled: boolean | undefined; + let spaceId: string | undefined; + let baseId: string | undefined; + const userId = globalThis.testConfig.userId; + + const internalSchema = `byodb_e2e_${Date.now().toString(36)}`; + + beforeAll(async () => { + metaDb = createPgClient(metaDatabaseUrl!); + dataDb = createPgClient(byodbDataDatabaseUrl!); + + const appCtx = await initApp(); + app = appCtx.app; + baseConfigService = app.get(baseConfig.KEY) as IBaseConfig; + recordHistoryDisabled = baseConfigService.recordHistoryDisabled; + baseConfigService.recordHistoryDisabled = false; + ensureUndoRedoWindowIdHeader(`win_byodb_storage_${Date.now()}`); + }, 60_000); + + afterAll(async () => { + if (baseId) { + await permanentDeleteBase(baseId).catch(() => undefined); + } + if (spaceId) { + await permanentDeleteSpace(spaceId).catch(() => undefined); + } + + await safeDropSchema(dataDb, baseId); + await safeDropSchema(metaDb, baseId); + await safeDropSchema(dataDb, internalSchema); + await safeDropSchema(metaDb, internalSchema); + + if (baseConfigService) { + baseConfigService.recordHistoryDisabled = recordHistoryDisabled ?? false; + } + + await dataDb?.destroy().catch(() => undefined); + await metaDb?.destroy().catch(() => undefined); + await app?.close(); + }, 60_000); + + const uploadExportedBase = async (targetBaseId: string) => { + const awaitExportWithPreview = createAwaitWithEventWithResult<{ + status?: 'success' | 'failed'; + previewUrl: string; + attachment?: { name: string; path: string }; + errorMessage?: string; + }>(app.get(EventEmitterService), Events.BASE_EXPORT_COMPLETE); + const { status, previewUrl, attachment, errorMessage } = await awaitExportWithPreview( + async () => { + await exportBase(targetBaseId); + } + ); + + if (status === 'failed') { + throw new Error(`Exported base is not available: ${errorMessage ?? 'unknown error'}`); + } + + return await app.get(ClsService).runWith>( + { + user: { + id: userId, + name: 'Test User', + email: 'test@example.com', + isAdmin: null, + }, + } as unknown as ClsStore, + async () => { + if (!attachment) { + throw new Error(`Missing exported base attachment payload for ${previewUrl}`); + } + + const storageAdapter = app.get(Symbol.for('ObjectStorage')); + const exportStream = await storageAdapter.downloadFile( + StorageAdapter.getBucket(UploadType.ExportBase), + attachment.path + ); + const exportBuffer = await streamToBuffer(exportStream); + const { token, requestHeaders } = ( + await apiGetSignature({ + type: UploadType.Import, + contentType: 'application/octet-stream', + contentLength: exportBuffer.length, + }) + ).data; + await apiUploadFile(token, exportBuffer, requestHeaders); + + return (await apiNotify(token, undefined, attachment.name)).data; + } + ); + }; + + it('keeps metadata in the meta DB and physical data artifacts in the bound data DB', async () => { + const space = await createSpace({ + name: 'BYODB placement e2e', + dataDb: { + mode: 'byodb', + url: byodbDataDatabaseUrl!, + targetMode: 'initialize-empty', + internalSchema, + }, + }); + spaceId = space.id; + + const base = await createBase({ spaceId: space.id, name: 'BYODB placement base' }); + baseId = base.id; + + const mainTable = await createTable(base.id, { + name: 'BYODB placement main', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Amount', type: FieldType.Number }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'opt_todo', name: 'Todo', color: 'blue' }, + { id: 'opt_done', name: 'Done', color: 'green' }, + ], + }, + }, + ], + records: [{ fields: {} }, { fields: {} }, { fields: {} }], + }); + expect(mainTable.records).toHaveLength(3); + const foreignTable = await createTable(base.id, { + name: 'BYODB placement foreign', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [], + }); + + const linkField = await createField(mainTable.id, { + name: 'Foreign link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + }, + }); + const linkOptions = linkField.options as ILinkFieldOptions; + const primaryFieldId = mainTable.fields.find((field) => field.isPrimary)?.id; + const amountFieldId = mainTable.fields.find((field) => field.name === 'Amount')?.id; + const statusFieldId = mainTable.fields.find((field) => field.name === 'Status')?.id; + const foreignPrimaryFieldId = foreignTable.fields.find((field) => field.isPrimary)?.id; + expect(primaryFieldId).toBeTruthy(); + expect(amountFieldId).toBeTruthy(); + expect(statusFieldId).toBeTruthy(); + expect(foreignPrimaryFieldId).toBeTruthy(); + const defaultViewId = mainTable.defaultViewId!; + + await assertMetaPlaneRows(space.id, base.id, mainTable, foreignTable, linkField.id); + await assertSchemaOperationsReady(base.id, mainTable.id, foreignTable.id); + await assertMetaPlaneTablesAreNotCopiedToDataDb(base.id); + await assertDataPlaneBaseline(internalSchema); + await assertPhysicalTables(mainTable, foreignTable, linkOptions.fkHostTableName); + await expect(countDbTableRows(dataDb, mainTable.dbTableName)).resolves.toBe(3); + await expect(countDbTableRows(metaDb, mainTable.dbTableName)).resolves.toBe(0); + const initialRecordList = await getRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + viewId: defaultViewId, + }); + expect(initialRecordList.records).toHaveLength(3); + expect(initialRecordList.records.map((record) => record.id)).toEqual( + mainTable.records.map((record) => record.id) + ); + await Promise.all( + mainTable.records.map((record, index) => + updateRecord(mainTable.id, record.id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [primaryFieldId!]: `Seed row ${index + 1}`, + [amountFieldId!]: (index + 1) * 10, + [statusFieldId!]: index === 2 ? 'Done' : 'Todo', + }, + }, + }) + ) + ); + const initialRowCount = await getRowCount(mainTable.id, { + viewId: defaultViewId, + }); + expect(initialRowCount.data.rowCount).toBe(3); + await expect(countDbTableRows(dataDb, mainTable.dbTableName)).resolves.toBe(3); + await expect(countDbTableRows(metaDb, mainTable.dbTableName)).resolves.toBe(0); + + const mainRecords = await createRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [primaryFieldId!]: 'Source row' } }], + }); + const foreignRecords = await createRecords(foreignTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [foreignPrimaryFieldId!]: 'Foreign row' } }], + }); + const recordId = mainRecords.records[0].id; + const foreignRecordId = foreignRecords.records[0].id; + + await expect( + countDbTableRowsWhere(dataDb, mainTable.dbTableName, `${quoteIdent('__id')} = ?`, [recordId]) + ).resolves.toBe(1); + await expect( + countDbTableRowsWhere(dataDb, foreignTable.dbTableName, `${quoteIdent('__id')} = ?`, [ + foreignRecordId, + ]) + ).resolves.toBe(1); + await expect( + countDbTableRowsWhere(metaDb, mainTable.dbTableName, `${quoteIdent('__id')} = ?`, [recordId]) + ).resolves.toBe(0); + await expect( + countDbTableRowsWhere(metaDb, foreignTable.dbTableName, `${quoteIdent('__id')} = ?`, [ + foreignRecordId, + ]) + ).resolves.toBe(0); + + const updatedRecord = await updateRecord(mainTable.id, recordId, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [primaryFieldId!]: 'Updated source row', + [amountFieldId!]: 7, + [statusFieldId!]: 'Done', + [linkField.id]: [{ id: foreignRecordId }], + }, + }, + }); + expect(updatedRecord.fields[primaryFieldId!]).toBe('Updated source row'); + expect(updatedRecord.fields[linkField.id]).toEqual([ + expect.objectContaining({ id: foreignRecordId }), + ]); + const rowCountAfterInsert = await getRowCount(mainTable.id, { + viewId: defaultViewId, + }); + expect(rowCountAfterInsert.data.rowCount).toBe(4); + const aggregation = ( + await getAggregation(mainTable.id, { + viewId: defaultViewId, + field: { + [StatisticsFunc.Sum]: [amountFieldId!], + [StatisticsFunc.Count]: [primaryFieldId!], + }, + groupBy: [{ fieldId: statusFieldId!, order: SortFunc.Asc }], + }) + ).data; + const amountAggregation = aggregation.aggregations?.find( + (item) => item.fieldId === amountFieldId + ); + const primaryAggregation = aggregation.aggregations?.find( + (item) => item.fieldId === primaryFieldId + ); + expect(Number(amountAggregation?.total?.value)).toBe(67); + expect(Number(primaryAggregation?.total?.value)).toBe(4); + expect(Object.keys(amountAggregation?.group ?? {})).toHaveLength(2); + + const groupPoints = ( + await getGroupPoints(mainTable.id, { + viewId: defaultViewId, + groupBy: [{ fieldId: statusFieldId!, order: SortFunc.Asc }], + }) + ).data; + expect(groupPoints?.filter((point) => point.type === GroupPointType.Header)).toHaveLength(2); + expect( + groupPoints?.reduce( + (sum, point) => (point.type === GroupPointType.Row ? sum + point.count : sum), + 0 + ) + ).toBe(4); + + await updateRecordOrders(mainTable.id, defaultViewId, { + anchorId: mainTable.records[0].id, + position: 'before', + recordIds: [recordId], + }); + const reorderedRecords = await getRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + viewId: defaultViewId, + }); + expect(reorderedRecords.records[0].id).toBe(recordId); + + await toggleTableIndex(base.id, mainTable.id, { type: TableIndex.search }); + expect((await getTableActivatedIndex(base.id, mainTable.id)).data).toContain(TableIndex.search); + + const extraField = await createField(mainTable.id, { + name: 'BYODB extra notes', + type: FieldType.LongText, + }); + const extraDbFieldName = extraField.dbFieldName!; + await expect(columnExists(dataDb, mainTable.dbTableName, extraDbFieldName)).resolves.toBe(true); + await expect(columnExists(metaDb, mainTable.dbTableName, extraDbFieldName)).resolves.toBe( + false + ); + await deleteField(mainTable.id, extraField.id); + await expect(columnExists(dataDb, mainTable.dbTableName, extraDbFieldName)).resolves.toBe( + false + ); + await expect(columnExists(metaDb, mainTable.dbTableName, extraDbFieldName)).resolves.toBe( + false + ); + + await expect(countDbTableRows(dataDb, mainTable.dbTableName)).resolves.toBe(4); + await expect(countDbTableRows(metaDb, mainTable.dbTableName)).resolves.toBe(0); + + await expect( + waitForAtLeast(() => countDbTableRows(dataDb, linkOptions.fkHostTableName), 1) + ).resolves.toBeGreaterThan(0); + await expect(countDbTableRows(metaDb, linkOptions.fkHostTableName)).resolves.toBe(0); + + await expect( + waitForAtLeast( + () => + countRows( + dataDb, + internalSchema, + 'record_history', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [mainTable.id, recordId] + ), + 1 + ) + ).resolves.toBeGreaterThan(0); + const { data: recordHistory } = await getRecordHistory(mainTable.id, recordId, {}); + expect(recordHistory.historyList.length).toBeGreaterThan(0); + await expect( + countRows( + metaDb, + 'public', + 'record_history', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [mainTable.id, recordId] + ) + ).resolves.toBe(0); + + await deleteRecord(mainTable.id, recordId); + await assertRecordTrashPlacement(mainTable.id, recordId, 1); + + const undoResult = await undo(mainTable.id); + expect(undoResult.data.status).toBe('fulfilled'); + await assertRecordTrashPlacement(mainTable.id, recordId, 0); + await expect( + countDbTableRowsWhere(dataDb, mainTable.dbTableName, `${quoteIdent('__id')} = ?`, [recordId]) + ).resolves.toBe(1); + + const redoResult = await redo(mainTable.id); + expect(redoResult.data.status).toBe('fulfilled'); + await assertRecordTrashPlacement(mainTable.id, recordId, 1); + await expect( + countDbTableRowsWhere(dataDb, mainTable.dbTableName, `${quoteIdent('__id')} = ?`, [recordId]) + ).resolves.toBe(0); + + await assertTableLifecycleRouting(base.id); + await assertImportedTableRouting(base.id); + await assertDotTeaBaseImportRouting(space.id); + await assertComputedSideEffectsStayOutOfMetaDb(base.id, mainTable.id, recordId); + }, 240_000); + + const assertMetaPlaneRows = async ( + targetSpaceId: string, + targetBaseId: string, + mainTable: ITableFullVo, + foreignTable: ITableFullVo, + linkFieldId: string + ) => { + await expect( + countRows(metaDb, 'public', 'space', `${quoteIdent('id')} = ?`, [targetSpaceId]) + ).resolves.toBe(1); + await expect( + countRows(dataDb, 'public', 'space', `${quoteIdent('id')} = ?`, [targetSpaceId]) + ).resolves.toBe(0); + + await expect( + countRows(metaDb, 'public', 'space_data_db_binding', `${quoteIdent('space_id')} = ?`, [ + targetSpaceId, + ]) + ).resolves.toBe(1); + await expect( + countRows(dataDb, 'public', 'space_data_db_binding', `${quoteIdent('space_id')} = ?`, [ + targetSpaceId, + ]) + ).resolves.toBe(0); + + await expect( + countRows(metaDb, 'public', 'base', `${quoteIdent('id')} = ?`, [targetBaseId]) + ).resolves.toBe(1); + await expect( + countRows(dataDb, 'public', 'base', `${quoteIdent('id')} = ?`, [targetBaseId]) + ).resolves.toBe(0); + await expect( + countRows(metaDb, 'public', 'table_meta', `${quoteIdent('base_id')} = ?`, [targetBaseId]) + ).resolves.toBeGreaterThanOrEqual(2); + await expect( + countRows(dataDb, 'public', 'table_meta', `${quoteIdent('base_id')} = ?`, [targetBaseId]) + ).resolves.toBe(0); + + await expect( + countRows(metaDb, 'public', 'field', `${quoteIdent('table_id')} IN (?, ?)`, [ + mainTable.id, + foreignTable.id, + ]) + ).resolves.toBeGreaterThanOrEqual(3); + await expect( + countRows(dataDb, 'public', 'field', `${quoteIdent('table_id')} IN (?, ?)`, [ + mainTable.id, + foreignTable.id, + ]) + ).resolves.toBe(0); + + const selectFields = await rawRows<{ options: string | Record | null }>( + metaDb, + ` + SELECT options + FROM public.field + WHERE table_id = ? AND type = ? + `, + [mainTable.id, FieldType.SingleSelect] + ); + expect(selectFields).toHaveLength(1); + const selectOptions = + typeof selectFields[0]?.options === 'string' + ? JSON.parse(selectFields[0].options) + : selectFields[0]?.options; + expect(selectOptions).toMatchObject({ + choices: [ + expect.objectContaining({ name: 'Todo' }), + expect.objectContaining({ name: 'Done' }), + ], + }); + + await expect( + countRows(metaDb, 'public', 'view', `${quoteIdent('table_id')} IN (?, ?)`, [ + mainTable.id, + foreignTable.id, + ]) + ).resolves.toBeGreaterThanOrEqual(2); + await expect( + countRows(dataDb, 'public', 'view', `${quoteIdent('table_id')} IN (?, ?)`, [ + mainTable.id, + foreignTable.id, + ]) + ).resolves.toBe(0); + + await expect( + countRows( + metaDb, + 'public', + 'reference', + `${quoteIdent('from_field_id')} = ? OR ${quoteIdent('to_field_id')} = ?`, + [linkFieldId, linkFieldId] + ) + ).resolves.toBeGreaterThan(0); + await expect( + countRows( + dataDb, + 'public', + 'reference', + `${quoteIdent('from_field_id')} = ? OR ${quoteIdent('to_field_id')} = ?`, + [linkFieldId, linkFieldId] + ) + ).resolves.toBe(0); + }; + + const assertSchemaOperationsReady = async ( + targetBaseId: string, + mainTableId: string, + foreignTableId: string + ) => { + const tableIdsPredicate = `${quoteIdent('base_id')} = ? AND ${quoteIdent( + 'table_id' + )} IN (?, ?)`; + const tableIdsParams = [targetBaseId, mainTableId, foreignTableId]; + + await expect( + countRows( + metaDb, + 'public', + 'schema_operation', + `${tableIdsPredicate} AND ${quoteIdent('type')} = ? AND ${quoteIdent('status')} = ?`, + [...tableIdsParams, 'table.create', 'ready'] + ) + ).resolves.toBe(2); + await expect( + countRows( + metaDb, + 'public', + 'schema_operation', + `${tableIdsPredicate} AND ${quoteIdent('type')} = ? AND ${quoteIdent('status')} = ?`, + [...tableIdsParams, 'table.update', 'ready'] + ) + ).resolves.toBe(2); + await expect( + countRows( + metaDb, + 'public', + 'schema_operation', + `${tableIdsPredicate} AND ${quoteIdent('status')} <> ?`, + [...tableIdsParams, 'ready'] + ) + ).resolves.toBe(0); + await expect( + countRows(dataDb, 'public', 'schema_operation', tableIdsPredicate, tableIdsParams) + ).resolves.toBe(0); + }; + + const assertMetaPlaneTablesAreNotCopiedToDataDb = async (targetBaseId: string) => { + for (const tableName of metaPlaneTables) { + await expect(relationExists(metaDb, 'public', tableName)).resolves.toBe(true); + await expect(relationExists(dataDb, 'public', tableName)).resolves.toBe(false); + await expect(relationExists(dataDb, internalSchema, tableName)).resolves.toBe(false); + await expect(relationExists(dataDb, targetBaseId, tableName)).resolves.toBe(false); + } + }; + + const assertDataPlaneBaseline = async (targetInternalSchema: string) => { + await expect(schemaExists(dataDb, targetInternalSchema)).resolves.toBe(true); + await expect(schemaExists(metaDb, targetInternalSchema)).resolves.toBe(false); + + for (const tableName of dataPlaneSystemTables) { + await expect(relationExists(dataDb, targetInternalSchema, tableName)).resolves.toBe(true); + await expect(relationExists(metaDb, targetInternalSchema, tableName)).resolves.toBe(false); + } + + await expect(dataDbMigrationVersions(dataDb, targetInternalSchema)).resolves.toEqual( + expect.arrayContaining([{ id: '20260421000000_init_data_db_baseline' }]) + ); + await expect( + constraintExists(dataDb, targetInternalSchema, 'computed_update_outbox_seed_task_id_fkey') + ).resolves.toBe(true); + }; + + it('initializes data DB migrations independently for multiple internal schemas', async () => { + const firstInternalSchema = `byodb_migration_a_${Date.now().toString(36)}`; + const secondInternalSchema = `byodb_migration_b_${Date.now().toString(36)}`; + let firstSpaceId: string | undefined; + let secondSpaceId: string | undefined; + + try { + const firstSpace = await createSpace({ + name: 'BYODB migration smoke A', + dataDb: { + mode: 'byodb', + url: byodbDataDatabaseUrl!, + targetMode: 'initialize-empty', + internalSchema: firstInternalSchema, + }, + }); + firstSpaceId = firstSpace.id; + const secondSpace = await createSpace({ + name: 'BYODB migration smoke B', + dataDb: { + mode: 'byodb', + url: byodbDataDatabaseUrl!, + targetMode: 'initialize-empty', + internalSchema: secondInternalSchema, + }, + }); + secondSpaceId = secondSpace.id; + + await assertDataPlaneBaseline(firstInternalSchema); + await assertDataPlaneBaseline(secondInternalSchema); + + const firstVersions = await dataDbMigrationVersions(dataDb, firstInternalSchema); + const secondVersions = await dataDbMigrationVersions(dataDb, secondInternalSchema); + await expect(dataDbConnectionVersionForSpace(metaDb, firstSpace.id)).resolves.toEqual([ + { schema_version: firstVersions.at(-1)?.id }, + ]); + await expect(dataDbConnectionVersionForSpace(metaDb, secondSpace.id)).resolves.toEqual([ + { schema_version: secondVersions.at(-1)?.id }, + ]); + } finally { + if (secondSpaceId) { + await permanentDeleteSpace(secondSpaceId).catch(() => undefined); + } + if (firstSpaceId) { + await permanentDeleteSpace(firstSpaceId).catch(() => undefined); + } + await safeDropSchema(dataDb, secondInternalSchema); + await safeDropSchema(dataDb, firstInternalSchema); + await safeDropSchema(metaDb, secondInternalSchema); + await safeDropSchema(metaDb, firstInternalSchema); + } + }); + + const assertPhysicalTables = async ( + mainTable: ITableFullVo, + foreignTable: ITableFullVo, + junctionTableName: string + ) => { + await expect(schemaExists(dataDb, baseId!)).resolves.toBe(true); + await expect(schemaExists(metaDb, baseId!)).resolves.toBe(false); + + for (const dbTableName of [ + mainTable.dbTableName, + foreignTable.dbTableName, + junctionTableName, + ]) { + await expect(tableExists(dataDb, dbTableName)).resolves.toBe(true); + await expect(tableExists(metaDb, dbTableName)).resolves.toBe(false); + await expect(countDbTableRows(dataDb, dbTableName)).resolves.toBeGreaterThanOrEqual(0); + await expect(countDbTableRows(metaDb, dbTableName)).resolves.toBe(0); + } + }; + + const assertRecordTrashPlacement = async ( + tableId: string, + recordId: string, + expectedCount: number + ) => { + await expect( + waitForCount( + () => + countRows( + dataDb, + internalSchema, + 'table_trash', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('resource_type')} = ?`, + [tableId, ResourceType.Record] + ), + expectedCount + ) + ).resolves.toBe(expectedCount); + await expect( + waitForCount( + () => + countRows( + dataDb, + internalSchema, + 'record_trash', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [tableId, recordId] + ), + expectedCount + ) + ).resolves.toBe(expectedCount); + + await expect( + countRows( + metaDb, + 'public', + 'table_trash', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('resource_type')} = ?`, + [tableId, ResourceType.Record] + ) + ).resolves.toBe(0); + await expect( + countRows( + metaDb, + 'public', + 'record_trash', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('record_id')} = ?`, + [tableId, recordId] + ) + ).resolves.toBe(0); + }; + + const assertTableLifecycleRouting = async (targetBaseId: string) => { + const lifecycleTable = await createTable(targetBaseId, { + name: 'BYODB lifecycle table', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [{ fields: { Name: 'Lifecycle row' } }], + }); + const oldDbTableName = lifecycleTable.dbTableName; + const renamedTableName = `byodb_lifecycle_${Date.now().toString(36)}`; + const renamedDbTableName = `${targetBaseId}.${renamedTableName}`; + + await expect(tableExists(dataDb, oldDbTableName)).resolves.toBe(true); + await expect(tableExists(metaDb, oldDbTableName)).resolves.toBe(false); + + await updateDbTableName(targetBaseId, lifecycleTable.id, { + dbTableName: renamedTableName, + }); + await expect(tableExists(dataDb, oldDbTableName)).resolves.toBe(false); + await expect(tableExists(dataDb, renamedDbTableName)).resolves.toBe(true); + await expect(tableExists(metaDb, renamedDbTableName)).resolves.toBe(false); + await expect(countDbTableRows(dataDb, renamedDbTableName)).resolves.toBe(1); + await expect(countDbTableRows(metaDb, renamedDbTableName)).resolves.toBe(0); + + const renamedRecords = await getRecords(lifecycleTable.id, { + fieldKeyType: FieldKeyType.Id, + viewId: lifecycleTable.defaultViewId, + }); + expect(renamedRecords.records).toHaveLength(1); + + await deleteTable(targetBaseId, lifecycleTable.id, 200); + await expect( + waitForAtLeast( + () => + countRows( + dataDb, + internalSchema, + 'table_trash', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('resource_type')} = ?`, + [lifecycleTable.id, ResourceType.Table] + ), + 1 + ) + ).resolves.toBeGreaterThan(0); + await expect( + countRows( + metaDb, + 'public', + 'table_trash', + `${quoteIdent('table_id')} = ? AND ${quoteIdent('resource_type')} = ?`, + [lifecycleTable.id, ResourceType.Table] + ) + ).resolves.toBe(0); + + await permanentDeleteTable(targetBaseId, lifecycleTable.id, 200); + await expect(tableExists(dataDb, renamedDbTableName)).resolves.toBe(false); + await expect(tableExists(metaDb, renamedDbTableName)).resolves.toBe(false); + await expect( + countRows(metaDb, 'public', 'table_meta', `${quoteIdent('id')} = ?`, [lifecycleTable.id]) + ).resolves.toBe(0); + }; + + const assertImportedTableRouting = async (targetBaseId: string) => { + const attachmentUrl = await uploadImportCsv(); + const { + data: { worksheets }, + } = await apiAnalyzeFile({ + attachmentUrl, + fileType: SUPPORTEDTYPE.CSV, + }); + const columns = worksheets[CsvImporter.DEFAULT_SHEETKEY].columns.map((column, index) => ({ + ...column, + sourceColumnIndex: index, + })); + + const importResult = await apiImportTableFromFile(targetBaseId, { + attachmentUrl, + fileType: SUPPORTEDTYPE.CSV, + worksheets: { + [CsvImporter.DEFAULT_SHEETKEY]: { + name: 'BYODB imported table', + columns, + useFirstRowAsHeader: true, + importData: true, + }, + }, + tz: 'Asia/Shanghai', + }); + const importedTable = importResult.data[0]; + expect(importedTable.fields.map((field) => ({ name: field.name, type: field.type }))).toEqual([ + { name: 'You_Xiang', type: FieldType.SingleLineText }, + { name: 'Ming_Zi', type: FieldType.SingleLineText }, + { name: 'order_count', type: FieldType.Number }, + ]); + + await waitForImportCompleted(importedTable.id, 2); + + const importedRecords = await getRecords(importedTable.id, { + fieldKeyType: FieldKeyType.Name, + viewId: importedTable.defaultViewId, + }); + expect(importedRecords.records).toHaveLength(2); + expect(importedRecords.records.map((record) => record.fields)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + ['You_Xiang']: 'ada@example.com', + ['Ming_Zi']: 'Ada', + order_count: 3, + }), + expect.objectContaining({ + ['You_Xiang']: 'bob@example.com', + ['Ming_Zi']: 'Bob', + order_count: 5, + }), + ]) + ); + + await expect( + countRows(metaDb, 'public', 'table_meta', `${quoteIdent('id')} = ?`, [importedTable.id]) + ).resolves.toBe(1); + await expect( + countRows(dataDb, 'public', 'table_meta', `${quoteIdent('id')} = ?`, [importedTable.id]) + ).resolves.toBe(0); + await expect( + countRows(metaDb, 'public', 'field', `${quoteIdent('table_id')} = ?`, [importedTable.id]) + ).resolves.toBe(3); + await expect( + countRows(dataDb, 'public', 'field', `${quoteIdent('table_id')} = ?`, [importedTable.id]) + ).resolves.toBe(0); + await expect( + countRows(metaDb, 'public', 'view', `${quoteIdent('table_id')} = ?`, [importedTable.id]) + ).resolves.toBeGreaterThanOrEqual(1); + await expect( + countRows(dataDb, 'public', 'view', `${quoteIdent('table_id')} = ?`, [importedTable.id]) + ).resolves.toBe(0); + + await expect(tableExists(dataDb, importedTable.dbTableName)).resolves.toBe(true); + await expect(tableExists(metaDb, importedTable.dbTableName)).resolves.toBe(false); + await expect(countDbTableRows(dataDb, importedTable.dbTableName)).resolves.toBe(2); + await expect(countDbTableRows(metaDb, importedTable.dbTableName)).resolves.toBe(0); + + await expect( + countRows( + metaDb, + 'public', + 'schema_operation', + `${quoteIdent('base_id')} = ? AND ${quoteIdent('table_id')} = ? AND ${quoteIdent( + 'type' + )} = ? AND ${quoteIdent('status')} = ?`, + [targetBaseId, importedTable.id, 'table.create', 'ready'] + ) + ).resolves.toBe(1); + await expect( + countRows( + metaDb, + 'public', + 'schema_operation', + `${quoteIdent('base_id')} = ? AND ${quoteIdent('table_id')} = ? AND ${quoteIdent( + 'status' + )} <> ?`, + [targetBaseId, importedTable.id, 'ready'] + ) + ).resolves.toBe(0); + await expect( + countRows(dataDb, 'public', 'schema_operation', `${quoteIdent('table_id')} = ?`, [ + importedTable.id, + ]) + ).resolves.toBe(0); + + await expect( + waitForAtLeast( + () => + countRows(dataDb, internalSchema, 'record_history', `${quoteIdent('table_id')} = ?`, [ + importedTable.id, + ]), + 1 + ) + ).resolves.toBeGreaterThan(0); + await expect( + countRows(metaDb, 'public', 'record_history', `${quoteIdent('table_id')} = ?`, [ + importedTable.id, + ]) + ).resolves.toBe(0); + }; + + const assertDotTeaBaseImportRouting = async (targetSpaceId: string) => { + let sourceBaseId: string | undefined; + let importedBaseId: string | undefined; + const previousEnableCanaryFeature = process.env.ENABLE_CANARY_FEATURE; + process.env.ENABLE_CANARY_FEATURE = 'true'; + + try { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: [targetSpaceId], + }, + }); + + const sourceBase = await createBase({ + spaceId: targetSpaceId, + name: 'BYODB dottea source', + }); + sourceBaseId = sourceBase.id; + + const foreignTable = await createTable(sourceBase.id, { + name: 'BYODB dottea foreign', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + records: [{ fields: { Title: 'Foreign row' } }], + }); + const foreignPrimaryFieldId = foreignTable.fields.find((field) => field.isPrimary)?.id; + expect(foreignPrimaryFieldId).toBeTruthy(); + + const hostTable = await createTable(sourceBase.id, { + name: 'BYODB dottea host', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [{ fields: { Name: 'Host row' } }], + }); + const hostRecordId = hostTable.records[0]?.id; + const foreignRecordId = foreignTable.records[0]?.id; + expect(hostRecordId).toBeTruthy(); + expect(foreignRecordId).toBeTruthy(); + + const linkField = await createField(hostTable.id, { + name: 'Foreign link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + }, + }); + await updateRecord(hostTable.id, hostRecordId!, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: [{ id: foreignRecordId! }], + }, + }, + }); + + const notify = await uploadExportedBase(sourceBase.id); + const previousCanaryHeader = axios.defaults.headers.common[X_CANARY_HEADER]; + axios.defaults.headers.common[X_CANARY_HEADER] = 'true'; + const importedResponse = await importBase({ + notify: notify as unknown as INotifyVo, + spaceId: targetSpaceId, + }).finally(() => { + if (previousCanaryHeader === undefined) { + delete axios.defaults.headers.common[X_CANARY_HEADER]; + } else { + axios.defaults.headers.common[X_CANARY_HEADER] = previousCanaryHeader; + } + }); + expect({ + useV2: importedResponse.headers[X_TEABLE_V2_HEADER], + reason: importedResponse.headers[X_TEABLE_V2_REASON_HEADER], + }).toEqual({ useV2: 'true', reason: expect.any(String) }); + const imported = importedResponse.data; + importedBaseId = imported.base.id; + + const importedTables = (await getTableList(importedBaseId)).data; + expect(importedTables.map((table) => table.name).sort()).toEqual( + ['BYODB dottea foreign', 'BYODB dottea host'].sort() + ); + + const importedHostMeta = importedTables.find((table) => table.name === hostTable.name)!; + const importedForeignMeta = importedTables.find((table) => table.name === foreignTable.name)!; + const importedHost = await getTable(importedBaseId, importedHostMeta.id, { + includeContent: true, + }); + const importedForeign = await getTable(importedBaseId, importedForeignMeta.id, { + includeContent: true, + }); + + await expect( + countRows(metaDb, 'public', 'base', `${quoteIdent('id')} = ?`, [importedBaseId]) + ).resolves.toBe(1); + await expect( + countRows(dataDb, 'public', 'base', `${quoteIdent('id')} = ?`, [importedBaseId]) + ).resolves.toBe(0); + await expect( + countRows(metaDb, 'public', 'table_meta', `${quoteIdent('base_id')} = ?`, [importedBaseId]) + ).resolves.toBe(2); + await expect( + countRows(dataDb, 'public', 'table_meta', `${quoteIdent('base_id')} = ?`, [importedBaseId]) + ).resolves.toBe(0); + await expect( + countRows(metaDb, 'public', 'field', `${quoteIdent('table_id')} IN (?, ?)`, [ + importedHostMeta.id, + importedForeignMeta.id, + ]) + ).resolves.toBeGreaterThanOrEqual(3); + await expect( + countRows(dataDb, 'public', 'field', `${quoteIdent('table_id')} IN (?, ?)`, [ + importedHostMeta.id, + importedForeignMeta.id, + ]) + ).resolves.toBe(0); + + await expect(tableExists(dataDb, importedHost.dbTableName)).resolves.toBe(true); + await expect(tableExists(dataDb, importedForeign.dbTableName)).resolves.toBe(true); + await expect(tableExists(metaDb, importedHost.dbTableName)).resolves.toBe(false); + await expect(tableExists(metaDb, importedForeign.dbTableName)).resolves.toBe(false); + await expect( + waitForAtLeast(() => countDbTableRows(dataDb, importedHost.dbTableName), 1) + ).resolves.toBe(1); + await expect( + waitForAtLeast(() => countDbTableRows(dataDb, importedForeign.dbTableName), 1) + ).resolves.toBe(1); + await expect(countDbTableRows(metaDb, importedHost.dbTableName)).resolves.toBe(0); + await expect(countDbTableRows(metaDb, importedForeign.dbTableName)).resolves.toBe(0); + + const importedLinkField = (await getFields(importedHostMeta.id)).data.find( + (field) => field.type === FieldType.Link + ); + expect(importedLinkField).toBeDefined(); + await expect( + countRows( + metaDb, + 'public', + 'reference', + `${quoteIdent('to_field_id')} = ? OR ${quoteIdent('from_field_id')} = ?`, + [importedLinkField!.id, importedLinkField!.id] + ) + ).resolves.toBeGreaterThan(0); + await expect( + countRows( + dataDb, + 'public', + 'reference', + `${quoteIdent('to_field_id')} = ? OR ${quoteIdent('from_field_id')} = ?`, + [importedLinkField!.id, importedLinkField!.id] + ) + ).resolves.toBe(0); + + const importedLinkOptions = importedLinkField!.options as ILinkFieldOptions; + await expect(tableExists(dataDb, importedLinkOptions.fkHostTableName)).resolves.toBe(true); + await expect(tableExists(metaDb, importedLinkOptions.fkHostTableName)).resolves.toBe(false); + await expect( + waitForAtLeast(() => countDbTableRows(dataDb, importedLinkOptions.fkHostTableName), 1) + ).resolves.toBeGreaterThan(0); + await expect(countDbTableRows(metaDb, importedLinkOptions.fkHostTableName)).resolves.toBe(0); + } finally { + if (previousEnableCanaryFeature === undefined) { + delete process.env.ENABLE_CANARY_FEATURE; + } else { + process.env.ENABLE_CANARY_FEATURE = previousEnableCanaryFeature; + } + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + spaceIds: [], + }, + }).catch(() => undefined); + if (importedBaseId) { + await permanentDeleteBase(importedBaseId).catch(() => undefined); + } + if (sourceBaseId) { + await permanentDeleteBase(sourceBaseId).catch(() => undefined); + } + } + }; + + const assertComputedSideEffectsStayOutOfMetaDb = async ( + targetBaseId: string, + tableId: string, + recordId: string + ) => { + await expect( + countRows(metaDb, 'public', 'computed_update_outbox', `${quoteIdent('base_id')} = ?`, [ + targetBaseId, + ]) + ).resolves.toBe(0); + await expect( + countRows(metaDb, 'public', 'computed_update_dead_letter', `${quoteIdent('base_id')} = ?`, [ + targetBaseId, + ]) + ).resolves.toBe(0); + await expect( + countRows(metaDb, 'public', 'computed_update_outbox_seed', `${quoteIdent('table_id')} = ?`, [ + tableId, + ]) + ).resolves.toBe(0); + await expect( + countRows(metaDb, 'public', 'computed_update_outbox_seed', `${quoteIdent('record_id')} = ?`, [ + recordId, + ]) + ).resolves.toBe(0); + }; +}); diff --git a/apps/nestjs-backend/test/data-helpers/caces/view-default-share-meta.ts b/apps/nestjs-backend/test/data-helpers/caces/view-default-share-meta.ts index 6fd4a2ce37..4827cda77f 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/view-default-share-meta.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/view-default-share-meta.ts @@ -7,11 +7,7 @@ export const VIEW_DEFAULT_SHARE_META: { }[] = [ { viewType: ViewType.Form, - defaultShareMeta: { - submit: { - allow: true, - }, - }, + defaultShareMeta: {}, }, { viewType: ViewType.Kanban, diff --git a/apps/nestjs-backend/test/dual-db-split.e2e-spec.ts b/apps/nestjs-backend/test/dual-db-split.e2e-spec.ts index fd8bbb4546..681d71c518 100644 --- a/apps/nestjs-backend/test/dual-db-split.e2e-spec.ts +++ b/apps/nestjs-backend/test/dual-db-split.e2e-spec.ts @@ -48,7 +48,7 @@ const metaDatabaseUrl = process.env.PRISMA_META_DATABASE_URL ?? process.env.PRISMA_DATABASE_URL ?? process.env.DATABASE_URL; -const dataDatabaseUrl = process.env.PRISMA_DATA_DATABASE_URL; +const dataDatabaseUrl = undefined; const isTrueSplitDb = databaseIdentity(metaDatabaseUrl) != null && databaseIdentity(dataDatabaseUrl) != null && @@ -323,7 +323,7 @@ describeSplitDb('Dual DB split smoke (e2e)', () => { ); expect(recordTrashItem).toBeDefined(); - await restoreTrash(recordTrashItem!.id); + await restoreTrash(recordTrashItem!.id, table.id); await expect( waitForCount(() => countTableTrash(dataPrisma, table.id, ResourceType.Record), 0) diff --git a/apps/nestjs-backend/test/field-duplicate.e2e-spec.ts b/apps/nestjs-backend/test/field-duplicate.e2e-spec.ts index 30673deb5f..df96780acb 100644 --- a/apps/nestjs-backend/test/field-duplicate.e2e-spec.ts +++ b/apps/nestjs-backend/test/field-duplicate.e2e-spec.ts @@ -16,6 +16,7 @@ import { Relationship, ViewType, } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; import type { ICreateBaseVo, ITableFullVo } from '@teable/openapi'; import { createField, @@ -1190,6 +1191,66 @@ describe('OpenAPI FieldOpenApiController for duplicate field (e2e)', () => { }); }); + describe('duplicate field falls back to end when source has no columnMeta entry', () => { + let table: ITableFullVo; + let view: { id: string }; + let sourceField: { id: string; name: string }; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'sparse_columnmeta_table', + fields: [ + { type: FieldType.SingleLineText, name: 'source' } as IFieldRo, + { type: FieldType.SingleLineText, name: 'after' } as IFieldRo, + ], + }); + view = (await createView(table.id, { name: 'sparse_view', type: ViewType.Grid })).data; + sourceField = table.fields.find((f) => f.name === 'source')!; + + // Simulate sparse columnMeta: drop the source field's entry directly in + // DB. Reproduces the legacy/migrated-view shape where columnMeta is not + // exhaustive — the path that used to yield `NaN` orderIndex and dump the + // duplicate to position 0. + const prisma = app.get(PrismaService); + const row = await prisma.view.findUniqueOrThrow({ + where: { id: view.id }, + select: { columnMeta: true }, + }); + const meta = row.columnMeta + ? (JSON.parse(row.columnMeta) as Record) + : {}; + delete meta[sourceField.id]; + await prisma.view.update({ + where: { id: view.id }, + data: { columnMeta: JSON.stringify(meta) }, + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('places duplicate at the rightmost end, never at position 0', async () => { + const dup = ( + await duplicateField(table.id, sourceField.id, { + name: `${sourceField.name}_copy`, + viewId: view.id, + }) + ).data; + + const afterView = (await getView(table.id, view.id)).data; + const dupOrder = afterView.columnMeta[dup.id]?.order; + const otherOrders = Object.entries(afterView.columnMeta) + .filter(([fid]) => fid !== dup.id) + .map(([, c]) => (c as { order: number }).order) + .filter((o) => typeof o === 'number' && Number.isFinite(o)); + + expect(typeof dupOrder).toBe('number'); + expect(Number.isFinite(dupOrder)).toBe(true); + expect(otherOrders.every((o) => o < dupOrder)).toBe(true); + }); + }); + describe('duplicate field with view new field order should next to the original field', () => { let table: ITableFullVo; let subTable: ITableFullVo; diff --git a/apps/nestjs-backend/test/large-table-operations.e2e-spec.ts b/apps/nestjs-backend/test/large-table-operations.e2e-spec.ts index 5ecd6e95a7..c1c939b2bf 100644 --- a/apps/nestjs-backend/test/large-table-operations.e2e-spec.ts +++ b/apps/nestjs-backend/test/large-table-operations.e2e-spec.ts @@ -312,23 +312,21 @@ async function ensureLargeTableContext(): Promise { throw new Error('Benchmark setup failed: no linked records available.'); } - await Promise.all( - seededRecords.records.map((record, index) => { - const value = [ - { id: linkTargets[index % linkTargets.length] }, - { id: linkTargets[(index + 1) % linkTargets.length] }, - ]; - - return updateRecord(mainTable.id, record.id, { - fieldKeyType: FieldKeyType.Id, - record: { - fields: { - [linkField.id]: value, - }, + for (const [index, record] of seededRecords.records.entries()) { + const value = [ + { id: linkTargets[index % linkTargets.length] }, + { id: linkTargets[(index + 1) % linkTargets.length] }, + ]; + + await updateRecord(mainTable.id, record.id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: value, }, - }); - }) - ); + }, + }); + } const sampleRecordId = seededRecords.records[0]?.id; @@ -373,7 +371,7 @@ describe('Large table operations timing (e2e)', () => { beforeAll(async () => { context = await ensureLargeTableContext(); - }); + }, 300_000); afterAll(async () => { if (context) { diff --git a/apps/nestjs-backend/test/oauth-server.e2e-spec.ts b/apps/nestjs-backend/test/oauth-server.e2e-spec.ts index fe9efb4878..dab71db9ab 100644 --- a/apps/nestjs-backend/test/oauth-server.e2e-spec.ts +++ b/apps/nestjs-backend/test/oauth-server.e2e-spec.ts @@ -232,6 +232,18 @@ describe('OpenAPI OAuthController (e2e)', () => { expect(error?.message).toBe('Invalid user'); }); + it('/api/oauth/decision (POST) - user mismatch', async () => { + // A logged-in user must not be able to approve another user's authorization transaction + const intruder = await createNewUserAxios({ + email: `oauth-decision-intruder+${Date.now()}@example.com`, + password: '12345678', + }); + const { transactionID } = await getAuthorize(axios, oauth); + const error = await getError(() => decision(intruder, transactionID!)); + expect(error?.status).toBe(400); + expect(error?.message).toBe('Invalid user'); + }); + it('/api/oauth/access_token (POST)', async () => { const { transactionID } = await getAuthorize(axios, oauth); @@ -276,6 +288,40 @@ describe('OpenAPI OAuthController (e2e)', () => { expect(userInfo.data.email).toEqual(testEmail); }); + it('/api/oauth/access_token (POST) - code issued for another client should fail', async () => { + // A code minted for `oauth` must not be redeemable with a different client's credentials + const { transactionID } = await getAuthorize(axios, oauth); + const res = await decision(axios, transactionID!); + const code = new URL(res.headers.location).searchParams.get('code'); + + const otherClient = await oauthCreate(oauthData); + const otherSecret = await generateOAuthSecret(otherClient.data.clientId); + + const error = await getError(() => + anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: otherClient.data.clientId, + client_secret: otherSecret.data.secret, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ) + ); + expect(error?.status).toBe(401); + expect(error?.message).toBe('Invalid client'); + + await oauthDelete(otherClient.data.clientId); + }); + it('/api/oauth/access_token (POST) - has decision', async () => { const { transactionID } = await getAuthorize(axios, oauth); await decision(axios, transactionID!); diff --git a/apps/nestjs-backend/test/oauth.e2e-spec.ts b/apps/nestjs-backend/test/oauth.e2e-spec.ts index e3f7c55563..0712ed21c1 100644 --- a/apps/nestjs-backend/test/oauth.e2e-spec.ts +++ b/apps/nestjs-backend/test/oauth.e2e-spec.ts @@ -2,13 +2,22 @@ import type { INestApplication } from '@nestjs/common'; import { PrismaService } from '@teable/db-main-prisma'; import type { OAuthCreateVo } from '@teable/openapi'; import { + OAUTH_DELETE, + OAUTH_GET, + OAUTH_SECRET_DELETE, + OAUTH_SECRET_GENERATE, + OAUTH_UPDATE, + REVOKE_ACCESS, deleteOAuthSecret, generateOAuthSecret, oauthCreate, oauthDelete, oauthGet, oauthUpdate, + urlBuilder, } from '@teable/openapi'; +import type { AxiosInstance } from 'axios'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; import { getError } from './utils/get-error'; import { initApp } from './utils/init-app'; @@ -152,4 +161,47 @@ describe('OpenAPI OAuthController (e2e)', () => { }); expect(authorizedRes).toHaveLength(0); }); + + // The management endpoints below carry no @Permissions decorator, so ownership + // is enforced entirely by OAuthService.validateOwnership. These cases lock that in: + // a logged-in user must not be able to operate on an OAuth app they do not own. + describe('ownership (cross-user)', () => { + let ownerApp: OAuthCreateVo; + let ownerSecretId: string; + let intruder: AxiosInstance; + + beforeAll(async () => { + intruder = await createNewUserAxios({ + email: `oauth-intruder+${Date.now()}@example.com`, + password: '12345678', + }); + }); + + beforeEach(async () => { + ownerApp = (await oauthCreate(oauthData)).data; + ownerSecretId = (await generateOAuthSecret(ownerApp.clientId)).data.id; + }); + + afterEach(async () => { + await oauthDelete(ownerApp.clientId).catch(() => undefined); + }); + + const forbiddenCases: Array<[string, (clientId: string) => Promise]> = [ + ['GET client', (clientId) => intruder.get(urlBuilder(OAUTH_GET, { clientId }))], + ['PUT client', (clientId) => intruder.put(urlBuilder(OAUTH_UPDATE, { clientId }), oauthData)], + ['DELETE client', (clientId) => intruder.delete(urlBuilder(OAUTH_DELETE, { clientId }))], + ['POST secret', (clientId) => intruder.post(urlBuilder(OAUTH_SECRET_GENERATE, { clientId }))], + [ + 'DELETE secret', + (clientId) => + intruder.delete(urlBuilder(OAUTH_SECRET_DELETE, { clientId, secretId: ownerSecretId })), + ], + ['POST revoke-access', (clientId) => intruder.post(urlBuilder(REVOKE_ACCESS, { clientId }))], + ]; + + it.each(forbiddenCases)('non-owner %s -> 403', async (_label, call) => { + const error = await getError(() => call(ownerApp.clientId)); + expect(error?.status).toBe(403); + }); + }); }); diff --git a/apps/nestjs-backend/test/record.e2e-spec.ts b/apps/nestjs-backend/test/record.e2e-spec.ts index 4da22ccf13..6ea79efcf1 100644 --- a/apps/nestjs-backend/test/record.e2e-spec.ts +++ b/apps/nestjs-backend/test/record.e2e-spec.ts @@ -211,11 +211,13 @@ describe('OpenAPI RecordController (e2e)', () => { typecast: true, }); + const beforeNowTypecast = Date.now(); const res3 = await updateRecord(table.id, table.records[0].id, { record: { fields: { [dateField.id]: 'now' } }, fieldKeyType: FieldKeyType.Id, typecast: true, }); + const afterNowTypecast = Date.now(); expect(res1.fields[singleUserField.id]).toMatchObject({ email: 'test@e2e.com', @@ -229,9 +231,9 @@ describe('OpenAPI RecordController (e2e)', () => { ]); expect(res3.fields[dateField.id]).toBeDefined(); - expect(new Date(res3.fields[dateField.id] as string).toISOString().slice(0, -7)).toEqual( - new Date().toISOString().slice(0, -7) - ); + const typecastTime = new Date(res3.fields[dateField.id] as string).getTime(); + expect(typecastTime).toBeGreaterThanOrEqual(beforeNowTypecast - 1000); + expect(typecastTime).toBeLessThanOrEqual(afterNowTypecast + 1000); }); it('should not auto create options when preventAutoNewOptions is true', async () => { diff --git a/apps/nestjs-backend/test/space-owner-limit.e2e-spec.ts b/apps/nestjs-backend/test/space-owner-limit.e2e-spec.ts deleted file mode 100644 index c6c924dede..0000000000 --- a/apps/nestjs-backend/test/space-owner-limit.e2e-spec.ts +++ /dev/null @@ -1,205 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -/* eslint-disable sonarjs/no-duplicate-string */ -import type { INestApplication } from '@nestjs/common'; -import { Role } from '@teable/core'; -import type { ICreateSpaceVo, IUserMeVo } from '@teable/openapi'; -import { - CREATE_SPACE, - EMAIL_SPACE_INVITATION, - PERMANENT_DELETE_SPACE, - UPDATE_SPACE_COLLABORATE, - USER_ME, - urlBuilder, - PrincipalType, -} from '@teable/openapi'; -import type { AxiosInstance } from 'axios'; -import { createNewUserAxios } from './utils/axios-instance/new-user'; -import { getError } from './utils/get-error'; -import { initApp } from './utils/init-app'; - -const MAX_OWNER_COUNT = 3; - -describe('Space owner limit (e2e)', () => { - let app: INestApplication; - let preMaxSpaceOwnerCount: string | undefined; - beforeAll(async () => { - preMaxSpaceOwnerCount = process.env.MAX_SPACE_OWNER_COUNT; - process.env.MAX_SPACE_OWNER_COUNT = String(MAX_OWNER_COUNT); - const appCtx = await initApp(); - app = appCtx.app; - }); - - afterAll(async () => { - process.env.MAX_SPACE_OWNER_COUNT = preMaxSpaceOwnerCount; - await app.close(); - }); - - describe('createSpace limit', () => { - let userRequest: AxiosInstance; - const spaceIds: string[] = []; - - beforeAll(async () => { - userRequest = await createNewUserAxios({ - email: 'owner-limit-create@example.com', - password: '12345678', - }); - }); - - afterAll(async () => { - for (const id of spaceIds) { - await userRequest.delete(urlBuilder(PERMANENT_DELETE_SPACE, { spaceId: id })); - } - }); - - it(`should allow creating up to ${MAX_OWNER_COUNT} spaces`, async () => { - for (let i = 0; i < MAX_OWNER_COUNT; i++) { - const res = await userRequest.post(CREATE_SPACE, { - name: `limit-test-space-${i}`, - }); - expect(res.status).toBe(201); - spaceIds.push(res.data.id); - } - }); - - it(`should reject creating the ${MAX_OWNER_COUNT + 1}th space`, async () => { - const error = await getError(() => - userRequest.post(CREATE_SPACE, { - name: 'one-too-many', - }) - ); - expect(error?.status).toBe(400); - expect(error?.message).toContain('Owned space limit exceeded'); - }); - - it('should allow creating a new space after deleting one (deleted spaces do not count)', async () => { - const deletedId = spaceIds.pop()!; - await userRequest.delete(urlBuilder(PERMANENT_DELETE_SPACE, { spaceId: deletedId })); - - const res = await userRequest.post(CREATE_SPACE, { - name: 'replacement-space', - }); - expect(res.status).toBe(201); - spaceIds.push(res.data.id); - }); - }); - - describe('invite as owner limit', () => { - let ownerRequest: AxiosInstance; - let inviterRequest: AxiosInstance; - const ownerSpaceIds: string[] = []; - let inviterSpaceId: string; - - beforeAll(async () => { - ownerRequest = await createNewUserAxios({ - email: 'owner-limit-invite-target@example.com', - password: '12345678', - }); - - for (let i = 0; i < MAX_OWNER_COUNT; i++) { - const res = await ownerRequest.post(CREATE_SPACE, { - name: `owned-space-${i}`, - }); - ownerSpaceIds.push(res.data.id); - } - - inviterRequest = await createNewUserAxios({ - email: 'owner-limit-inviter@example.com', - password: '12345678', - }); - const inviterSpace = await inviterRequest.post(CREATE_SPACE, { - name: 'inviter-space', - }); - inviterSpaceId = inviterSpace.data.id; - }); - - afterAll(async () => { - for (const id of ownerSpaceIds) { - await ownerRequest.delete(urlBuilder(PERMANENT_DELETE_SPACE, { spaceId: id })); - } - await inviterRequest.delete(urlBuilder(PERMANENT_DELETE_SPACE, { spaceId: inviterSpaceId })); - }); - - it('should reject inviting a user as owner when they already own max spaces', async () => { - const targetEmail = 'owner-limit-invite-target@example.com'; - const error = await getError(() => - inviterRequest.post(urlBuilder(EMAIL_SPACE_INVITATION, { spaceId: inviterSpaceId }), { - emails: [targetEmail], - role: Role.Owner, - }) - ); - expect(error?.status).toBe(400); - expect(error?.message).toContain('Owned space limit exceeded'); - }); - - it('should allow inviting the same user as a non-owner role', async () => { - const targetEmail = 'owner-limit-invite-target@example.com'; - const res = await inviterRequest.post( - urlBuilder(EMAIL_SPACE_INVITATION, { spaceId: inviterSpaceId }), - { - emails: [targetEmail], - role: Role.Editor, - } - ); - expect(res.status).toBe(201); - }); - }); - - describe('promote to owner limit', () => { - let ownerRequest: AxiosInstance; - let promoterRequest: AxiosInstance; - let targetUserId: string; - const ownerSpaceIds: string[] = []; - let promoterSpaceId: string; - - beforeAll(async () => { - ownerRequest = await createNewUserAxios({ - email: 'owner-limit-promote-target@example.com', - password: '12345678', - }); - const meRes = await ownerRequest.get(USER_ME); - targetUserId = meRes.data.id; - - for (let i = 0; i < MAX_OWNER_COUNT; i++) { - const res = await ownerRequest.post(CREATE_SPACE, { - name: `promote-owned-space-${i}`, - }); - ownerSpaceIds.push(res.data.id); - } - - promoterRequest = await createNewUserAxios({ - email: 'owner-limit-promoter@example.com', - password: '12345678', - }); - const promoterSpace = await promoterRequest.post(CREATE_SPACE, { - name: 'promoter-space', - }); - promoterSpaceId = promoterSpace.data.id; - - await promoterRequest.post(urlBuilder(EMAIL_SPACE_INVITATION, { spaceId: promoterSpaceId }), { - emails: ['owner-limit-promote-target@example.com'], - role: Role.Editor, - }); - }); - - afterAll(async () => { - for (const id of ownerSpaceIds) { - await ownerRequest.delete(urlBuilder(PERMANENT_DELETE_SPACE, { spaceId: id })); - } - await promoterRequest.delete( - urlBuilder(PERMANENT_DELETE_SPACE, { spaceId: promoterSpaceId }) - ); - }); - - it('should reject promoting a user to owner when they already own max spaces', async () => { - const error = await getError(() => - promoterRequest.patch(urlBuilder(UPDATE_SPACE_COLLABORATE, { spaceId: promoterSpaceId }), { - role: Role.Owner, - principalId: targetUserId, - principalType: PrincipalType.User, - }) - ); - expect(error?.status).toBe(400); - expect(error?.message).toContain('Owned space limit exceeded'); - }); - }); -}); diff --git a/apps/nestjs-backend/test/table-trash.e2e-spec.ts b/apps/nestjs-backend/test/table-trash.e2e-spec.ts index 8c934752b9..3724915483 100644 --- a/apps/nestjs-backend/test/table-trash.e2e-spec.ts +++ b/apps/nestjs-backend/test/table-trash.e2e-spec.ts @@ -366,7 +366,7 @@ describe('Trash (e2e)', () => { await awaitWithViewEvent(() => deleteView(tableId, deletedViewId)); const result = await waitForTableTrashItems(tableId); - const restored = await restoreTrash(result.data.trashItems[0].id); + const restored = await restoreTrash(result.data.trashItems[0].id, tableId); expect(restored.status).toEqual(201); }); @@ -378,7 +378,7 @@ describe('Trash (e2e)', () => { await awaitWithFieldDeleteSync(async () => deleteFields(tableId, deletedFieldIds)); const result = await waitForTableTrashItems(tableId); - const restored = await restoreTrash(result.data.trashItems[0].id); + const restored = await restoreTrash(result.data.trashItems[0].id, tableId); expect(restored.status).toEqual(201); }); @@ -395,7 +395,7 @@ describe('Trash (e2e)', () => { await awaitWithFieldDeleteSync(async () => deleteFields(tableId, [formulaField.id])); const result = await waitForTableTrashItems(tableId); - const restored = await restoreTrash(result.data.trashItems[0].id); + const restored = await restoreTrash(result.data.trashItems[0].id, tableId); expect(restored.status).toEqual(201); }); @@ -451,7 +451,7 @@ describe('Trash (e2e)', () => { })), }); - const restored = await restoreTrash(recordTrashItem!.id); + const restored = await restoreTrash(recordTrashItem!.id, tableId); expect(restored.status).toEqual(201); const recordsAfterRestore = await getRecords(tableId, { @@ -497,7 +497,7 @@ describe('Trash (e2e)', () => { ) as ITableTrashItemVo | undefined; expect(recordTrashItem).toBeTruthy(); - const restored = await restoreTrash(recordTrashItem!.id); + const restored = await restoreTrash(recordTrashItem!.id, tableId); expect(restored.status).toEqual(201); expect(legacyRestoreSpy).not.toHaveBeenCalled(); @@ -558,7 +558,7 @@ describe('Trash (e2e)', () => { expect(fieldTrashItem).toBeTruthy(); - const restored = await restoreTrash(fieldTrashItem!.id); + const restored = await restoreTrash(fieldTrashItem!.id, tableId); expect(restored.status).toEqual(201); const afterFields = await getFields(tableId); @@ -572,7 +572,7 @@ describe('Trash (e2e)', () => { await deleteRecords(tableId, deletedRecordIds); const result = await waitForTableTrashItems(tableId, 1); - const restored = await restoreTrash(result.data.trashItems[0].id); + const restored = await restoreTrash(result.data.trashItems[0].id, tableId); expect(restored.status).toEqual(201); }); diff --git a/apps/nestjs-backend/test/v2-schema-operation-runner.e2e-spec.ts b/apps/nestjs-backend/test/v2-schema-operation-runner.e2e-spec.ts index 611518093a..759f5caedb 100644 --- a/apps/nestjs-backend/test/v2-schema-operation-runner.e2e-spec.ts +++ b/apps/nestjs-backend/test/v2-schema-operation-runner.e2e-spec.ts @@ -1,13 +1,19 @@ /* eslint-disable sonarjs/no-duplicate-string */ import type { INestApplication } from '@nestjs/common'; -import { FieldType } from '@teable/core'; +import { FieldKeyType, FieldType } from '@teable/core'; import { DataPrismaService } from '@teable/db-data-prisma'; import { PrismaService, ProvisionState } from '@teable/db-main-prisma'; import { createTable as apiCreateTable } from '@teable/openapi'; import type { ITableFullVo } from '@teable/openapi'; import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; import type { IDbProvider } from '../src/db-provider/db.provider.interface'; -import { initApp, permanentDeleteTable } from './utils/init-app'; +import { + createField, + createRecords, + initApp, + permanentDeleteTable, + updateRecord, +} from './utils/init-app'; process.env.V2_SCHEMA_OPERATION_RUNNER_POLL_INTERVAL_MS = '50'; process.env.V2_SCHEMA_OPERATION_RUNNER_MAX_BATCH = '5'; @@ -29,6 +35,8 @@ const parseDbTableName = (dbTableName: string) => { return { schemaName, tableName }; }; +const quoteIdent = (identifier: string) => `"${identifier.replace(/"/g, '""')}"`; + const tableExists = async (client: IRawQueryClient, dbTableName: string) => { const { schemaName, tableName } = parseDbTableName(dbTableName); const rows = await client.$queryRawUnsafe<{ exists: boolean }[]>( @@ -82,6 +90,40 @@ describeV2('V2 schema operation runner recovery (e2e)', () => { return rows.map((row) => row.name); }; + const createFailingUpdateTrigger = async (dbTableName: string, suffix: string) => { + const { schemaName, tableName } = parseDbTableName(dbTableName); + const functionName = `fail_record_update_${suffix}`; + const triggerName = `fail_record_update_${suffix}`; + const qualifiedFunction = `${quoteIdent(schemaName)}.${quoteIdent(functionName)}`; + const qualifiedTable = `${quoteIdent(schemaName)}.${quoteIdent(tableName)}`; + + await dataPrisma.$executeRawUnsafe(` + CREATE OR REPLACE FUNCTION ${qualifiedFunction}() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + RAISE EXCEPTION 'e2e simulated data write failure after metadata update'; + END; + $$; + `); + await dataPrisma.$executeRawUnsafe(` + CREATE TRIGGER ${quoteIdent(triggerName)} + BEFORE UPDATE ON ${qualifiedTable} + FOR EACH ROW + EXECUTE FUNCTION ${qualifiedFunction}(); + `); + + return async () => { + await dataPrisma.$executeRawUnsafe(` + DROP TRIGGER IF EXISTS ${quoteIdent(triggerName)} ON ${qualifiedTable}; + `); + await dataPrisma.$executeRawUnsafe(` + DROP FUNCTION IF EXISTS ${qualifiedFunction}(); + `); + }; + }; + const waitForRecoveredTable = async (table: ITableFullVo, timeoutMs = 8_000) => { const startedAt = Date.now(); let lastStatus: unknown; @@ -173,4 +215,124 @@ describeV2('V2 schema operation runner recovery (e2e)', () => { primaryField!.dbFieldName ); }); + + it('keeps a table ready when a typecast record update metadata change succeeds but data write fails', async () => { + const createRes = await apiCreateTable(baseId, { + name: 'Record update data failure availability', + fields: [ + { name: 'Name', type: FieldType.SingleLineText, isPrimary: true }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [{ name: 'Open', color: 'blue' }], + }, + }, + ], + records: [], + }); + expect(createRes.status).toBe(201); + expect(createRes.headers['x-teable-v2']).toBe('true'); + + const table = createRes.data; + createdTables.push(table); + const statusField = table.fields.find((field) => field.name === 'Status'); + expect(statusField?.id).toBeTruthy(); + const { records } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: { Name: 'Task 1', Status: 'Open' } }], + }); + const recordId = records[0]?.id; + expect(recordId).toBeTruthy(); + + const cleanupTrigger = await createFailingUpdateTrigger(table.dbTableName, table.id); + try { + await updateRecord( + table.id, + recordId!, + { + record: { + fields: { + [statusField!.id]: 'Blocked', + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }, + 500 + ); + } finally { + await cleanupTrigger(); + } + + const [tableMeta, operation] = await Promise.all([ + metaPrisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { provisionState: true }, + }), + metaPrisma.schemaOperation.findFirst({ + where: { tableId: table.id, type: 'table.update' }, + orderBy: { createdTime: 'desc' }, + }), + ]); + + expect(tableMeta.provisionState).toBe(ProvisionState.ready); + expect(operation?.phase).toBe('error'); + expect(['error', 'dead']).toContain(operation?.status); + await expect(tableExists(dataPrisma, table.dbTableName)).resolves.toBe(true); + }); + + it('keeps a table ready when computed field backfill fails during a schema update', async () => { + const createRes = await apiCreateTable(baseId, { + name: 'Computed backfill data failure availability', + fields: [ + { name: 'Name', type: FieldType.SingleLineText, isPrimary: true }, + { name: 'Amount', type: FieldType.Number }, + ], + records: [], + }); + expect(createRes.status).toBe(201); + expect(createRes.headers['x-teable-v2']).toBe('true'); + + const table = createRes.data; + createdTables.push(table); + const amountField = table.fields.find((field) => field.name === 'Amount'); + expect(amountField?.id).toBeTruthy(); + + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: { Name: 'Task 1', Amount: 2 } }], + }); + + const cleanupTrigger = await createFailingUpdateTrigger(table.dbTableName, table.id); + try { + await createField( + table.id, + { + name: 'Computed Amount', + type: FieldType.Formula, + options: { expression: `{${amountField!.id}} * 2` }, + }, + 500 + ); + } finally { + await cleanupTrigger(); + } + + const [tableMeta, operation] = await Promise.all([ + metaPrisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { provisionState: true }, + }), + metaPrisma.schemaOperation.findFirst({ + where: { tableId: table.id, type: 'table.update' }, + orderBy: { createdTime: 'desc' }, + }), + ]); + + expect(tableMeta.provisionState).toBe(ProvisionState.ready); + expect(operation?.phase).toBe('error'); + expect(['error', 'dead']).toContain(operation?.status); + await expect(tableExists(dataPrisma, table.dbTableName)).resolves.toBe(true); + }); }); diff --git a/apps/nestjs-backend/test/view.e2e-spec.ts b/apps/nestjs-backend/test/view.e2e-spec.ts index 77e9dbf42b..7d39d83e89 100644 --- a/apps/nestjs-backend/test/view.e2e-spec.ts +++ b/apps/nestjs-backend/test/view.e2e-spec.ts @@ -463,6 +463,12 @@ describe('OpenAPI ViewController (e2e)', () => { expect(shareMeta).toEqual(viewShareDefault.defaultShareMeta); } ); + + it('stores submit.requireLogin on form views', async () => { + await updateViewShareMeta(tableId, formViewId, { submit: { requireLogin: true } }); + const view = await getView(tableId, formViewId); + expect(view.shareMeta?.submit?.requireLogin).toBe(true); + }); }); describe('filter by view ', () => { diff --git a/apps/nextjs-app/src/components/Metrics.tsx b/apps/nextjs-app/src/components/Metrics.tsx index 3e57391206..2d5a456189 100644 --- a/apps/nextjs-app/src/components/Metrics.tsx +++ b/apps/nextjs-app/src/components/Metrics.tsx @@ -1,4 +1,14 @@ import Script from 'next/script'; +import { useEffect, useState } from 'react'; + +const GOOGLE_LINKER_DOMAINS = ['teable.ai', 'app.teable.ai']; + +declare global { + interface Window { + gtag?: (command: string, targetId: string | Date, config?: Record) => void; + dataLayer?: unknown[]; + } +} export const MicrosoftClarity = ({ clarityId, @@ -77,16 +87,60 @@ export const Umami = ({ export const GoogleAnalytics = ({ gaId, + googleAdsId, + marketingGaId, user, }: { gaId?: string; + googleAdsId?: string; + marketingGaId?: string; user?: { id?: string; name?: string; email?: string; }; }) => { - if (!gaId) { + const scriptId = gaId ?? googleAdsId ?? marketingGaId; + const userId = user?.id; + const userEmail = user?.email; + const [isGtagReady, setIsGtagReady] = useState(false); + + useEffect(() => { + if (!isGtagReady || !window.gtag) { + return; + } + + const linker = { domains: GOOGLE_LINKER_DOMAINS }; + + if (gaId) { + window.gtag( + 'config', + gaId, + userId ? { user_id: userId, custom_map: { custom_dimension_1: 'user_email' } } : undefined + ); + } + + if (googleAdsId) { + window.gtag('config', googleAdsId, { linker }); + } + + if (marketingGaId) { + window.gtag('config', marketingGaId, { linker }); + } + }, [gaId, googleAdsId, isGtagReady, marketingGaId, userId]); + + useEffect(() => { + if (!isGtagReady || !window.gtag || !gaId || !userEmail) { + return; + } + + window.gtag('event', 'login', { + send_to: gaId, + custom_dimension_1: userEmail, + }); + }, [gaId, isGtagReady, userEmail]); + + if (!scriptId) { return null; } @@ -95,18 +149,17 @@ export const GoogleAnalytics = ({