chore(db): add dormant org search recovery runbook and scripts

Organization indexed search is dormant now that live search serves those
queries, but an organization search index can still hold most of the
knowledge search rows and all of the Tin keyword projection, inflating the
shared vector index for every workspace search. This adds an ordered,
resumable runbook and operator scripts to reclaim that space. Every script
only reads unless --execute is passed.

- disable-tin-projection: drops the three Tin sync triggers and truncates
  embedding_keyword_tin in one transaction, with short lock timeouts retried
  within a budget. The shared document ACL fan-out is left intact and becomes
  a probe of an empty table.
- restore-tin-projection: reinstalls those triggers from the 0019 and 0024
  migration functions themselves, with an optional 0019 backfill.
- delete-search-index-documents: deletes one search index's connector
  documents and chunks in cursor pages, mirroring the app's connector cleanup
  worker. It refuses unless the base is a search index and every live
  connector is paused or disabled with no sync lease, re-checked before every
  page. Storage objects are queued through the app's own storage cleanup
  outbox in the deleting transaction, with a backlog ceiling. The knowledge
  base and connectors are kept; their listing cursors are reset so a resumed
  connector lists everything again. Deletes fire no row triggers and write no
  projector marks, since projection rows go by foreign-key cascade.
- maintenance: health report, REINDEX INDEX CONCURRENTLY (HNSW first, before
  vacuum) and one-table-at-a-time VACUUM (VERBOSE, ANALYZE).

Unit tests cover the guards, paging and resume, retries, backpressure and
dry runs. A local-only PostgreSQL integration test exercises the real
scripts end to end and is not registered in CI. No migration or schema
changes.

Author: waleedlatif1

apps/sim/scripts/dormant-org-search/README.md

@@ -0,0 +1,50 @@
+import { resolveMigrationDatabaseUrl } from '@sim/db/script-migrations/database-url'
+import postgres, { type Sql } from 'postgres'
+
+/**
+ * Whether a run may write. Every script in this directory reads and reports unless the operator
+ * passes `--execute`; `--dry-run` is accepted for explicitness, and passing both is refused rather
+ * than resolved in either direction.
+ */
+export function resolveExecuteFlag(flags: { execute?: boolean; 'dry-run'?: boolean }): boolean {
+  if (flags.execute && flags['dry-run']) {
+    throw new Error('Pass either --execute or --dry-run, not both')
+  }
+  return flags.execute === true
+}
+
+/** Parses a positive integer flag, refusing anything a typo could turn into an unbounded run. */
+export function parsePositiveInteger(name: string, value: string | undefined, fallback: number) {
+  if (value === undefined) return fallback
+  const parsed = Number(value)
+  if (!Number.isSafeInteger(parsed) || parsed < 1) {
+    throw new Error(`--${name} must be a positive integer, got ${value}`)
+  }
+  return parsed
+}
+
+/** Parses a non-negative integer flag, such as a pause in milliseconds. */
+export function parseNonNegativeInteger(name: string, value: string | undefined, fallback: number) {
+  if (value === undefined) return fallback
+  const parsed = Number(value)
+  if (!Number.isSafeInteger(parsed) || parsed < 0) {
+    throw new Error(`--${name} must be a non-negative integer, got ${value}`)
+  }
+  return parsed
+}
+
+/**
+ * One dedicated connection on the migrations role (`MIGRATION_DATABASE_URL`, else `DATABASE_URL`),
+ * the role that owns the knowledge tables and can therefore drop their triggers, truncate, reindex
+ * and vacuum them. One connection keeps session settings on the connection that uses them.
+ */
+export function connectMigrationRole(): Sql {
+  const url = resolveMigrationDatabaseUrl()
+  if (!url) throw new Error('MIGRATION_DATABASE_URL or DATABASE_URL is required')
+  return postgres(url, {
+    max: 1,
+    max_lifetime: null,
+    onnotice: () => undefined,
+    connection: { application_name: 'sim-dormant-org-search-ops' },
+  })
+}

apps/sim/scripts/dormant-org-search/cli.ts

@@ -0,0 +1,60 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { parseDeleteSearchIndexDocumentsArgs } from '@/scripts/dormant-org-search/delete-search-index-documents'
+
+describe('parseDeleteSearchIndexDocumentsArgs', () => {
+  it('defaults to a dry run with the documented bounds', () => {
+    expect(parseDeleteSearchIndexDocumentsArgs(['--knowledge-base-id=kb-1'])).toEqual({
+      knowledgeBaseId: 'kb-1',
+      execute: false,
+      pageSize: 200,
+      chunkBatchSize: 1_000,
+      pauseMs: 250,
+      maxPages: undefined,
+      afterId: '',
+      storageCleanupCeiling: 2_000,
+      resetConnectors: true,
+      lockTimeoutMs: 5_000,
+      statementTimeoutMs: 60_000,
+    })
+  })
+
+  it('writes only with --execute', () => {
+    expect(
+      parseDeleteSearchIndexDocumentsArgs(['--knowledge-base-id=kb-1', '--execute']).execute
+    ).toBe(true)
+    expect(
+      parseDeleteSearchIndexDocumentsArgs(['--knowledge-base-id=kb-1', '--dry-run']).execute
+    ).toBe(false)
+    expect(() =>
+      parseDeleteSearchIndexDocumentsArgs(['--knowledge-base-id=kb-1', '--execute', '--dry-run'])
+    ).toThrow('not both')
+  })
+
+  it('parses resume and bound flags', () => {
+    expect(
+      parseDeleteSearchIndexDocumentsArgs([
+        '--knowledge-base-id=kb-1',
+        '--max-pages=5',
+        '--after-id=doc-9',
+        '--pause-ms=0',
+        '--no-connector-reset',
+      ])
+    ).toMatchObject({ maxPages: 5, afterId: 'doc-9', pauseMs: 0, resetConnectors: false })
+  })
+
+  it('refuses a missing base, unknown flags and non-positive bounds', () => {
+    expect(() => parseDeleteSearchIndexDocumentsArgs([])).toThrow('--knowledge-base-id')
+    expect(() =>
+      parseDeleteSearchIndexDocumentsArgs(['--knowledge-base-id=kb-1', '--force'])
+    ).toThrow()
+    expect(() =>
+      parseDeleteSearchIndexDocumentsArgs(['--knowledge-base-id=kb-1', '--page-size=0'])
+    ).toThrow('positive integer')
+    expect(() =>
+      parseDeleteSearchIndexDocumentsArgs(['--knowledge-base-id=kb-1', '--max-pages=1.5'])
+    ).toThrow('positive integer')
+  })
+})

apps/sim/scripts/dormant-org-search/delete-search-index-documents.test.ts

@@ -0,0 +1,145 @@
+#!/usr/bin/env bun
+
+/**
+ * Deletes every connector-owned document and chunk of one organization search index, keeping the
+ * knowledge base row and its (paused) connectors, then clears the connectors' listing cursors so a
+ * resumed connector lists its source from scratch.
+ *
+ * Usage:
+ *   DATABASE_URL=<migrations-role-dsn> bun apps/sim/scripts/dormant-org-search/delete-search-index-documents.ts \
+ *     --knowledge-base-id=<knowledge-base-id>                        # dry run: guard + first pages
+ *   DATABASE_URL=<migrations-role-dsn> bun apps/sim/scripts/dormant-org-search/delete-search-index-documents.ts \
+ *     --knowledge-base-id=<knowledge-base-id> --execute [--max-pages=50] [--after-id=<document-id>]
+ *
+ * Options:
+ *   --page-size=200                 documents per page
+ *   --chunk-batch-size=1000         chunks per delete transaction
+ *   --pause-ms=250                  pause after every committed transaction
+ *   --max-pages=N                   stop after N pages (dry run default 3; unbounded when executing)
+ *   --after-id=<document-id>        resume after this document id
+ *   --storage-cleanup-ceiling=2000  pending storage cleanup events before the run waits
+ *   --lock-timeout-ms=5000          per transaction
+ *   --statement-timeout-ms=60000    per statement
+ *   --no-connector-reset            keep the connectors' listing cursors
+ *
+ * Exit codes: 0 finished or stopped at --max-pages (resume with the logged --after-id), 1 failed,
+ * 2 refused by a precondition (checked before every page; nothing after the refusal is written).
+ */
+
+import { parseArgs } from 'node:util'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateShortId } from '@sim/utils/id'
+import {
+  parseNonNegativeInteger,
+  parsePositiveInteger,
+  resolveExecuteFlag,
+} from '@/scripts/dormant-org-search/cli'
+import {
+  DEFAULT_CHUNK_BATCH_SIZE,
+  DEFAULT_DOCUMENT_PAGE_SIZE,
+  DEFAULT_PAUSE_MS,
+  DEFAULT_STORAGE_CLEANUP_CEILING,
+  deleteSearchIndexDocuments,
+  type SearchIndexDeletionOptions,
+  SearchIndexDeletionRefused,
+} from '@/scripts/dormant-org-search/search-index-deletion'
+
+const logger = createLogger('DeleteSearchIndexDocuments')
+
+/** Refused by a precondition, before the run or before one of its pages. */
+export const EXIT_REFUSED = 2
+
+export interface DeleteSearchIndexDocumentsArgs
+  extends Omit<SearchIndexDeletionOptions, 'requestId' | 'sleep'> {
+  lockTimeoutMs: number
+  statementTimeoutMs: number
+}
+
+/** Parses the command line; every mutation requires `--execute`. */
+export function parseDeleteSearchIndexDocumentsArgs(
+  argv: readonly string[]
+): DeleteSearchIndexDocumentsArgs {
+  const { values } = parseArgs({
+    args: [...argv],
+    options: {
+      'knowledge-base-id': { type: 'string' },
+      execute: { type: 'boolean' },
+      'dry-run': { type: 'boolean' },
+      'page-size': { type: 'string' },
+      'chunk-batch-size': { type: 'string' },
+      'pause-ms': { type: 'string' },
+      'max-pages': { type: 'string' },
+      'after-id': { type: 'string' },
+      'storage-cleanup-ceiling': { type: 'string' },
+      'lock-timeout-ms': { type: 'string' },
+      'statement-timeout-ms': { type: 'string' },
+      'no-connector-reset': { type: 'boolean' },
+    },
+    strict: true,
+  })
+  const knowledgeBaseId = values['knowledge-base-id']?.trim()
+  if (!knowledgeBaseId) throw new Error('--knowledge-base-id is required')
+  return {
+    knowledgeBaseId,
+    execute: resolveExecuteFlag(values),
+    pageSize: parsePositiveInteger('page-size', values['page-size'], DEFAULT_DOCUMENT_PAGE_SIZE),
+    chunkBatchSize: parsePositiveInteger(
+      'chunk-batch-size',
+      values['chunk-batch-size'],
+      DEFAULT_CHUNK_BATCH_SIZE
+    ),
+    pauseMs: parseNonNegativeInteger('pause-ms', values['pause-ms'], DEFAULT_PAUSE_MS),
+    maxPages:
+      values['max-pages'] === undefined
+        ? undefined
+        : parsePositiveInteger('max-pages', values['max-pages'], 1),
+    afterId: values['after-id'] ?? '',
+    storageCleanupCeiling: parsePositiveInteger(
+      'storage-cleanup-ceiling',
+      values['storage-cleanup-ceiling'],
+      DEFAULT_STORAGE_CLEANUP_CEILING
+    ),
+    resetConnectors: values['no-connector-reset'] !== true,
+    lockTimeoutMs: parsePositiveInteger('lock-timeout-ms', values['lock-timeout-ms'], 5_000),
+    statementTimeoutMs: parsePositiveInteger(
+      'statement-timeout-ms',
+      values['statement-timeout-ms'],
+      60_000
+    ),
+  }
+}
+
+async function main(): Promise<number> {
+  const args = parseDeleteSearchIndexDocumentsArgs(process.argv.slice(2))
+  /** Imported after parsing, so an argument error is reported before the app's database client requires DATABASE_URL. */
+  const { drizzleSearchIndexDeletionStore } = await import(
+    '@/scripts/dormant-org-search/search-index-deletion-store'
+  )
+  const { lockTimeoutMs, statementTimeoutMs, ...options } = args
+  try {
+    await deleteSearchIndexDocuments(
+      drizzleSearchIndexDeletionStore({ lockTimeoutMs, statementTimeoutMs }),
+      { ...options, requestId: `dormant-org-search:${generateShortId()}` }
+    )
+    return 0
+  } catch (error) {
+    if (error instanceof SearchIndexDeletionRefused) {
+      logger.error('Refused by a precondition; no further page was written', {
+        reasons: error.reasons,
+      })
+      return EXIT_REFUSED
+    }
+    throw error
+  }
+}
+
+if (import.meta.main) {
+  main().then(
+    (code) => process.exit(code),
+    (error) => {
+      logger.error('Search index deletion failed', toError(error))
+      process.exit(1)
+    }
+  )
+}

apps/sim/scripts/dormant-org-search/delete-search-index-documents.ts

@@ -0,0 +1,77 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import {
+  disableTinProjection,
+  TIN_TRIGGERS,
+  type TinProjectionDatabase,
+  type TinProjectionState,
+} from '@/scripts/dormant-org-search/disable-tin-projection'
+
+/** A database whose Tin triggers and rows change only through the calls under test. */
+function fakeDatabase(initial: Partial<TinProjectionState> = {}) {
+  const state: TinProjectionState = {
+    triggers: [...TIN_TRIGGERS],
+    hasRows: true,
+    estimatedRows: 1_000,
+    totalBytes: 8_192,
+    functionsInstalled: true,
+    ...initial,
+  }
+  const database: TinProjectionDatabase & { drops: number } = {
+    drops: 0,
+    readState: async () => ({ ...state, triggers: [...state.triggers] }),
+    dropTriggersAndTruncate: async () => {
+      database.drops += 1
+      state.triggers = []
+      state.hasRows = false
+      state.totalBytes = 0
+    },
+  }
+  return { database, state }
+}
+
+describe('disableTinProjection', () => {
+  it('drops nothing in a dry run', async () => {
+    const { database } = fakeDatabase()
+    const result = await disableTinProjection(database, { execute: false })
+    expect(database.drops).toBe(0)
+    expect(result).toMatchObject({ executed: false, after: null })
+    expect(result.before.triggers.map((trigger) => trigger.name)).toEqual([
+      'embedding_keyword_tin_sync',
+      'knowledge_base_keyword_tin_sync',
+      'embedding_keyword_tin_source_acl_set',
+    ])
+  })
+
+  it('drops the triggers and truncates when executing', async () => {
+    const { database } = fakeDatabase()
+    const result = await disableTinProjection(database, { execute: true })
+    expect(database.drops).toBe(1)
+    expect(result.after).toMatchObject({ triggers: [], hasRows: false })
+  })
+
+  it('does nothing when the triggers are gone and the table is empty', async () => {
+    const { database } = fakeDatabase({ triggers: [], hasRows: false, estimatedRows: -1 })
+    const result = await disableTinProjection(database, { execute: true })
+    expect(database.drops).toBe(0)
+    expect(result.executed).toBe(false)
+  })
+
+  it('truncates again when only rows remain', async () => {
+    const { database } = fakeDatabase({ triggers: [] })
+    await disableTinProjection(database, { execute: true })
+    expect(database.drops).toBe(1)
+  })
+
+  it('fails when a trigger survives the drop', async () => {
+    const { database, state } = fakeDatabase()
+    database.dropTriggersAndTruncate = async () => {
+      state.triggers = [TIN_TRIGGERS[0]]
+    }
+    await expect(disableTinProjection(database, { execute: true })).rejects.toThrow(
+      'still installed'
+    )
+  })
+})