chore(search): partition dormant indexed org search behind a single gate

Live Search is the only active Sim Search backend. The indexed backend
(retrieval over is_search_index knowledge bases) stays in the tree for a
possible re-enable, so it now lives in one dormant module behind one switch.

- Move the exclusively-indexed code into apps/sim/lib/sim-search/indexed/
  with git mv: scoped owner search, indexed document reads, Tin keyword
  ranking, and the projection-fill probe. Shared code (projections,
  projector, member/ACL connector machinery, source overview/setup, prewarm)
  stays where it is. README documents the dormant state, the gate,
  re-enable steps, and the DB objects it depends on.
- Add isIndexedOrgSearchEnabled() in indexed/gate.ts (the inverse of
  SIM_SEARCH_LIVE) and use it at every indexed branch: the internal search
  route, MCP tools, Sim's search/read tools, the indexed document page,
  shared retrieval, and connector indexing policy.
- Add check:indexed-org-search-boundary (runs in check:audits): outside the
  directory only allowlisted entry files may import the use-case or
  retrieval barrel, each must import the gate, and no deep imports.
- Refuse a search that names a search-index knowledge base while dormant
  (Knowledge block, internal, v1, v2) with SearchIndexDormantError -> 409
  "This search index is inactive; use Sim Search." Workspace knowledge
  bases are unaffected.
- While dormant, the projector writes no embedding_keyword_tin rows and the
  source/ACL fill passes over search-index rows; the decision is passed
  into packages/db from the app caller. Workspace projection is unchanged.
- Cap the projection-fill probe at 250 ms of the leg budget and remember an
  unknown answer as unfilled for 5 s instead of re-probing every search.

Author: waleedlatif1

apps/sim/app/api/knowledge/search/route.test.ts

@@ -6,7 +6,7 @@ import { NextRequest } from 'next/server'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 const mocks = vi.hoisted(() => ({ search: vi.fn() }))
-vi.mock('@/lib/knowledge/application/workspace-search', () => ({
+vi.mock('@/lib/sim-search/indexed', () => ({
   searchScopedKnowledge: { operation: { id: 'knowledge.search' }, execute: mocks.search },
 }))
 

apps/sim/app/api/knowledge/search/route.ts

@@ -4,12 +4,12 @@ import {
   internalRateLimits,
   internalSessionAuth,
 } from '@/lib/api/server/routes'
-import { isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags'
 import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies'
 import { knowledgeOperations } from '@/lib/knowledge/application/operations'
-import { searchScopedKnowledge } from '@/lib/knowledge/application/workspace-search'
 import { DEFAULT_RERANKER_MODEL } from '@/lib/knowledge/reranker-models'
 import { sourceAuthor } from '@/lib/knowledge/search/author'
+import { searchScopedKnowledge } from '@/lib/sim-search/indexed'
+import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate'
 import { searchLiveKnowledge } from '@/lib/sim-search/live/application'
 
 const DIRECT_SEARCH_VECTOR_BUDGET_MS = 3000
@@ -81,4 +81,5 @@ const liveSearchRoute = defineInternalJsonRoute({
   present: (data) => ({ success: true as const, data }),
 })
 
-export const POST = isLiveEnterpriseSearchEnabled ? liveSearchRoute : indexedSearchRoute
+/** Indexed organization search is dormant unless its gate is on; Live Search serves otherwise. */
+export const POST = isIndexedOrgSearchEnabled() ? indexedSearchRoute : liveSearchRoute

apps/sim/app/api/v1/knowledge/search/route.test.ts

@@ -7,7 +7,13 @@
  * @vitest-environment node
  */
 
-import { createMockRequest, knowledgeApiUtilsMock, knowledgeApiUtilsMockFns } from '@sim/testing'
+import {
+  createMockRequest,
+  knowledgeApiUtilsMock,
+  knowledgeApiUtilsMockFns,
+  resetEnvFlagsMock,
+  setEnvFlags,
+} from '@sim/testing'
 import { getErrorMessage } from '@sim/utils/errors'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
@@ -118,6 +124,7 @@ const baseKb = (id: string, embeddingModel: string, embeddingDimension = 1536) =
 describe('v1 knowledge search route — per-KB embedding model', () => {
   beforeEach(() => {
     vi.clearAllMocks()
+    resetEnvFlagsMock()
     mockResolveV1KnowledgeReadAccess.mockResolvedValue({ kind: 'workspace', tokens: ['pub', 'ws'] })
     mockAuthenticateRequest.mockResolvedValue({
       requestId: 'req-1',
@@ -163,6 +170,38 @@ describe('v1 knowledge search route — per-KB embedding model', () => {
     expect(response.status).toBe(500)
   })
 
+  it('refuses a search index while indexed organization search is dormant, before any spend', async () => {
+    setEnvFlags({ isLiveEnterpriseSearchEnabled: true })
+    mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
+      hasAccess: true,
+      knowledgeBase: { ...baseKb('kb-1', 'text-embedding-3-small'), isSearchIndex: true },
+    })
+    const response = await POST(
+      createMockRequest('POST', { workspaceId: 'ws-1', knowledgeBaseIds: 'kb-1', query: 'hello' })
+    )
+    expect(response.status).toBe(409)
+    expect(await response.json()).toEqual({
+      error: 'This search index is inactive; use Sim Search.',
+    })
+    expect(mockGenerateSearchEmbedding).not.toHaveBeenCalled()
+    expect(mockExecuteKnowledgeSearch).not.toHaveBeenCalled()
+  })
+
+  it('searches a workspace knowledge base while indexed organization search is dormant', async () => {
+    setEnvFlags({ isLiveEnterpriseSearchEnabled: true })
+    mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
+      hasAccess: true,
+      knowledgeBase: { ...baseKb('kb-1', 'text-embedding-3-small'), isSearchIndex: false },
+    })
+    const response = await POST(
+      createMockRequest('POST', { workspaceId: 'ws-1', knowledgeBaseIds: 'kb-1', query: 'hello' })
+    )
+    expect(response.status).toBe(200)
+    expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith(
+      expect.objectContaining({ knowledgeBaseIds: ['kb-1'], searchIndexOnly: false })
+    )
+  })
+
   it('retains the reader provider for ranked results and returned document metadata', async () => {
     const access = { kind: 'user' as const, userId: 'user-1', tokens: ['reader-token'] }
     const provider = {

apps/sim/app/api/v1/knowledge/search/route.ts

@@ -24,6 +24,10 @@ import {
 import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service'
 import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils'
 import type { StructuredFilter } from '@/lib/knowledge/types'
+import {
+  isIndexedOrgSearchEnabled,
+  SEARCH_INDEX_DORMANT_MESSAGE,
+} from '@/lib/sim-search/indexed/gate'
 import { checkKnowledgeBaseAccess, type KnowledgeBaseAccessResult } from '@/app/api/knowledge/utils'
 import { handleError, resolveV1KnowledgeReadAccess } from '@/app/api/v1/knowledge/utils'
 import {
@@ -121,6 +125,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
       )
     }
 
+    /** A search index is readable only while indexed organization search is on. */
+    if (!isIndexedOrgSearchEnabled() && accessibleKbs.some((kb) => kb.isSearchIndex)) {
+      return NextResponse.json({ error: SEARCH_INDEX_DORMANT_MESSAGE }, { status: 409 })
+    }
+
     let structuredFilters: StructuredFilter[] = []
     const tagDefsCache = new Map<string, Awaited<ReturnType<typeof getDocumentTagDefinitions>>>()
 

apps/sim/app/o/[organizationId]/knowledge/[knowledgeBaseId]/[documentId]/page.tsx

@@ -4,7 +4,8 @@ import type { SearchParams } from 'nuqs/server'
 import { readSearchDocumentResultSchema } from '@/lib/api/contracts/knowledge/documents'
 import { getSession } from '@/lib/auth'
 import { OrchestrationError } from '@/lib/core/orchestration/types'
-import { readSearchDocument } from '@/lib/knowledge/application/read-search-document'
+import { readSearchDocument } from '@/lib/sim-search/indexed'
+import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate'
 import { buildAuthCrossLink } from '@/app/(auth)/auth-redirect'
 import {
   loadDocumentReadParams,
@@ -22,6 +23,8 @@ export default async function OrganizationDocumentPage({
   params,
   searchParams,
 }: OrganizationDocumentPageProps) {
+  /** Indexed documents are served only while indexed organization search is on. */
+  if (!isIndexedOrgSearchEnabled()) notFound()
   const { organizationId, knowledgeBaseId, documentId } = await params
   const position = await loadDocumentReadParams(searchParams, { strict: true }).catch(() =>
     notFound()

apps/sim/lib/knowledge/__integration__/excluded-member-documents.integration.ts

@@ -17,6 +17,11 @@ import { eq, inArray } from 'drizzle-orm'
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
 
 const provider = vi.hoisted(() => ({ list: vi.fn(), get: vi.fn(), changes: vi.fn() }))
+/** This suite covers indexed organization search, which is dormant unless Live Search is off. */
+vi.mock('@/lib/core/config/env-flags', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@/lib/core/config/env-flags')>()),
+  isLiveEnterpriseSearchEnabled: false,
+}))
 vi.mock('@/connectors/registry.server', () => ({
   CONNECTOR_REGISTRY: {
     google_drive: {

apps/sim/lib/knowledge/__integration__/github-member.integration.ts

@@ -30,6 +30,11 @@ import { generateId } from '@sim/utils/id'
 import { and, eq, inArray, isNull, sql } from 'drizzle-orm'
 import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
+/** This suite covers indexed organization search, which is dormant unless Live Search is off. */
+vi.mock('@/lib/core/config/env-flags', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@/lib/core/config/env-flags')>()),
+  isLiveEnterpriseSearchEnabled: false,
+}))
 vi.mock('@/lib/embeddings', async () => ({
   ...(await import('@/lib/embeddings/client')),
   assertKnowledgeEmbeddingCapacity: async () => {},
@@ -83,7 +88,6 @@ import {
   listKnowledgeConnectorDocuments,
 } from '@/lib/knowledge/application/connectors'
 import { readKnowledgeDocument } from '@/lib/knowledge/application/documents'
-import { readIndexedKnowledgeDocument } from '@/lib/knowledge/application/read-indexed-document'
 import { searchKnowledge } from '@/lib/knowledge/application/search'
 import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview'
 import { listSearchSources } from '@/lib/knowledge/application/search-sources'
@@ -95,6 +99,7 @@ import {
 } from '@/lib/knowledge/connectors/sync-limits'
 import { getDocuments } from '@/lib/knowledge/documents/service'
 import { getTagUsageStats } from '@/lib/knowledge/tags/service'
+import { readIndexedKnowledgeDocument } from '@/lib/sim-search/indexed/documents/read-indexed-document'
 import { deleteFile } from '@/lib/uploads/core/storage-service'
 import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'

apps/sim/lib/knowledge/__integration__/gitlab-live.integration.ts

@@ -86,9 +86,9 @@ import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks'
 import { updateKnowledgeConnectorAccess } from '@/lib/knowledge/application/connector-access'
 import { updateKnowledgeConnector } from '@/lib/knowledge/application/connectors'
 import { readKnowledgeDocument } from '@/lib/knowledge/application/documents'
-import { readSearchDocument } from '@/lib/knowledge/application/read-search-document'
 import { searchKnowledge } from '@/lib/knowledge/application/search'
 import { executeSync } from '@/lib/knowledge/connectors/sync-engine'
+import { readSearchDocument } from '@/lib/sim-search/indexed/documents/read-search-document'
 import * as storage from '@/lib/uploads/core/storage-service'
 import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server'
 import { PATCH as updateConnectorRoute } from '@/app/api/knowledge/[id]/connectors/[connectorId]/route'

apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts

@@ -24,6 +24,11 @@ import { eq, inArray } from 'drizzle-orm'
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
 
 const counters = vi.hoisted(() => ({ embeddedTexts: 0 }))
+/** This suite covers indexed organization search, which is dormant unless Live Search is off. */
+vi.mock('@/lib/core/config/env-flags', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@/lib/core/config/env-flags')>()),
+  isLiveEnterpriseSearchEnabled: false,
+}))
 vi.mock('@/lib/embeddings', async () => ({
   ...(await import('@/lib/embeddings/client')),
   assertKnowledgeEmbeddingCapacity: async () => {},

apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts

@@ -24,6 +24,11 @@ import { generateId } from '@sim/utils/id'
 import { and, eq, inArray, isNull } from 'drizzle-orm'
 import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
 
+/** This suite covers indexed organization search, which is dormant unless Live Search is off. */
+vi.mock('@/lib/core/config/env-flags', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@/lib/core/config/env-flags')>()),
+  isLiveEnterpriseSearchEnabled: false,
+}))
 vi.mock('@/lib/embeddings', async () => ({
   ...(await import('@/lib/embeddings/client')),
   assertKnowledgeEmbeddingCapacity: async () => {},