fix(knowledge): preserve fallback ranking and measure result bytes

Author: icecrasher321

apps/sim/lib/copilot/request/tools/executor.test.ts

@@ -146,6 +146,42 @@ function buildPendingToolCall(): ToolCallState {
   }
 }
 
+describe('tool result size diagnostics', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    completeAsyncToolCall.mockResolvedValue(null)
+    markAsyncToolRunning.mockResolvedValue(null)
+    upsertAsyncToolCall.mockResolvedValue(null)
+  })
+
+  it.each(['é🔎', { content: 'é🔎' }])(
+    'records UTF-8 bytes after result projection for %j',
+    async (output) => {
+      executeTool.mockResolvedValueOnce({ success: true, output })
+      const toolCall = buildPendingToolCall()
+      const context = buildStreamingContext(toolCall)
+      const endSpan = vi.spyOn(context.trace, 'endSpan')
+
+      const completion = await executeToolAndReport(toolCall.id, context, {
+        userId: 'user-1',
+        workflowId: 'workflow-1',
+        resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(),
+      })
+
+      expect(completion.status).toBe(MothershipStreamV1ToolOutcome.success)
+      const serialized =
+        typeof completion.data === 'string' ? completion.data : JSON.stringify(completion.data)
+      expect(endSpan).toHaveBeenCalledWith(
+        expect.objectContaining({
+          kind: 'tool.execute',
+          attributes: expect.objectContaining({ outputBytes: Buffer.byteLength(serialized) }),
+        }),
+        'ok'
+      )
+    }
+  )
+})
+
 describe('toolWatchdogTimeoutMs', () => {
   it('gives request-scoped MCP tools the long-running watchdog', () => {
     expect(toolWatchdogTimeoutMs('mcp-363de040-web_search_exa')).toBe(TOOL_WATCHDOG_LONG_RUNNING_MS)

apps/sim/lib/copilot/request/tools/executor.ts

@@ -125,11 +125,11 @@ function summarizeToolResultForSpan(result: {
   const output = (result as { output: unknown }).output
   if (typeof output === 'string') {
     summary.outputKind = 'string'
-    summary.outputBytes = output.length
+    summary.outputBytes = Buffer.byteLength(output)
   } else if (output && typeof output === 'object') {
     summary.outputKind = Array.isArray(output) ? 'array' : 'object'
     try {
-      summary.outputBytes = JSON.stringify(output).length
+      summary.outputBytes = Buffer.byteLength(JSON.stringify(output))
     } catch {
       summary.outputBytes = 0
     }
@@ -143,7 +143,7 @@ function summarizeToolResultForSpan(result: {
     }
   } else if (output !== undefined && output !== null) {
     summary.outputKind = typeof output
-    summary.outputBytes = String(output).length
+    summary.outputBytes = Buffer.byteLength(String(output))
   }
   return summary
 }

apps/sim/lib/copilot/tools/server/knowledge/workspace-search.test.ts

@@ -1,7 +1,15 @@
 /** @vitest-environment node */
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const mocks = vi.hoisted(() => ({ search: vi.fn(), read: vi.fn(), authorizeChat: vi.fn() }))
+const mocks = vi.hoisted(() => ({
+  search: vi.fn(),
+  read: vi.fn(),
+  authorizeChat: vi.fn(),
+  info: vi.fn(),
+}))
+vi.mock('@sim/logger', () => ({
+  createLogger: () => ({ info: mocks.info, error: vi.fn(), warn: vi.fn() }),
+}))
 vi.mock('@/lib/copilot/chat/organization-chats', () => ({
   authorizeOrganizationChatDelegation: { execute: mocks.authorizeChat },
 }))
@@ -179,6 +187,48 @@ describe('Assistant retrieval tools', () => {
       })
     )
   })
+  it.each([0, 20, 50])(
+    'measures UTF-8 bytes for %i passages without logging their content',
+    async (count) => {
+      const content = 'Confidential passage é🔎'.repeat(100)
+      mocks.search.mockResolvedValueOnce({
+        knowledgeBases: [{ id: 'index', name: 'Enterprise Search' }],
+        results: Array.from({ length: count }, (_, index) => ({
+          knowledgeBaseId: 'index',
+          documentId: `doc-${index % 4}`,
+          documentName: 'Private title',
+          sourceUrl: null,
+          sourceModifiedAt: null,
+          metadata: {},
+          content,
+          chunkIndex: index,
+          similarity: 1,
+        })),
+      })
+
+      const output = await searchWorkspaceServerTool.execute(
+        { query: 'Private query', ...(count === 50 ? { topK: 50 } : {}) },
+        context
+      )
+
+      expect(output.success).toBe(true)
+      expect(mocks.info).toHaveBeenCalledWith(
+        'Knowledge search completed',
+        expect.objectContaining({
+          toolCallId: 'call',
+          toolResultBytes: Buffer.byteLength(JSON.stringify(output)),
+          passageBytes: count * Buffer.byteLength(content),
+          maxPassageBytes: count ? Buffer.byteLength(content) : 0,
+          uniqueDocumentCount: Math.min(count, 4),
+        })
+      )
+      const logged = JSON.stringify(mocks.info.mock.calls)
+      expect(logged).not.toContain('Confidential passage')
+      expect(logged).not.toContain('Private title')
+      expect(logged).not.toContain('Private query')
+    }
+  )
+
   it('returns stable citation IDs with internal links for uploaded documents', async () => {
     const result = await searchWorkspaceServerTool.execute({ query: 'orion' }, context)
     expect(result).toMatchObject({

apps/sim/lib/copilot/tools/server/knowledge/workspace-search.ts

@@ -17,6 +17,7 @@ import {
 import { sourceAuthor } from '@/lib/knowledge/search/author'
 import { createKnowledgeDocumentCitation } from '@/lib/knowledge/search/citation'
 import {
+  annotateSearchDiagnostics,
   measureSearchStage,
   recordSearchStageDuration,
   withSearchDiagnostics,
@@ -84,7 +85,7 @@ export const searchWorkspaceServerTool: BaseServerTool = {
           )
           return await measureSearchStage('tool_presentation', () => {
             const names = new Map(result.knowledgeBases.map((base) => [base.id, base.name]))
-            return {
+            const output = {
               success: true,
               message: `Found ${result.results.length} passages. ${CITATION_INSTRUCTION}`,
               data: {
@@ -114,6 +115,14 @@ export const searchWorkspaceServerTool: BaseServerTool = {
                 })),
               },
             }
+            const passageBytes = output.data.results.map((item) => Buffer.byteLength(item.content))
+            annotateSearchDiagnostics({
+              toolResultBytes: Buffer.byteLength(JSON.stringify(output)),
+              passageBytes: passageBytes.reduce((total, bytes) => total + bytes, 0),
+              maxPassageBytes: Math.max(0, ...passageBytes),
+              uniqueDocumentCount: new Set(output.data.results.map((item) => item.documentId)).size,
+            })
+            return output
           })
         } catch (error) {
           logger.error('Workspace search failed', { error })

apps/sim/lib/knowledge/__integration__/search-latency.integration.ts

@@ -132,6 +132,10 @@ const diagnosticSchema = z
     surface: z.enum(['dashboard', 'copilot']),
     outcome: z.literal('success'),
     elapsedMs: z.number(),
+    toolResultBytes: z.number().int().nonnegative().optional(),
+    passageBytes: z.number().int().nonnegative().optional(),
+    maxPassageBytes: z.number().int().nonnegative().optional(),
+    uniqueDocumentCount: z.number().int().nonnegative().optional(),
     stages: z.record(
       z.string(),
       z.object({
@@ -197,6 +201,15 @@ async function sample(label: string, run: () => ReturnType<typeof search>) {
   const diagnostics = diagnosticSchema.parse(completed[0][1])
   expect(diagnostics.stages.embedding.count).toBe(1)
   expect(diagnostics.stages.retrieval.count).toBe(1)
+  if (diagnostics.surface === 'copilot') {
+    const passageBytes = result.data.results.map((row) => Buffer.byteLength(row.content))
+    expect(diagnostics.passageBytes).toBe(passageBytes.reduce((total, bytes) => total + bytes, 0))
+    expect(diagnostics.maxPassageBytes).toBe(Math.max(0, ...passageBytes))
+    expect(diagnostics.uniqueDocumentCount).toBe(
+      new Set(result.data.results.map((row) => row.documentId)).size
+    )
+    expect(diagnostics.toolResultBytes).toBeGreaterThan(diagnostics.passageBytes!)
+  }
   expect(captured.length).toBeLessThan(300)
   const searches = captured.filter(
     (item) =>

apps/sim/lib/knowledge/search/diagnostics.ts

@@ -65,6 +65,11 @@ export interface SearchDiagnosticMetadata {
   boostRecency?: boolean
   embeddingDimensions?: number
   resultCount?: number
+  /** Tool output before the executor's final egress projection; counts only, never content. */
+  toolResultBytes?: number
+  passageBytes?: number
+  maxPassageBytes?: number
+  uniqueDocumentCount?: number
 }
 
 interface StageTiming {

apps/sim/lib/knowledge/search/queries.test.ts

@@ -606,6 +606,76 @@ describe('live repository authorization follows ranked candidates', () => {
     )
   })
 
+  it('restarts exact ranking at zero and advances past already considered ANN candidates', async () => {
+    const probe = Array.from({ length: 200 }, (_, index) =>
+      candidate(`probe-${index}`, 'allowed-source')
+    )
+    const approximate = Array.from({ length: 20 }, (_, index) =>
+      candidate(`approximate-${index}`, 'allowed-source')
+    )
+    queueTableRows(schemaMock.embedding, probe)
+    queueTableRows(schemaMock.embedding, approximate)
+    queueTableRows(schemaMock.embedding, [])
+    queueTableRows(schemaMock.embedding, probe)
+    queueTableRows(schemaMock.embedding, [])
+    queueTableRows(schemaMock.embedding, approximate)
+    queueTableRows(schemaMock.embedding, probe)
+    queueTableRows(schemaMock.embedding, [candidate('selected', 'allowed-source')])
+    queueTableRows(schemaMock.embedding, [
+      { id: 'selected', content: 'Reachable after the exact restart', distance: 0.1 },
+    ])
+
+    const rows = await handleVectorOnlySearch({ ...params, structuredFilters: undefined })
+
+    expect(rows.map((row) => row.id)).toEqual(['selected'])
+    expect(dbChainMockFns.offset.mock.calls).toEqual([[0], [20], [0], [20]])
+    expect(getForConnectors).toHaveBeenCalledTimes(2)
+    expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(3)
+  })
+
+  it('keeps the nearest exact results when an earlier ANN page hydrated only a farther result', async () => {
+    const probe = Array.from({ length: 200 }, (_, index) =>
+      candidate(`probe-${index}`, 'allowed-source')
+    )
+    queueTableRows(schemaMock.embedding, probe)
+    queueTableRows(schemaMock.embedding, [
+      { ...candidate('far', 'allowed-source'), distance: 0.7 },
+      ...Array.from({ length: 19 }, (_, index) => candidate(`hidden-${index}`, 'allowed-source')),
+    ])
+    queueTableRows(schemaMock.embedding, [{ id: 'far', content: 'Far result', distance: 0.7 }])
+    queueTableRows(schemaMock.embedding, probe)
+    queueTableRows(schemaMock.embedding, [])
+    queueTableRows(schemaMock.embedding, [
+      candidate('near', 'allowed-source'),
+      candidate('nearer', 'allowed-source'),
+      candidate('far', 'allowed-source'),
+    ])
+    queueTableRows(schemaMock.embedding, [
+      { id: 'near', content: 'Near result', distance: 0.2 },
+      { id: 'nearer', content: 'Nearest result', distance: 0.1 },
+    ])
+
+    const rows = await handleVectorOnlySearch({
+      ...params,
+      topK: 2,
+      structuredFilters: undefined,
+    })
+
+    expect(rows.map((row) => row.id)).toEqual(['nearer', 'near'])
+    expect(dbChainMockFns.offset.mock.calls).toEqual([[0], [20], [0]])
+    expect(
+      hasMockCondition(
+        dbChainMockFns.where.mock.calls.at(-1)![0],
+        (node) =>
+          node.type === 'inArray' &&
+          node.column === schemaMock.embedding.id &&
+          Array.isArray(node.values) &&
+          node.values.length === 2 &&
+          !node.values.includes('far')
+      )
+    ).toBe(true)
+  })
+
   it.each(['vector', 'tag-vector', 'tags', 'keyword'] as const)(
     '%s ranks identifiers before verification and loads content under the full predicate',
     async (mode) => {