Skip to content

Commit 9cf2d31

Browse files
authored
feat(search): read Drive documents and Gmail conversations (#8352)
* feat(search): read Drive documents and Gmail conversations * fix(search): bound document conversion and preserve complete reads
1 parent c979b63 commit 9cf2d31

30 files changed

Lines changed: 1665 additions & 79 deletions

‎.github/workflows/test-build.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,24 @@ jobs:
129129
if-no-files-found: warn
130130
retention-days: 14
131131

132+
- name: Verify Google document reads over real HTTP
133+
if: matrix.provision == 'push'
134+
working-directory: apps/sim
135+
env:
136+
NEXT_PUBLIC_APP_URL: http://127.0.0.1:3040
137+
NEXT_PUBLIC_FORCE_HOSTED: 'false'
138+
SEARCH_GOOGLE_CONTENT_REPORT_PATH: ${{ runner.temp }}/search-google-content.json
139+
run: bun scripts/test-search-google-content-e2e.ts
140+
141+
- name: Upload Google content acceptance report
142+
if: failure() && matrix.provision == 'push'
143+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
144+
with:
145+
name: search-google-content
146+
path: ${{ runner.temp }}/search-google-content.json
147+
if-no-files-found: ignore
148+
retention-days: 7
149+
132150
- name: Verify SCIM and administration over real HTTP
133151
working-directory: apps/sim
134152
env:

‎apps/sim/lib/file-parsers/docx-parser.ts‎

Lines changed: 106 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { readFile } from 'fs/promises'
22
import { createLogger } from '@sim/logger'
3+
import { isRecordLike, toRecord } from '@sim/utils/object'
34
import mammoth from 'mammoth'
45
import {
56
FileParserError,
@@ -19,6 +20,79 @@ import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard'
1920

2021
const logger = createLogger('DocxParser')
2122

23+
/** Bounds repeated notes and generated markup independently of the normalized text budget. */
24+
const MAX_DOCX_CONVERSION_NODES = 50_000
25+
const MAX_DOCX_CONVERSION_BYTES = 2 * 1024 * 1024
26+
27+
/**
28+
* Mammoth's supported transform hook runs before HTML generation. Count every reference
29+
* expansion, including repeated notes, without retaining the expanded graph. The node ceiling
30+
* also stops cyclic references. The separate 2 MiB ceiling charges raw UTF-8 model strings,
31+
* including attributes, before HTML escaping. Fixed default styles and omitted image data
32+
* bound conversion amplification; embedded style maps could add arbitrary wrapper markup.
33+
*/
34+
function assertDocxConversionWithinLimits(document: unknown, signal?: AbortSignal): void {
35+
const notes = toRecord(toRecord(document).notes)
36+
const pending: unknown[] = [document]
37+
let nodes = 0
38+
let bytes = 0
39+
const complexity = () =>
40+
new FileParserError('complexity_limit', 'DOCX conversion exceeds its graph budget')
41+
const append = (children: unknown) => {
42+
if (!Array.isArray(children))
43+
throw new FileParserError('invalid_format', 'DOCX conversion has invalid children')
44+
if (nodes + pending.length + children.length > MAX_DOCX_CONVERSION_NODES) throw complexity()
45+
for (let index = children.length - 1; index >= 0; index--) pending.push(children[index])
46+
}
47+
while (pending.length) {
48+
signal?.throwIfAborted()
49+
const node = pending.pop()
50+
if (!isRecordLike(node) || typeof node.type !== 'string')
51+
throw new FileParserError('invalid_format', 'DOCX conversion has an invalid node')
52+
if (++nodes > MAX_DOCX_CONVERSION_NODES) throw complexity()
53+
for (const value of Object.values(node)) {
54+
if (typeof value === 'string') bytes += Buffer.byteLength(value, 'utf8')
55+
}
56+
if (bytes > MAX_DOCX_CONVERSION_BYTES) throw complexity()
57+
switch (node.type) {
58+
case 'document':
59+
case 'paragraph':
60+
case 'run':
61+
case 'hyperlink':
62+
case 'table':
63+
case 'tableRow':
64+
case 'tableCell':
65+
append(node.children)
66+
break
67+
case 'note':
68+
case 'comment':
69+
append(node.body)
70+
break
71+
case 'noteReference': {
72+
if (typeof notes.resolve !== 'function')
73+
throw new FileParserError('invalid_format', 'DOCX note resolver is unavailable')
74+
const note: unknown = Reflect.apply(notes.resolve, notes, [node])
75+
if (!note) throw new FileParserError('invalid_format', 'DOCX references a missing note')
76+
append([note])
77+
break
78+
}
79+
case 'text':
80+
if (typeof node.value !== 'string')
81+
throw new FileParserError('invalid_format', 'DOCX text is malformed')
82+
break
83+
case 'image':
84+
case 'tab':
85+
case 'checkbox':
86+
case 'break':
87+
case 'bookmarkStart':
88+
case 'commentReference':
89+
break
90+
default:
91+
throw new FileParserError('invalid_format', 'DOCX conversion has an unsupported node')
92+
}
93+
}
94+
}
95+
2296
/**
2397
* Extracts DOCX text by rendering the document to HTML with mammoth and walking
2498
* that HTML with the shared structured-text walker. mammoth's HTML keeps the
@@ -45,25 +119,49 @@ export class DocxParser implements FileParser {
45119
}
46120

47121
assertOoxmlArchiveWithinLimits(buffer)
122+
const maxTextBytes =
123+
options.docxTextMode === 'complete'
124+
? (options.maxTextBytes ?? MAX_DOCX_CONVERSION_BYTES)
125+
: undefined
126+
if (maxTextBytes !== undefined && (!Number.isSafeInteger(maxTextBytes) || maxTextBytes <= 0))
127+
throw new FileParserError('complexity_limit', 'Invalid DOCX text byte budget')
48128

49129
const extractionErrors: unknown[] = []
50130
let parserReturnedEmpty = false
51131

52132
try {
53-
const htmlResult = await mammoth.convertToHtml({ buffer })
133+
const htmlResult = await mammoth.convertToHtml(
134+
{ buffer },
135+
maxTextBytes === undefined
136+
? undefined
137+
: {
138+
includeEmbeddedStyleMap: false,
139+
convertImage: mammoth.images.imgElement(async () => ({ src: '' })),
140+
transformDocument: (document: unknown) => {
141+
assertDocxConversionWithinLimits(document, options.signal)
142+
return document
143+
},
144+
}
145+
)
54146
options.signal?.throwIfAborted()
55147

56-
const structured = this.structuredTextFromHtml(htmlResult.value)
148+
const structured = this.structuredTextFromHtml(htmlResult.value, maxTextBytes !== undefined)
57149
if (structured) {
150+
const content = sanitizeTextForUTF8(structured)
151+
if (maxTextBytes !== undefined && Buffer.byteLength(content, 'utf8') > maxTextBytes)
152+
throw new FileParserError('complexity_limit', 'DOCX text exceeds its byte budget')
58153
return {
59-
content: sanitizeTextForUTF8(structured),
154+
content,
60155
metadata: {
61156
extractionMethod: 'mammoth-html',
62157
messages: htmlResult.messages,
63158
},
64159
}
65160
}
66161

162+
if (maxTextBytes !== undefined)
163+
throw new FileParserError('no_extractable_text', 'No complete DOCX text was extracted')
164+
67165
const rawResult = await mammoth.extractRawText({ buffer })
68166
options.signal?.throwIfAborted()
69167

@@ -79,6 +177,7 @@ export class DocxParser implements FileParser {
79177
parserReturnedEmpty = true
80178
} catch (mammothError) {
81179
options.signal?.throwIfAborted()
180+
if (maxTextBytes !== undefined) throw mammothError
82181
logger.warn('mammoth failed, trying officeparser:', mammothError)
83182
extractionErrors.push(mammothError)
84183
}
@@ -161,14 +260,16 @@ export class DocxParser implements FileParser {
161260
/**
162261
* Walks mammoth's HTML rendering under the HTML parser's size caps. A rendering
163262
* too large to walk safely falls back to the raw-text path by returning empty,
164-
* since mammoth has already materialised the document once at that point.
263+
* since mammoth has already materialised the document once at that point. Budgeted reads
264+
* reject instead: the fallback cannot preserve the conversion and completeness bounds.
165265
*/
166-
private structuredTextFromHtml(html: string): string {
266+
private structuredTextFromHtml(html: string, bounded = false): string {
167267
if (!html || html.trim().length === 0) return ''
168268
try {
169269
assertHtmlStringWithinLimits(html)
170270
} catch (error) {
171271
if (isHtmlComplexityError(error)) {
272+
if (bounded) throw error
172273
logger.warn('mammoth HTML exceeds walker limits, using raw text:', error.message)
173274
return ''
174275
}

‎apps/sim/lib/file-parsers/pdf-parser.ts‎

Lines changed: 42 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -36,17 +36,18 @@ export const MAX_PDF_TEXT_CHARS = 10_000_000
3636
/** Complete extraction shares the ingestion pipeline's bounded text-output envelope. */
3737
export const MAX_COMPLETE_PDF_TEXT_BYTES = 20 * 1024 * 1024
3838

39+
/** Independent retained-text ceiling, including conservative layout overhead. */
40+
const MAX_COMPLETE_PDF_RETAINED_TEXT_BYTES = MAX_COMPLETE_PDF_TEXT_BYTES
41+
3942
/** Bounds expansion on one page independently of a long document's output budget. */
4043
export const MAX_COMPLETE_PDF_PAGE_CHARS = 250_000
4144

4245
/** Wall-clock ceiling for extracting text from a whole document. */
4346
const PDF_EXTRACTION_TIMEOUT_MS = 60_000
4447

4548
/**
46-
* Upper bound on what line reconstruction adds per line after the budget is
47-
* spent: a two-character paragraph break plus a three-character heading marker.
48-
* The complete-mode byte ceiling therefore trips slightly earlier than it did
49-
* when pages were flattened to one line, by at most this many bytes per line.
49+
* Conservative retained-text allowance for a paragraph break and heading marker.
50+
* This protects parser state independently of the caller's rendered-output budget.
5051
*/
5152
const MAX_LINE_DECORATION_BYTES = 5
5253

@@ -249,8 +250,8 @@ function readPageHeight(page: PDFPageProxy): number | undefined {
249250
}
250251
}
251252

252-
/** Bytes a page's lines can occupy in the output once joined and decorated. */
253-
function estimatePageBytes(lines: readonly PdfLine[]): number {
253+
/** Conservative text storage for retained lines and their possible decorations. */
254+
function estimateRetainedPageBytes(lines: readonly PdfLine[]): number {
254255
let bytes = 0
255256
for (const line of lines) {
256257
bytes += Buffer.byteLength(line.text, 'utf8') + MAX_LINE_DECORATION_BYTES
@@ -267,7 +268,8 @@ function estimatePageBytes(lines: readonly PdfLine[]): number {
267268
async function assemblePages(
268269
pages: readonly PdfPageLines[],
269270
complete: boolean,
270-
signal: AbortSignal | undefined
271+
signal: AbortSignal | undefined,
272+
maxTextBytes: number
271273
): Promise<string> {
272274
signal?.throwIfAborted()
273275
const filteredPages = suppressFurniture(pages)
@@ -280,14 +282,25 @@ async function assemblePages(
280282
headingMarkers: PDF_HEADING_MARKERS_ENABLED && headingMarkersViable(allLines, bodyHeight),
281283
}
282284
const pageTexts: string[] = []
285+
let outputBytes = 0
283286
for (const [index, lines] of filteredPages.entries()) {
284287
if (index > 0 && index % ASSEMBLY_YIELD_EVERY_PAGES === 0) {
285288
await sleep(0)
286289
signal?.throwIfAborted()
287290
}
288291
const joined = joinLines(lines, options)
289292
const text = complete ? normalizePdfWhitespace(sanitizeTextForUTF8(joined)).trim() : joined
290-
if (text.length > 0) pageTexts.push(text)
293+
if (text.length > 0) {
294+
if (complete) {
295+
outputBytes +=
296+
Buffer.byteLength(text, 'utf8') + (pageTexts.length > 0 ? PAGE_SEPARATOR.length : 0)
297+
if (outputBytes > maxTextBytes)
298+
throw completeExtractionLimit(
299+
`PDF text exceeds the safe ${maxTextBytes.toLocaleString()}-byte output limit.`
300+
)
301+
}
302+
pageTexts.push(text)
303+
}
291304
}
292305
const text = pageTexts.join(PAGE_SEPARATOR)
293306
return complete ? text : normalizePdfWhitespace(text).trim()
@@ -297,6 +310,13 @@ function completeExtractionLimit(message: string): FileParserError {
297310
return new FileParserError('complexity_limit', `${message} Split or simplify the PDF and retry.`)
298311
}
299312

313+
function completeBudget(value: number | undefined, ceiling: number): number {
314+
if (value === undefined) return ceiling
315+
if (!Number.isSafeInteger(value) || value < 1)
316+
throw completeExtractionLimit('PDF extraction limits must be positive safe integers.')
317+
return Math.min(value, ceiling)
318+
}
319+
300320
async function extractTextWithinBudget(
301321
pdf: PDFDocumentProxy,
302322
options: FileParseOptions,
@@ -305,17 +325,21 @@ async function extractTextWithinBudget(
305325
const { signal } = options
306326
const complete = options.pdfTextMode === 'complete'
307327
const totalPages = pdf.numPages
308-
const pageLimit = Math.min(totalPages, MAX_PDF_PAGES)
328+
const maxPages = complete ? completeBudget(options.pdfMaxPages, MAX_PDF_PAGES) : MAX_PDF_PAGES
329+
const maxTextBytes = complete
330+
? completeBudget(options.maxTextBytes, MAX_COMPLETE_PDF_TEXT_BYTES)
331+
: MAX_COMPLETE_PDF_TEXT_BYTES
332+
const pageLimit = Math.min(totalPages, maxPages)
309333
const pages: PdfPageLines[] = []
310334

311335
let remainingChars = MAX_PDF_TEXT_CHARS
312-
let outputBytes = 0
336+
let retainedTextBytes = 0
313337
let pagesRead = 0
314338
let truncated = totalPages > pageLimit
315339

316340
if (complete && truncated) {
317341
throw completeExtractionLimit(
318-
`PDF exceeds the safe limit of ${MAX_PDF_PAGES.toLocaleString()} pages.`
342+
`PDF exceeds the safe limit of ${maxPages.toLocaleString()} pages.`
319343
)
320344
}
321345

@@ -341,13 +365,9 @@ async function extractTextWithinBudget(
341365
const page = pageResult
342366
const pageHeight = readPageHeight(page)
343367
let extraction: PageExtraction
368+
const pageCharLimit = complete ? MAX_COMPLETE_PDF_PAGE_CHARS : remainingChars
344369
try {
345-
extraction = await readPageWithinBudget(
346-
page,
347-
complete ? MAX_COMPLETE_PDF_PAGE_CHARS : remainingChars,
348-
deadline,
349-
signal
350-
)
370+
extraction = await readPageWithinBudget(page, pageCharLimit, deadline, signal)
351371
} finally {
352372
page.cleanup()
353373
}
@@ -361,10 +381,11 @@ async function extractTextWithinBudget(
361381
pagesRead++
362382
if (complete) {
363383
if (lines.length > 0) {
364-
outputBytes += estimatePageBytes(lines) + (pages.length > 0 ? PAGE_SEPARATOR.length : 0)
365-
if (outputBytes > MAX_COMPLETE_PDF_TEXT_BYTES) {
384+
retainedTextBytes +=
385+
estimateRetainedPageBytes(lines) + (pages.length > 0 ? PAGE_SEPARATOR.length : 0)
386+
if (retainedTextBytes > MAX_COMPLETE_PDF_RETAINED_TEXT_BYTES) {
366387
throw completeExtractionLimit(
367-
`PDF text exceeds the safe ${MAX_COMPLETE_PDF_TEXT_BYTES.toLocaleString()}-byte output limit.`
388+
`PDF text exceeds the safe ${MAX_COMPLETE_PDF_RETAINED_TEXT_BYTES.toLocaleString()}-byte output limit.`
368389
)
369390
}
370391
pages.push({ lines, pageHeight })
@@ -387,7 +408,7 @@ async function extractTextWithinBudget(
387408
}
388409
}
389410

390-
let text = await assemblePages(pages, complete, signal)
411+
let text = await assemblePages(pages, complete, signal, maxTextBytes)
391412

392413
/** Paragraph breaks land after the budget is spent; trimming that overflow is a truncation too. */
393414
if (!complete && text.length > MAX_PDF_TEXT_CHARS) {

‎apps/sim/lib/file-parsers/types.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,22 @@ export interface FileParseOptions {
3636
/** Indexing callers require complete extraction; preview row limits must not discard content. */
3737
contentMode?: 'preview' | 'complete'
3838
/**
39-
* CSV/XLSX byte budget when contentMode is 'complete' (default 25 MiB).
39+
* Complete text byte budget for CSV/XLSX (default 25 MiB) and PDF (default 20 MiB).
40+
* PDF applies this when pdfTextMode is 'complete' and never exceeds its safe default.
4041
* Exceeding it throws complexity_limit instead of returning a truncated prefix.
42+
* DOCX applies this only when docxTextMode is 'complete' (default 2 MiB); separate
43+
* conversion graph limits can reject structurally complex inputs before normalization.
4144
* Preview mode retains its own limits; other formats use their parser-specific budgets.
4245
*/
4346
maxTextBytes?: number
4447
/** Preserve textual markup in a canonical .txt artifact instead of interpreting it as HTML or RTF. */
4548
textMode?: 'literal'
49+
/** Opt into bounded DOCX conversion without lossy or unchecked parser fallbacks. */
50+
docxTextMode?: 'complete'
4651
/** Complete PDF extraction rejects safety limits instead of returning preview text. */
4752
pdfTextMode?: 'preview' | 'complete'
53+
/** Lower page ceiling for complete PDF extraction; defaults to the parser's safe limit. */
54+
pdfMaxPages?: number
4855
}
4956

5057
export interface FileParser {

‎apps/sim/lib/integrations/availability.server.oauth-configured.test.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { setEnv } from '@sim/testing/mocks/env.mock'
2-
import { describe, expect, it } from 'vitest'
1+
import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock'
2+
import { afterAll, describe, expect, it } from 'vitest'
33
import {
44
getIntegrationAvailability,
55
getOAuthServiceAvailability,
@@ -8,7 +8,10 @@ import {
88
setEnv({
99
GITHUB_APP_CLIENT_ID: 'repository-client',
1010
GITHUB_APP_CLIENT_SECRET: 'repository-secret',
11+
GOOGLE_CLIENT_ID: undefined,
12+
GOOGLE_CLIENT_SECRET: undefined,
1113
})
14+
afterAll(resetEnvMock)
1215

1316
describe('OAuth service availability projection', () => {
1417
it('uses the repository App while the workflow block keeps its API-key path', () => {

0 commit comments

Comments
 (0)