diff --git a/lib/http/upload-to-disk.util.ts b/lib/http/upload-to-disk.util.ts index 356f00e..f1bb89c 100644 --- a/lib/http/upload-to-disk.util.ts +++ b/lib/http/upload-to-disk.util.ts @@ -7,11 +7,21 @@ import type { import { randomUUID } from 'node:crypto'; import type { Readable } from 'node:stream'; import { PayloadTooLargeException, UnsupportedMediaTypeException } from '@nestjs/common'; -import { DEFAULT_CONTENT_TYPE, detectContentType, EXTENSION_BY_TYPE, SNIFF_BYTES } from '../utils/content-type.util.js'; +import { + contentTypeEssence, + DEFAULT_CONTENT_TYPE, + detectContentType, + extensionForContentType, + SNIFF_BYTES, +} from '../utils/content-type.util.js'; import { StorageError } from '../errors/storage.error.js'; import type { StorageDisk } from '../disks/storage.disk.js'; import { storageForRequest } from './app-registry.util.js'; +const DEFAULT_CONTENT_TYPE_SAMPLE_BYTES = 4 * 1024; +const MIN_CONTENT_TYPE_SAMPLE_BYTES = 1; +const MAX_CONTENT_TYPE_SAMPLE_BYTES = 64 * 1024; + /** * A storage engine for `FileInterceptor()` and friends that streams each upload straight into a * disk, on Express (multer) and Fastify (`@nestjs/platform-fastify/multipart`) alike: @@ -23,7 +33,17 @@ import { storageForRequest } from './app-registry.util.js'; * })) * ``` * - * - The type is checked on the file's first bytes, before anything is written. + * - The type is checked before anything is written. Signature detection runs first; when it + * cannot identify a type, an optional `detectContentType` callback can classify a bounded + * prefix. A prefix detector does not validate the whole file. + * - For a text format such as CSV, opt in with `detectContentType` and allow its returned type: + * ```ts + * uploadToDisk({ + * contentTypes: ['text/csv'], + * detectContentType: (bytes) => + * bytes.toString('utf8').startsWith('sku,name\n') ? 'text/csv; charset=utf-8' : undefined, + * }); + * ``` * - A file over `limits.fileSize` is refused with a 413, and nothing is written: a truncated * upload never becomes visible, and never replaces the file at its key. * - When the request fails after a file was stored (another file too large, a missing @@ -37,6 +57,26 @@ export function uploadToDisk(options: UploadToDiskOptions = {}): UploadStorageEn if (allowed !== undefined && (!Array.isArray(allowed) || allowed.length === 0)) { throw new TypeError('uploadToDisk(): `contentTypes` must be a non-empty array'); } + const allowedEssences = allowed?.map(contentTypeEssence); + if (options.detectContentType !== undefined && typeof options.detectContentType !== 'function') { + throw new TypeError('uploadToDisk(): `detectContentType` must be a function'); + } + if (options.contentTypeSampleBytes !== undefined && options.detectContentType === undefined) { + throw new TypeError('uploadToDisk(): `contentTypeSampleBytes` requires `detectContentType`'); + } + const contentTypeSampleBytes = + options.contentTypeSampleBytes ?? DEFAULT_CONTENT_TYPE_SAMPLE_BYTES; + if ( + options.contentTypeSampleBytes !== undefined && + (!Number.isSafeInteger(contentTypeSampleBytes) || + contentTypeSampleBytes < MIN_CONTENT_TYPE_SAMPLE_BYTES || + contentTypeSampleBytes > MAX_CONTENT_TYPE_SAMPLE_BYTES) + ) { + throw new TypeError( + 'uploadToDisk(): `contentTypeSampleBytes` must be a whole number between ' + + `${MIN_CONTENT_TYPE_SAMPLE_BYTES} and ${MAX_CONTENT_TYPE_SAMPLE_BYTES}`, + ); + } const diskName = typeof options.disk === 'string' ? options.disk : undefined; const resolveDisk = (req: any): StorageDisk => { @@ -69,28 +109,41 @@ export function uploadToDisk(options: UploadToDiskOptions = {}): UploadStorageEn const head: Buffer[] = []; let headLength = 0; let ended = false; - while (headLength < SNIFF_BYTES) { - const next = await source.next(); - if (next.done) { - ended = true; - break; + const readHead = async (target: number) => { + while (!ended && headLength < target) { + const next = await source.next(); + if (next.done) { + ended = true; + } else { + head.push(next.value); + headLength += next.value.length; + } } - head.push(next.value); - headLength += next.value.length; - } + }; + + const sampleBytes = options.detectContentType + ? Math.max(SNIFF_BYTES, contentTypeSampleBytes) + : SNIFF_BYTES; + await readHead(sampleBytes); + const sample = Buffer.concat(head, Math.min(headLength, sampleBytes)); - const detected = detectContentType(Buffer.concat(head)); - if (allowed && (detected === undefined || !allowed.includes(detected))) { + const declared = { fieldname: file.fieldname, originalname: file.originalname, mimetype: file.mimetype }; + let detected = detectContentType(sample.subarray(0, SNIFF_BYTES)); + if (detected === undefined && options.detectContentType) { + detected = await options.detectContentType(sample.subarray(0, contentTypeSampleBytes), { + ...declared, + contentType: undefined, + extension: '', + }); + if (detected !== undefined && (typeof detected !== 'string' || detected.trim() === '')) { + throw new TypeError('uploadToDisk(): `detectContentType` must return a content type or undefined'); + } + } + if (allowed && (detected === undefined || !allowedEssences!.includes(contentTypeEssence(detected)))) { throw new UnsupportedMediaTypeException(`File type not allowed. Allowed types: ${allowed.join(', ')}`); } - const info: UploadFileInfo = { - fieldname: file.fieldname, - originalname: file.originalname, - mimetype: file.mimetype, - contentType: detected, - extension: detected ? (EXTENSION_BY_TYPE[detected] ?? '') : '', - }; + const info: UploadFileInfo = { ...declared, contentType: detected, extension: extensionForContentType(detected) }; const key = options.key ? await options.key(info, req) : `${randomUUID()}${info.extension}`; const body = async function* () { diff --git a/lib/interfaces/upload-to-disk.interface.ts b/lib/interfaces/upload-to-disk.interface.ts index a15faa1..d6c3502 100644 --- a/lib/interfaces/upload-to-disk.interface.ts +++ b/lib/interfaces/upload-to-disk.interface.ts @@ -7,9 +7,9 @@ export interface UploadFileInfo { originalname: string; /** The client's declared type. Untrusted. */ mimetype: string; - /** The type detected from the first bytes (JPEG, PNG, GIF, WebP, AVIF, HEIC, PDF), if any. */ + /** The type detected from the file bytes by built-in or custom detection, if any. */ contentType: string | undefined; - /** The usual extension for `contentType` (`.jpg`), or `''`. */ + /** The usual extension for `contentType` (`.jpg`), or `''` when it has no known mapping. */ extension: string; } @@ -40,10 +40,32 @@ export interface UploadToDiskOptions { */ key?: (file: UploadFileInfo, req: any) => string | Promise; /** - * Accept only these types, detected from the file's first bytes (see `detectContentType()`). - * Anything else is refused with a 415 before a byte is written. + * Accept only these types, detected from the file's bytes: JPEG, PNG, GIF, WebP, AVIF, HEIC + * and PDF by their signature (see `detectContentType()`), anything else through the + * `detectContentType` option. Types match without their parameters (`text/csv` allows + * `text/csv; charset=utf-8`). Anything else is refused with a 415 before a byte is written. */ contentTypes?: string[]; + /** + * Optional fallback for file types without a recognizable signature. It runs only when the + * built-in `detectContentType()` returns `undefined`, so passing that function here does + * nothing. It receives at most `contentTypeSampleBytes` from + * the start of the file. This classifies a prefix; it does not validate the whole file. + * `file` contains client-provided values such as `originalname` and `mimetype`, which are + * untrusted. Its `contentType` is `undefined` and `extension` is `''` during this callback. + * Return the detected content type, with its charset for text (`text/csv; charset=utf-8`): + * it is the type the file is stored and served with. Return `undefined` when the type is + * unknown. The upload waits for it while holding the request, so it must settle promptly. + */ + detectContentType?: ( + bytes: Buffer, + file: UploadFileInfo, + ) => string | undefined | Promise; + /** + * Maximum prefix size passed to `detectContentType`. Defaults to 4 KiB; must be a whole + * number between 1 byte and 64 KiB. This option requires `detectContentType`. + */ + contentTypeSampleBytes?: number; cacheControl?: string; metadata?: (file: UploadFileInfo, req: any) => Record; } diff --git a/lib/utils/content-type.util.ts b/lib/utils/content-type.util.ts index 2944a92..694c58a 100644 --- a/lib/utils/content-type.util.ts +++ b/lib/utils/content-type.util.ts @@ -67,6 +67,22 @@ export const EXTENSION_BY_TYPE: Record = { 'application/pdf': '.pdf', }; +/** `EXTENSION_BY_TYPE`, plus the first extension `BY_EXTENSION` maps to each other type. */ +const EXTENSION_BY_ESSENCE: Record = { ...EXTENSION_BY_TYPE }; +for (const [extension, type] of Object.entries(BY_EXTENSION)) { + EXTENSION_BY_ESSENCE[contentTypeEssence(type)] ??= `.${extension}`; +} + +/** A content type without its parameters, lowercased: `text/csv; charset=utf-8` is `text/csv`. */ +export function contentTypeEssence(type: string): string { + return type.split(';', 1)[0].trim().toLowerCase(); +} + +/** The usual extension for a content type (`.csv` for `text/csv; charset=utf-8`), or `''`. */ +export function extensionForContentType(type: string | undefined): string { + return type === undefined ? '' : (EXTENSION_BY_ESSENCE[contentTypeEssence(type)] ?? ''); +} + /** How many leading bytes `detectContentType()` needs to recognize every format it knows. */ export const SNIFF_BYTES = 16; diff --git a/tests/upload-engine.spec.ts b/tests/upload-engine.spec.ts index 83993e2..560e48c 100644 --- a/tests/upload-engine.spec.ts +++ b/tests/upload-engine.spec.ts @@ -15,8 +15,8 @@ import { InMemoryDisk, type StoredUpload, type UploadStorageEngine, uploadToDisk const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52]); -function incoming(stream: PassThrough, originalname = 'a.png') { - const file = { fieldname: 'file', originalname, encoding: '7bit', mimetype: 'image/png' }; +function incoming(stream: PassThrough, originalname = 'a.png', mimetype = 'image/png') { + const file = { fieldname: 'file', originalname, encoding: '7bit', mimetype }; Object.defineProperty(file, 'stream', { value: stream, enumerable: false }); return file as typeof file & { stream: PassThrough }; } @@ -48,6 +48,209 @@ describe('uploadToDisk() engine', () => { expect((await disk.getBuffer('k.png')).length).toBe(116); }); + it('uses an async custom detector for unknown types and limits its byte sample', async () => { + const body = Buffer.from('sku,name\n1,widget\n'.padEnd(80, 'x')); + let receivedBytes: Buffer | undefined; + let receivedFile: + | { + contentType: string | undefined; + extension: string; + originalname: string; + mimetype: string; + } + | undefined; + const engine = uploadToDisk({ + disk, + contentTypes: ['text/csv'], + contentTypeSampleBytes: 24, + detectContentType: async (bytes, file) => { + receivedBytes = bytes; + receivedFile = file; + return bytes.toString('utf8').startsWith('sku,name\n') ? 'text/csv' : undefined; + }, + key: (file) => `reports/import${file.extension}`, + }); + const stream = new PassThrough(); + const result = handle(engine, incoming(stream, 'import.csv', 'application/pdf')); + + stream.write(body.subarray(0, 7)); + stream.end(body.subarray(7)); + + const { error, info } = await result; + expect(error).toBeNull(); + expect(receivedBytes).toEqual(body.subarray(0, 24)); + expect(receivedFile).toMatchObject({ + contentType: undefined, + extension: '', + originalname: 'import.csv', + mimetype: 'application/pdf', + }); + expect(info).toMatchObject({ + key: 'reports/import.csv', + size: body.length, + contentType: 'text/csv', + }); + expect(await disk.getBuffer('reports/import.csv')).toEqual(body); + }); + + it('allows a one-byte sample for custom content type detection', async () => { + const body = Buffer.from('not a known signature'); + let receivedBytes: Buffer | undefined; + const engine = uploadToDisk({ + disk, + contentTypes: ['text/csv'], + contentTypeSampleBytes: 1, + key: () => 'detected.csv', + detectContentType: (bytes) => { + receivedBytes = bytes; + return 'text/csv'; + }, + }); + const stream = new PassThrough(); + const result = handle(engine, incoming(stream, 'data.csv')); + stream.end(body); + + const { error, info } = await result; + expect(error).toBeNull(); + expect(receivedBytes).toEqual(body.subarray(0, 1)); + expect(info).toMatchObject({ contentType: 'text/csv', key: 'detected.csv' }); + expect(await disk.getBuffer('detected.csv')).toEqual(body); + }); + + it('keeps signature detection ahead of the custom detector', async () => { + const detector = vi.fn(() => 'text/csv'); + const engine = uploadToDisk({ + disk, + detectContentType: detector, + contentTypes: ['application/pdf'], + }); + const stream = new PassThrough(); + const result = handle(engine, incoming(stream)); + stream.end(Buffer.from('%PDF-1.7\n')); + + const { error, info } = await result; + expect(error).toBeNull(); + expect(detector).not.toHaveBeenCalled(); + expect(info).toMatchObject({ + contentType: 'application/pdf', + key: expect.stringMatching(/\.pdf$/), + }); + }); + + it('does not let a custom detector reclassify a known type outside the allow-list', async () => { + const detector = vi.fn(() => 'text/csv'); + const engine = uploadToDisk({ + disk, + contentTypes: ['text/csv'], + detectContentType: detector, + }); + const stream = new PassThrough(); + const result = handle(engine, incoming(stream, 'data.csv', 'text/csv')); + stream.end(Buffer.from('%PDF-1.7\n')); + + const { error } = await result; + expect(error.getStatus()).toBe(415); + expect(detector).not.toHaveBeenCalled(); + expect(disk.keys()).toEqual([]); + }); + + it('rejects an unknown result against the allow-list without writing', async () => { + const engine = uploadToDisk({ + disk, + contentTypes: ['text/csv'], + detectContentType: () => undefined, + }); + const stream = new PassThrough(); + const result = handle(engine, incoming(stream, 'data.csv')); + stream.end(Buffer.from('not recognized')); + + const { error } = await result; + expect(error.getStatus()).toBe(415); + expect(disk.keys()).toEqual([]); + }); + + it('matches a detected type with parameters against the allow-list and stores it whole', async () => { + const engine = uploadToDisk({ + disk, + contentTypes: ['text/csv'], + detectContentType: () => 'text/csv; charset=utf-8', + }); + const stream = new PassThrough(); + const result = handle(engine, incoming(stream, 'data.csv')); + stream.end(Buffer.from('sku,name\n1,widget\n')); + + const { error, info } = await result; + expect(error).toBeNull(); + expect(info).toMatchObject({ + contentType: 'text/csv; charset=utf-8', + key: expect.stringMatching(/\.csv$/), + }); + }); + + it('stores the custom detector type when there is no allow-list', async () => { + const engine = uploadToDisk({ disk, detectContentType: () => 'application/json' }); + const stream = new PassThrough(); + const result = handle(engine, incoming(stream, 'data.json')); + stream.end(Buffer.from('{"sku":1}')); + + const { error, info } = await result; + expect(error).toBeNull(); + expect(info).toMatchObject({ + contentType: 'application/json', + key: expect.stringMatching(/\.json$/), + }); + expect(await disk.getBuffer(info.key)).toEqual(Buffer.from('{"sku":1}')); + }); + + it('rejects a custom detector type outside the allow-list without writing', async () => { + const engine = uploadToDisk({ + disk, + contentTypes: ['text/csv'], + detectContentType: () => 'text/html', + }); + const stream = new PassThrough(); + const result = handle(engine, incoming(stream, 'data.csv')); + stream.end(Buffer.from('')); + + const { error } = await result; + expect(error.getStatus()).toBe(415); + expect(disk.keys()).toEqual([]); + }); + + it('fails clearly when a custom detector returns an empty type', async () => { + const engine = uploadToDisk({ disk, detectContentType: () => ' ' }); + const stream = new PassThrough(); + const result = handle(engine, incoming(stream, 'data.csv')); + stream.end(Buffer.from('not recognized')); + + const { error } = await result; + expect(error).toBeInstanceOf(TypeError); + expect(error.message).toContain('`detectContentType` must return a content type'); + expect(disk.keys()).toEqual([]); + }); + + it('does not write and drains the stream when a custom detector throws', async () => { + const failure = new Error('detector failed'); + const engine = uploadToDisk({ + disk, + contentTypeSampleBytes: 16, + detectContentType: async () => Promise.reject(failure), + }); + const stream = new PassThrough(); + const result = handle(engine, incoming(stream, 'data.csv')); + stream.write(Buffer.from('unrecognized prefix')); + + const { error } = await result; + expect(error).toBe(failure); + + const closed = new Promise((resolve) => stream.once('close', resolve)); + stream.end(Buffer.alloc(100_000)); + await closed; + expect(stream.readableEnded).toBe(true); + expect(stream.errored).toBeNull(); + expect(disk.keys()).toEqual([]); + }); + it('stores nothing when the client vanishes (the stream closes without ending or an error)', async () => { await disk.put('k.png', 'previous'); const engine = uploadToDisk({ disk, key: () => 'k.png' }); @@ -161,5 +364,17 @@ describe('uploadToDisk() engine', () => { it('validates its options when it is created', () => { expect(() => uploadToDisk({ contentTypes: [] })).toThrow('`contentTypes` must be a non-empty array'); + expect(() => uploadToDisk({ contentTypeSampleBytes: 16 })).toThrow( + 'requires `detectContentType`', + ); + + const detectContentType = () => undefined; + expect(() => uploadToDisk({ detectContentType, contentTypeSampleBytes: 1 })).not.toThrow(); + expect(() => uploadToDisk({ detectContentType, contentTypeSampleBytes: 65_536 })).not.toThrow(); + for (const contentTypeSampleBytes of [0, 1.5, 65_537]) { + expect(() => uploadToDisk({ detectContentType, contentTypeSampleBytes })).toThrow( + 'between 1 and 65536', + ); + } }); }); diff --git a/tests/uploads.e2e.spec.ts b/tests/uploads.e2e.spec.ts index 2bf0712..7d5fa01 100644 --- a/tests/uploads.e2e.spec.ts +++ b/tests/uploads.e2e.spec.ts @@ -54,6 +54,20 @@ function createController(api: UploadApi) { return file; } + @Post('csv-only') + @UseInterceptors( + api.FileInterceptor('file', { + storage: uploadToDisk({ + contentTypes: ['text/csv'], + detectContentType: (bytes) => + bytes.toString('utf8').startsWith('sku,name\n') ? 'text/csv' : undefined, + }), + }), + ) + csvOnly(@UploadedFile() file: StoredUpload) { + return file; + } + @Post('bad-key') @UseInterceptors(api.FileInterceptor('file', { storage: uploadToDisk({ key: (file) => `../${file.originalname}` }) })) badKey() {} @@ -147,6 +161,31 @@ describe.each(adapters.map((a) => a.name))('uploads on %s', (adapter) => { expect(uploads.keys()).toEqual(['only.pdf']); }); + it('uses custom detection for CSV and ignores the client MIME type', async () => { + const csv = Buffer.from('sku,name\n1,widget\n'); + const res = await http() + .post('/csv-only') + .attach('file', csv, { filename: 'data.csv', contentType: 'application/pdf' }) + .expect(201); + + expect(res.body).toMatchObject({ + originalname: 'data.csv', + mimetype: 'application/pdf', + contentType: 'text/csv', + size: csv.length, + key: expect.stringMatching(/^[0-9a-f-]{36}\.csv$/), + }); + expect(await uploads.getBuffer(res.body.key)).toEqual(csv); + expect(await uploads.stat(res.body.key)).toMatchObject({ contentType: 'text/csv' }); + + const before = uploads.keys(); + await http() + .post('/csv-only') + .attach('file', Buffer.from('unknown'), { filename: 'bad.csv' }) + .expect(415); + expect(uploads.keys()).toEqual(before); + }); + it('a key() that builds an invalid key fails the request, and stores nothing', async () => { const res = await http().post('/bad-key').attach('file', PDF, { filename: 'escape.pdf' }); expect(res.status).toBe(500);