feat(storage): support custom upload content-type detection

lib/http/upload-to-disk.util.ts

@@ -7,11 +7,21 @@ import type {
 import { randomUUID } from 'node:crypto';
 import type { Readable } from 'node:stream';
 import { PayloadTooLargeException, UnsupportedMediaTypeException } from '@nestjs/common';
-import { DEFAULT_CONTENT_TYPE, detectContentType, EXTENSION_BY_TYPE, SNIFF_BYTES } from '../utils/content-type.util.js';
+import {
+  contentTypeEssence,
+  DEFAULT_CONTENT_TYPE,
+  detectContentType,
+  extensionForContentType,
+  SNIFF_BYTES,
+} from '../utils/content-type.util.js';
 import { StorageError } from '../errors/storage.error.js';
 import type { StorageDisk } from '../disks/storage.disk.js';
 import { storageForRequest } from './app-registry.util.js';
 
+const DEFAULT_CONTENT_TYPE_SAMPLE_BYTES = 4 * 1024;
+const MIN_CONTENT_TYPE_SAMPLE_BYTES = 1;
+const MAX_CONTENT_TYPE_SAMPLE_BYTES = 64 * 1024;
+
 /**
  * A storage engine for `FileInterceptor()` and friends that streams each upload straight into a
  * disk, on Express (multer) and Fastify (`@nestjs/platform-fastify/multipart`) alike:
@@ -23,7 +33,17 @@ import { storageForRequest } from './app-registry.util.js';
  * }))
  * ```
  *
- * - The type is checked on the file's first bytes, before anything is written.
+ * - The type is checked before anything is written. Signature detection runs first; when it
+ *   cannot identify a type, an optional `detectContentType` callback can classify a bounded
+ *   prefix. A prefix detector does not validate the whole file.
+ * - For a text format such as CSV, opt in with `detectContentType` and allow its returned type:
+ *   ```ts
+ *   uploadToDisk({
+ *     contentTypes: ['text/csv'],
+ *     detectContentType: (bytes) =>
+ *       bytes.toString('utf8').startsWith('sku,name\n') ? 'text/csv; charset=utf-8' : undefined,
+ *   });
+ *   ```
  * - A file over `limits.fileSize` is refused with a 413, and nothing is written: a truncated
  *   upload never becomes visible, and never replaces the file at its key.
  * - When the request fails after a file was stored (another file too large, a missing
@@ -37,6 +57,26 @@ export function uploadToDisk(options: UploadToDiskOptions = {}): UploadStorageEn
   if (allowed !== undefined && (!Array.isArray(allowed) || allowed.length === 0)) {
     throw new TypeError('uploadToDisk(): `contentTypes` must be a non-empty array');
   }
+  const allowedEssences = allowed?.map(contentTypeEssence);
+  if (options.detectContentType !== undefined && typeof options.detectContentType !== 'function') {
+    throw new TypeError('uploadToDisk(): `detectContentType` must be a function');
+  }
+  if (options.contentTypeSampleBytes !== undefined && options.detectContentType === undefined) {
+    throw new TypeError('uploadToDisk(): `contentTypeSampleBytes` requires `detectContentType`');
+  }
+  const contentTypeSampleBytes =
+    options.contentTypeSampleBytes ?? DEFAULT_CONTENT_TYPE_SAMPLE_BYTES;
+  if (
+    options.contentTypeSampleBytes !== undefined &&
+    (!Number.isSafeInteger(contentTypeSampleBytes) ||
+      contentTypeSampleBytes < MIN_CONTENT_TYPE_SAMPLE_BYTES ||
+      contentTypeSampleBytes > MAX_CONTENT_TYPE_SAMPLE_BYTES)
+  ) {
+    throw new TypeError(
+      'uploadToDisk(): `contentTypeSampleBytes` must be a whole number between ' +
+        `${MIN_CONTENT_TYPE_SAMPLE_BYTES} and ${MAX_CONTENT_TYPE_SAMPLE_BYTES}`,
+    );
+  }
 
   const diskName = typeof options.disk === 'string' ? options.disk : undefined;
   const resolveDisk = (req: any): StorageDisk => {
@@ -69,28 +109,41 @@ export function uploadToDisk(options: UploadToDiskOptions = {}): UploadStorageEn
         const head: Buffer[] = [];
         let headLength = 0;
         let ended = false;
-        while (headLength < SNIFF_BYTES) {
-          const next = await source.next();
-          if (next.done) {
-            ended = true;
-            break;
+        const readHead = async (target: number) => {
+          while (!ended && headLength < target) {
+            const next = await source.next();
+            if (next.done) {
+              ended = true;
+            } else {
+              head.push(next.value);
+              headLength += next.value.length;
+            }
           }
-          head.push(next.value);
-          headLength += next.value.length;
-        }
+        };
+
+        const sampleBytes = options.detectContentType
+          ? Math.max(SNIFF_BYTES, contentTypeSampleBytes)
+          : SNIFF_BYTES;
+        await readHead(sampleBytes);
+        const sample = Buffer.concat(head, Math.min(headLength, sampleBytes));
 
-        const detected = detectContentType(Buffer.concat(head));
-        if (allowed && (detected === undefined || !allowed.includes(detected))) {
+        const declared = { fieldname: file.fieldname, originalname: file.originalname, mimetype: file.mimetype };
+        let detected = detectContentType(sample.subarray(0, SNIFF_BYTES));
+        if (detected === undefined && options.detectContentType) {
+          detected = await options.detectContentType(sample.subarray(0, contentTypeSampleBytes), {
+            ...declared,
+            contentType: undefined,
+            extension: '',
+          });
+          if (detected !== undefined && (typeof detected !== 'string' || detected.trim() === '')) {
+            throw new TypeError('uploadToDisk(): `detectContentType` must return a content type or undefined');
+          }
+        }
+        if (allowed && (detected === undefined || !allowedEssences!.includes(contentTypeEssence(detected)))) {
           throw new UnsupportedMediaTypeException(`File type not allowed. Allowed types: ${allowed.join(', ')}`);
         }
 
-        const info: UploadFileInfo = {
-          fieldname: file.fieldname,
-          originalname: file.originalname,
-          mimetype: file.mimetype,
-          contentType: detected,
-          extension: detected ? (EXTENSION_BY_TYPE[detected] ?? '') : '',
-        };
+        const info: UploadFileInfo = { ...declared, contentType: detected, extension: extensionForContentType(detected) };
         const key = options.key ? await options.key(info, req) : `${randomUUID()}${info.extension}`;
 
         const body = async function* () {

lib/interfaces/upload-to-disk.interface.ts

@@ -7,9 +7,9 @@ export interface UploadFileInfo {
   originalname: string;
   /** The client's declared type. Untrusted. */
   mimetype: string;
-  /** The type detected from the first bytes (JPEG, PNG, GIF, WebP, AVIF, HEIC, PDF), if any. */
+  /** The type detected from the file bytes by built-in or custom detection, if any. */
   contentType: string | undefined;
-  /** The usual extension for `contentType` (`.jpg`), or `''`. */
+  /** The usual extension for `contentType` (`.jpg`), or `''` when it has no known mapping. */
   extension: string;
 }
 
@@ -40,10 +40,32 @@ export interface UploadToDiskOptions {
    */
   key?: (file: UploadFileInfo, req: any) => string | Promise<string>;
   /**
-   * Accept only these types, detected from the file's first bytes (see `detectContentType()`).
-   * Anything else is refused with a 415 before a byte is written.
+   * Accept only these types, detected from the file's bytes: JPEG, PNG, GIF, WebP, AVIF, HEIC
+   * and PDF by their signature (see `detectContentType()`), anything else through the
+   * `detectContentType` option. Types match without their parameters (`text/csv` allows
+   * `text/csv; charset=utf-8`). Anything else is refused with a 415 before a byte is written.
    */
   contentTypes?: string[];
+  /**
+   * Optional fallback for file types without a recognizable signature. It runs only when the
+   * built-in `detectContentType()` returns `undefined`, so passing that function here does
+   * nothing. It receives at most `contentTypeSampleBytes` from
+   * the start of the file. This classifies a prefix; it does not validate the whole file.
+   * `file` contains client-provided values such as `originalname` and `mimetype`, which are
+   * untrusted. Its `contentType` is `undefined` and `extension` is `''` during this callback.
+   * Return the detected content type, with its charset for text (`text/csv; charset=utf-8`):
+   * it is the type the file is stored and served with. Return `undefined` when the type is
+   * unknown. The upload waits for it while holding the request, so it must settle promptly.
+   */
+  detectContentType?: (
+    bytes: Buffer,
+    file: UploadFileInfo,
+  ) => string | undefined | Promise<string | undefined>;
+  /**
+   * Maximum prefix size passed to `detectContentType`. Defaults to 4 KiB; must be a whole
+   * number between 1 byte and 64 KiB. This option requires `detectContentType`.
+   */
+  contentTypeSampleBytes?: number;
   cacheControl?: string;
   metadata?: (file: UploadFileInfo, req: any) => Record<string, string>;
 }

lib/utils/content-type.util.ts

@@ -67,6 +67,22 @@ export const EXTENSION_BY_TYPE: Record<string, string> = {
   'application/pdf': '.pdf',
 };
 
+/** `EXTENSION_BY_TYPE`, plus the first extension `BY_EXTENSION` maps to each other type. */
+const EXTENSION_BY_ESSENCE: Record<string, string> = { ...EXTENSION_BY_TYPE };
+for (const [extension, type] of Object.entries(BY_EXTENSION)) {
+  EXTENSION_BY_ESSENCE[contentTypeEssence(type)] ??= `.${extension}`;
+}
+
+/** A content type without its parameters, lowercased: `text/csv; charset=utf-8` is `text/csv`. */
+export function contentTypeEssence(type: string): string {
+  return type.split(';', 1)[0].trim().toLowerCase();
+}
+
+/** The usual extension for a content type (`.csv` for `text/csv; charset=utf-8`), or `''`. */
+export function extensionForContentType(type: string | undefined): string {
+  return type === undefined ? '' : (EXTENSION_BY_ESSENCE[contentTypeEssence(type)] ?? '');
+}
+
 /** How many leading bytes `detectContentType()` needs to recognize every format it knows. */
 export const SNIFF_BYTES = 16;