[30/33] Thumbnails made outside the request, and bounded

backend/package.json

@@ -18,6 +18,8 @@
   "license": "ISC",
   "dependencies": {
     "@homebridge/node-pty-prebuilt-multiarch": "^0.13.1",
+    "@tus/file-store": "^2.1.0",
+    "@tus/server": "^2.4.1",
     "adm-zip": "^0.5.16",
     "archiver": "^6.0.2",
     "axios": "^1.7.7",
@@ -36,8 +38,8 @@
     "fluent-ffmpeg": "^2.1.2",
     "jsonwebtoken": "^9.0.2",
     "memorystore": "^1.6.7",
-    "p-limit": "^3.1.0",
     "multer": "^2.0.2",
+    "p-limit": "^3.1.0",
     "p-queue": "^7.4.1",
     "pino": "^10.1.0",
     "pino-http": "^11.0.0",

backend/src/app.js

@@ -22,6 +22,7 @@ const { bootstrap } = require('./utils/bootstrap');
 const { configureSession } = require('./middleware/session');
 const logger = require('./utils/logger');
 const { errorHandler, notFoundHandler } = require('./middleware/errorHandler');
+const { uploads } = require('./config');
 
 /**
  * Creates and configures the Express application.
@@ -49,8 +50,10 @@ const createApp = async (options = {}) => {
   configureHttpLogging(app);
 
   configureCors(app);
-  app.use(express.json());
-  app.use(express.urlencoded({ extended: true }));
+  // Large enough to carry back whatever the text editor was allowed to open;
+  // see the reasoning beside the two limits in the configuration.
+  app.use(express.json({ limit: uploads.maxJsonBodyBytes }));
+  app.use(express.urlencoded({ extended: true, limit: uploads.maxJsonBodyBytes }));
   app.use(cookieParser());
   app.use(requestContextMiddleware);
   logger.debug('Mounted cookie parser middleware');

backend/src/config/env.js

@@ -100,6 +100,14 @@ module.exports = {
   ONLYOFFICE_SECRET: process.env.ONLYOFFICE_SECRET || null,
   ONLYOFFICE_LANG: process.env.ONLYOFFICE_LANG?.trim() || 'en',
   ONLYOFFICE_FORCE_SAVE: normalizeBoolean(process.env.ONLYOFFICE_FORCE_SAVE) || false,
+  ONLYOFFICE_FORCE_SAVE_TIMEOUT_MS: Number(process.env.ONLYOFFICE_FORCE_SAVE_TIMEOUT_MS) || 10000,
+  // 0 disables proactive writes to the external storage. A bounded interval
+  // keeps the Document Server's internal autosave from becoming a full document
+  // conversion on every edit.
+  ONLYOFFICE_AUTO_SAVE_INTERVAL_MS: (() => {
+    const value = Number(process.env.ONLYOFFICE_AUTO_SAVE_INTERVAL_MS);
+    return Number.isFinite(value) && value >= 0 ? value : 30000;
+  })(),
   ONLYOFFICE_FILE_EXTENSIONS: process.env.ONLYOFFICE_FILE_EXTENSIONS || '',
   ONLYOFFICE_DOWNLOAD_ORIGINS: process.env.ONLYOFFICE_DOWNLOAD_ORIGINS || '',
 
@@ -139,6 +147,83 @@ module.exports = {
   // Uploads (direct, non-chunked)
   MAX_DIRECT_UPLOAD_SIZE: process.env.MAX_DIRECT_UPLOAD_SIZE?.trim() || null,
   UPLOAD_STORAGE_RESERVE: process.env.UPLOAD_STORAGE_RESERVE?.trim() || '64M',
+  UPLOAD_CHUNK_SIZE: process.env.UPLOAD_CHUNK_SIZE,
+  UPLOAD_CHUNKED_ENABLED: normalizeBoolean(process.env.UPLOAD_CHUNKED_ENABLED),
+  MAX_CHUNK_SIZE_MIB: process.env.MAX_CHUNK_SIZE_MIB,
+  UPLOAD_INACTIVITY_TIMEOUT: process.env.UPLOAD_INACTIVITY_TIMEOUT,
+  THUMBNAILS_ENABLED: normalizeBoolean(process.env.THUMBNAILS_ENABLED) ?? true,
+  THUMBNAIL_CACHE_MAX_FILES:
+    process.env.THUMBNAIL_CACHE_MAX_FILES != null
+      ? Number(process.env.THUMBNAIL_CACHE_MAX_FILES)
+      : 3000,
+  THUMBNAIL_CACHE_CLEANUP_INTERVAL_MS:
+    process.env.THUMBNAIL_CACHE_CLEANUP_INTERVAL_MS != null
+      ? Number(process.env.THUMBNAIL_CACHE_CLEANUP_INTERVAL_MS)
+      : 60 * 60 * 1000,
+  THUMBNAIL_CACHE_CLEANUP_BATCH_SIZE:
+    process.env.THUMBNAIL_CACHE_CLEANUP_BATCH_SIZE != null
+      ? Number(process.env.THUMBNAIL_CACHE_CLEANUP_BATCH_SIZE)
+      : 500,
+  THUMBNAIL_CACHE_TTL_DAYS:
+    process.env.THUMBNAIL_CACHE_TTL_DAYS != null
+      ? Number(process.env.THUMBNAIL_CACHE_TTL_DAYS)
+      : 30,
+  // ExifTool from the machine rather than the one in the archive: 23 MB of
+  // Perl somebody who already has it would rather not carry twice (#9).
+  // Empty means the bundled copy, which is the default and needs nothing.
+  EXIFTOOL_PATH: process.env.EXIFTOOL_PATH || '',
+  // Embedded RAW previews are full-size JPEGs, far larger than a thumbnail, and
+  // a new one is extracted whenever a RAW file changes.
+  RAW_PREVIEW_CACHE_MAX_FILES:
+    process.env.RAW_PREVIEW_CACHE_MAX_FILES != null
+      ? Number(process.env.RAW_PREVIEW_CACHE_MAX_FILES)
+      : 500,
+  THUMBNAIL_SHARP_CACHE_MEMORY_MB:
+    process.env.THUMBNAIL_SHARP_CACHE_MEMORY_MB != null
+      ? Number(process.env.THUMBNAIL_SHARP_CACHE_MEMORY_MB)
+      : 0,
+  THUMBNAIL_VIDEO_CONCURRENCY:
+    process.env.THUMBNAIL_VIDEO_CONCURRENCY != null
+      ? Number(process.env.THUMBNAIL_VIDEO_CONCURRENCY)
+      : 3,
+  THUMBNAIL_VIDEO_SEEK_SECONDS:
+    process.env.THUMBNAIL_VIDEO_SEEK_SECONDS != null
+      ? Number(process.env.THUMBNAIL_VIDEO_SEEK_SECONDS)
+      : 5,
+  THUMBNAIL_VIDEO_SEEK_PERCENT:
+    process.env.THUMBNAIL_VIDEO_SEEK_PERCENT != null &&
+    process.env.THUMBNAIL_VIDEO_SEEK_PERCENT.trim() !== ''
+      ? Number(process.env.THUMBNAIL_VIDEO_SEEK_PERCENT)
+      : null,
+  THUMBNAIL_VIDEO_THREADS:
+    process.env.THUMBNAIL_VIDEO_THREADS != null ? Number(process.env.THUMBNAIL_VIDEO_THREADS) : 2,
+  THUMBNAIL_VIDEO_SCALE_FLAGS: process.env.THUMBNAIL_VIDEO_SCALE_FLAGS?.trim() || 'fast_bilinear',
+  THUMBNAIL_BACKGROUND_QUEUE_LIMIT:
+    process.env.THUMBNAIL_BACKGROUND_QUEUE_LIMIT != null
+      ? Number(process.env.THUMBNAIL_BACKGROUND_QUEUE_LIMIT)
+      : 16,
+  THUMBNAIL_DIAGNOSTICS_ENABLED:
+    normalizeBoolean(process.env.THUMBNAIL_DIAGNOSTICS_ENABLED) ?? false,
+  THUMBNAIL_DIAGNOSTICS_INTERVAL_MS:
+    process.env.THUMBNAIL_DIAGNOSTICS_INTERVAL_MS != null
+      ? Number(process.env.THUMBNAIL_DIAGNOSTICS_INTERVAL_MS)
+      : 30000,
+  THUMBNAIL_SLOW_JOB_MS:
+    process.env.THUMBNAIL_SLOW_JOB_MS != null ? Number(process.env.THUMBNAIL_SLOW_JOB_MS) : 10000,
+  // How long one ffmpeg may take over a single thumbnail before it is killed.
+  // Not a deadline anything waits on — the queue has given up long before —
+  // but the only thing that ends a process that has stopped making progress.
+  THUMBNAIL_FFMPEG_TIMEOUT_MS:
+    process.env.THUMBNAIL_FFMPEG_TIMEOUT_MS != null
+      ? Number(process.env.THUMBNAIL_FFMPEG_TIMEOUT_MS)
+      : 5 * 60 * 1000,
+  // Niceness applied to child ffmpeg/convert processes (0 = disabled, 1-19 lowers
+  // their CPU priority so the Node event loop stays responsive during generation).
+  THUMBNAIL_PROCESS_NICE:
+    process.env.THUMBNAIL_PROCESS_NICE != null ? Number(process.env.THUMBNAIL_PROCESS_NICE) : 10,
+  TUS_UPLOAD_DIR: process.env.TUS_UPLOAD_DIR?.trim() || null,
+  TUS_INCOMPLETE_UPLOAD_TTL_MS: process.env.TUS_INCOMPLETE_UPLOAD_TTL_MS,
+  TUS_CLEANUP_INTERVAL_MS: process.env.TUS_CLEANUP_INTERVAL_MS,
   MAX_FILES_PER_UPLOAD: Number(process.env.MAX_FILES_PER_UPLOAD) || 50,
 
   // Editor

backend/src/config/index.js

@@ -4,6 +4,8 @@ const env = require('./env');
 const constants = require('./constants');
 const loggingConfig = require('./logging');
 const { parseByteSize } = require('../utils/env');
+// logger reads config/logging, never this file — requiring it here makes no cycle.
+const logger = require('../utils/logger');
 
 const parseCommaOrSpaceList = (raw) => {
   if (!raw) return [];
@@ -277,16 +279,110 @@ const searchMaxFileSizeBytes = (() => {
 })();
 
 // --- Uploads ---
+// --- Editor ---
+/**
+ * What the inline text editor opens, and what a JSON request body may weigh.
+ *
+ * They are one decision rather than two. The editor sends a file back through a
+ * JSON body when it saves it, so a body limit under the size the editor opens
+ * produces a file that opens and cannot be saved — answered with "request
+ * entity too large", which names neither setting. Express's own default is
+ * 100 kB, against an editor that opens two megabytes.
+ *
+ * Escaping is why the body has to be worth more than the file: in the worst
+ * case every character of the content is a quote, a backslash or a newline and
+ * becomes two, and the path travels in the same body. A file whose bytes would
+ * expand further than that is one the editor refuses to open anyway, as binary.
+ */
+const JSON_ESCAPE_WORST_CASE = 2;
+const JSON_BODY_OVERHEAD_BYTES = 64 * 1024;
+const DEFAULT_JSON_BODY_BYTES = 8 * 1024 * 1024;
+
+const bodyNeededFor = (fileBytes) => fileBytes * JSON_ESCAPE_WORST_CASE + JSON_BODY_OVERHEAD_BYTES;
+const fileAllowedBy = (bodyBytes) =>
+  Math.max(0, Math.floor((bodyBytes - JSON_BODY_OVERHEAD_BYTES) / JSON_ESCAPE_WORST_CASE));
+
+const { editorMaxFileSizeBytes, maxJsonBodyBytes } = (() => {
+  const parsedEditor = parseByteSize(env.EDITOR_MAX_FILESIZE);
+  // Default: 2 MiB if not configured or invalid
+  const editorAsked =
+    Number.isFinite(parsedEditor) && parsedEditor > 0 ? parsedEditor : 2 * 1024 * 1024;
+
+  const parsedBody = parseByteSize(env.MAX_JSON_BODY_SIZE);
+  const bodyWasChosen = Number.isFinite(parsedBody) && parsedBody > 0;
+
+  // A body limit someone set is a ceiling they meant — it is a guard, not a
+  // detail — so it is never raised from here. The editor is what gives way, and
+  // it gives way by refusing to open what it could not save back.
+  if (bodyWasChosen) {
+    const allowed = fileAllowedBy(parsedBody);
+    if (editorAsked > allowed) {
+      logger.warn(
+        { editorAsked, loweredTo: allowed, maxJsonBodyBytes: parsedBody },
+        'EDITOR_MAX_FILESIZE is larger than MAX_JSON_BODY_SIZE can carry back and has been ' +
+          'lowered to match; the editor would otherwise open files it could not save'
+      );
+    }
+    return { editorMaxFileSizeBytes: Math.min(editorAsked, allowed), maxJsonBodyBytes: parsedBody };
+  }
+
+  // Nobody chose the body limit, so the editor's size is the only wish there is
+  // to honour: the default body limit rises to carry it.
+  const needed = bodyNeededFor(editorAsked);
+  if (needed > DEFAULT_JSON_BODY_BYTES) {
+    logger.info(
+      { editorMaxFileSizeBytes: editorAsked, maxJsonBodyBytes: needed },
+      'Raised the JSON body limit above its default so the text editor can save what it opens'
+    );
+  }
+
+  return {
+    editorMaxFileSizeBytes: editorAsked,
+    maxJsonBodyBytes: Math.max(DEFAULT_JSON_BODY_BYTES, needed),
+  };
+})();
+
 // Ceilings for direct (non-chunked) uploads. They exist so a single request
 // cannot stream until the disk is full; they are generous on purpose, since
 // large files are a normal use of a file manager. Chunked uploads have their
 // own storage guard in the TUS service.
+// How long a direct upload may go without a byte arriving before it is given
+// up on. A client that goes away mid-body otherwise holds the request, and the
+// half-written file with it, until the socket itself times out.
+const uploadInactivityTimeoutMs = (() => {
+  const value = Number(env.UPLOAD_INACTIVITY_TIMEOUT);
+  return Number.isFinite(value) && value >= 0 ? value : 120000;
+})();
+
+// Where a chunked upload's parts live until the whole file is there, and how
+// long an unfinished one is kept. Under the cache rather than beside the
+// destination: a part file is not a file anybody asked for, and a volume should
+// never show one.
+const tusUploadDir = env.TUS_UPLOAD_DIR
+  ? path.resolve(env.TUS_UPLOAD_DIR)
+  : path.join(cacheDir, 'tus-uploads');
+
+const tusIncompleteUploadTtlMs = (() => {
+  const value = Number(env.TUS_INCOMPLETE_UPLOAD_TTL_MS);
+  return Number.isFinite(value) && value >= 0 ? Math.floor(value) : 60 * 60 * 1000;
+})();
+
+const tusCleanupIntervalMs = (() => {
+  const value = Number(env.TUS_CLEANUP_INTERVAL_MS);
+  return Number.isFinite(value) && value >= 0 ? Math.floor(value) : 10 * 60 * 1000;
+})();
+
 const uploads = {
+  maxJsonBodyBytes,
   maxDirectUploadBytes: (() => {
     const parsed = parseByteSize(env.MAX_DIRECT_UPLOAD_SIZE);
     return Number.isFinite(parsed) && parsed > 0 ? parsed : 64 * 1024 * 1024 * 1024;
   })(),
   maxFilesPerRequest: env.MAX_FILES_PER_UPLOAD,
+  inactivityTimeoutMs: uploadInactivityTimeoutMs,
+  tusUploadDir,
+  tusIncompleteUploadTtlMs,
+  tusCleanupIntervalMs,
   // Free space kept in reserve when accepting writes, so a full volume never
   // takes the database down with it. The trash gives space back before this
   // floor is crossed.
@@ -320,6 +416,8 @@ const onlyoffice = {
   secret: env.ONLYOFFICE_SECRET || deriveSecret('onlyoffice'),
   lang: env.ONLYOFFICE_LANG,
   forceSave: env.ONLYOFFICE_FORCE_SAVE,
+  forceSaveTimeoutMs: Math.min(30000, Math.max(7000, env.ONLYOFFICE_FORCE_SAVE_TIMEOUT_MS)),
+  autoSaveIntervalMs: Math.min(300000, Math.max(0, env.ONLYOFFICE_AUTO_SAVE_INTERVAL_MS)),
   extensions: env.ONLYOFFICE_FILE_EXTENSIONS.split(',')
     .map((s) => s.trim().toLowerCase())
     .filter(Boolean),
@@ -365,13 +463,6 @@ const collabora = {
     .filter(Boolean),
 };
 
-// --- Editor ---
-const editorMaxFileSizeBytes = (() => {
-  const parsed = parseByteSize(env.EDITOR_MAX_FILESIZE);
-  // Default: 2 MiB if not configured or invalid
-  return Number.isFinite(parsed) && parsed > 0 ? parsed : 2 * 1024 * 1024;
-})();
-
 const editor = {
   extensions: parseExtensionList(env.EDITOR_EXTENSIONS),
   maxFileSizeBytes: editorMaxFileSizeBytes,

backend/src/middleware/multipartRefusals.js

@@ -0,0 +1,36 @@
+const multer = require('multer');
+
+/**
+ * A size as a person reads it: "2 MB", "64 GB".
+ */
+const describeBytes = (bytes) => {
+  const units = ['bytes', 'KB', 'MB', 'GB', 'TB'];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit += 1;
+  }
+  return `${Math.round(value * 10) / 10} ${units[unit]}`;
+};
+
+/**
+ * Run a multer middleware whose refusals say what this route's limits are.
+ *
+ * multer names the limit a request met and not its value — "File too large" —
+ * and only the route knows the value, and the setting that governs it. The
+ * sentence for a code is attached here; the status comes from the error
+ * handler, which knows every code whichever route raised it.
+ *
+ * @param {import('express').RequestHandler} middleware what `upload.single()` or `.fields()` returned
+ * @param {Record<string, string>} sentences what to tell the client, by multer error code
+ */
+const explainMultipartRefusals = (middleware, sentences) => (req, res, next) =>
+  middleware(req, res, (error) => {
+    if (error instanceof multer.MulterError && sentences[error.code]) {
+      error.clientMessage = sentences[error.code];
+    }
+    next(error);
+  });
+
+module.exports = { describeBytes, explainMultipartRefusals };

backend/src/middleware/responseEndCompat.js

@@ -0,0 +1,64 @@
+/**
+ * @tus/server hands its responses to srvx, which finishes them with
+ * `res.end(callback)`. Node accepts that form — a function in first position is
+ * the completion callback, not a body — but express-session replaces res.end
+ * with a two-argument `(chunk, encoding)` wrapper that has no notion of it. On
+ * any request where the session has to be saved or touched, that wrapper writes
+ * the body out before ending, and the callback reaches res.write() as a chunk:
+ *
+ *   TypeError [ERR_INVALID_ARG_TYPE]: The "chunk" argument must be of type
+ *   string or an instance of Buffer or Uint8Array. Received function
+ *
+ * Nothing catches it, so the process exits. Every chunked upload by a
+ * signed-in user took the server down with it: a 502 in the browser, and on a
+ * deployment whose storage is not persistent, a database recreated empty on the
+ * restart — favourites, shares and preferences gone with it.
+ *
+ * The store implements touch and `resave` is false, so this is the ordinary
+ * path for an established session rather than a rare one.
+ *
+ * Mounted after the session middleware, this wrapper is the one srvx reaches
+ * first: it moves the callback onto the response's own completion event and
+ * passes the plain `(chunk, encoding)` form down the chain, which is all
+ * express-session ever expects to see.
+ */
+const responseEndCompat = (req, res, next) => {
+  const end = res.end;
+
+  res.end = function normalizedEnd(chunk, encoding, callback) {
+    if (typeof chunk === 'function') {
+      callback = chunk;
+      chunk = undefined;
+      encoding = undefined;
+    } else if (typeof encoding === 'function') {
+      callback = encoding;
+      encoding = undefined;
+    }
+
+    if (typeof callback === 'function') {
+      let settled = false;
+      const settle = () => {
+        if (settled) return;
+        settled = true;
+        callback();
+      };
+
+      // A response that has already finished emits nothing further, and the
+      // caller would wait for ever on a reply that has gone.
+      if (res.writableEnded) {
+        setImmediate(settle);
+      } else {
+        // 'close' as well as 'finish': a client that walks away mid-upload
+        // still has to release whoever is awaiting the response.
+        res.once('finish', settle);
+        res.once('close', settle);
+      }
+    }
+
+    return end.call(this, chunk, encoding);
+  };
+
+  next();
+};
+
+module.exports = { responseEndCompat };