diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index 46c91bf9a..000000000 --- a/.eslintignore +++ /dev/null @@ -1,8 +0,0 @@ -node_modules -dist -dist-ssr -storybook-static -coverage -.vitepress/cache -.vitepress/dist -package-lock.json diff --git a/.eslintrc.cjs b/.eslintrc.cjs deleted file mode 100644 index e01d921a7..000000000 --- a/.eslintrc.cjs +++ /dev/null @@ -1,97 +0,0 @@ -/* eslint-env node */ -module.exports = { - root: true, - env: { - es2022: true, - }, - extends: ['eslint:recommended', 'prettier'], - ignorePatterns: [ - 'node_modules/', - 'coverage/', - 'dist/', - 'dist-ssr/', - 'storybook-static/', - '.vitepress/cache/', - '.vitepress/dist/', - ], - parserOptions: { - ecmaVersion: 'latest', - }, - overrides: [ - { - // Helper scripts shipped with the image run under plain Node. - files: ['docker/**/*.js', 'scripts/**/*.js', '*.cjs', '**/*.config.cjs'], - env: { - node: true, - }, - }, - { - // Deleting someone's files goes through the trash. Removing from disk - // directly is allowed only in the files below, which remove what the - // application itself created (temporary uploads, caches, extraction - // staging) or implement the permanent deletion the trash hands back to. - // A new file that deletes content must go through services/trash, or be - // added here with the reason it does not. - files: ['backend/src/**/*.js'], - excludedFiles: [ - 'backend/src/services/trash/**', - 'backend/src/services/versions/**', - 'backend/src/services/fileTransferService.js', - // Its own in-flight journal, and the placement/undo utilities that - // remove only empty placeholders and what an operation itself wrote. - 'backend/src/services/inFlightFiles.js', - 'backend/src/utils/placeWithoutOverwrite.js', - 'backend/src/utils/ownedTree.js', - 'backend/src/services/archiveService.js', - 'backend/src/services/rawPreviewService.js', - 'backend/src/services/thumbnailService.js', - 'backend/src/services/tusUploadService.js', - 'backend/src/services/uploadRemnants.js', - 'backend/src/services/uploadService.js', - 'backend/src/routes/onlyoffice.js', - // Its own logos, under the config directory: named by the content they - // hold, removed only once nothing points at one, and made again by - // uploading it. The settings route no longer removes them itself. - 'backend/src/services/brandingLogo.js', - 'backend/src/routes/zip.js', - 'backend/src/scripts/**', - ], - rules: { - 'no-restricted-properties': [ - 'error', - ...['fs', 'fsp', 'fss', 'fsSync', 'fsPromises', 'promises'].flatMap((object) => - ['rm', 'rmSync', 'unlink', 'unlinkSync', 'rmdir', 'rmdirSync'].map((property) => ({ - object, - property, - message: 'Deleting from disk goes through services/trash (see .eslintrc.cjs).', - })) - ), - ], - 'no-restricted-syntax': [ - 'error', - { - selector: - "CallExpression[callee.name='spawn'][arguments.0.value='rm'], CallExpression[callee.property.name='spawn'][arguments.0.value='rm']", - message: 'Deleting from disk goes through services/trash (see .eslintrc.cjs).', - }, - { - selector: - "VariableDeclarator[init.callee.name='require'][init.arguments.0.value=/^(node:)?fs(\\u002Fpromises)?$/] > ObjectPattern > Property[key.name=/^(rm|rmSync|unlink|unlinkSync|rmdir|rmdirSync)$/]", - message: 'Deleting from disk goes through services/trash (see .eslintrc.cjs).', - }, - ], - }, - }, - { - // Les outils de verification sont des modules ES: sans sourceType, eslint - // lit leur premier import comme une erreur de syntaxe. - files: ['scripts/**/*.mjs'], - env: { - node: true, - }, - parserOptions: { - sourceType: 'module', - }, - }, - ], -}; diff --git a/Dockerfile b/Dockerfile index 0b7ac9ca7..953bc4eae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,22 @@ +# syntax=docker/dockerfile:1 + +# Declared before the first stage so it can choose one. Repeated inside the +# runtime stage, where a value is needed in a RUN — a global ARG is visible to +# FROM lines and to nothing else. +ARG FFMPEG_VARIANT=apk + +# The ffmpeg both images carry, pinned once. The lean image compiles it (the +# ffmpeg_build stage); the full one builds Alpine's package at this version +# whenever Alpine's own is older (the ffmpeg_recipe stage). Each stage that +# needs them repeats the two names without a value, which is what brings a +# global ARG into a stage. +ARG FFMPEG_VERSION=8.1.3 +ARG FFMPEG_SHA256=7138d28c96d9d3e3af4ee3d8cad72741f8ffb40da90c1112235dea3ecd3178a3 + # --------------------------------------------------------------------------- # Base: Alpine with Node.js # --------------------------------------------------------------------------- -FROM public.ecr.aws/docker/library/node:24-alpine AS base +FROM public.ecr.aws/docker/library/node:24.21-alpine3.24 AS base WORKDIR /app # --------------------------------------------------------------------------- @@ -38,10 +53,216 @@ COPY frontend/ ./frontend/ RUN npm run -w frontend build -- --sourcemap false # --------------------------------------------------------------------------- -# Stage 3: Final runtime image — no compilers, no build tools +# Stage 3: Official static 7-Zip +# +# Alpine's p7zip build does not include the RAR codec. Use the official, +# architecture-specific static binary instead so zip, 7z and RAR extraction +# have the same capabilities in the full and lean images. +# +# Taking the binary out of apk's hands means its security updates are this +# pin's job, and the archive handlers are the part of the image a visitor +# reaches most directly: every extension in DEFAULT_ARCHIVE_EXTENSIONS is a +# parser fed bytes someone uploaded. 26.02 fixed a heap overflow in the XZ +# decoder (CVE-2026-14266, remote code execution), and `xz`/`txz` are in that +# list — so this version is not a detail to leave where it was. +# --------------------------------------------------------------------------- +FROM alpine:3.24 AS seven_zip +ARG TARGETARCH +ARG SEVEN_ZIP_VERSION=26.03 + +RUN apk add --no-cache curl libarchive-tools \ + && case "$TARGETARCH" in \ + amd64) archive_arch=x64; archive_sha256=dc99eff5008f1ab79bd7084c68513701547a808a89502bf4133683535ab3c695 ;; \ + arm64) archive_arch=arm64; archive_sha256=2389ba20e4d8295e8709c20b6263b69bd1ec4972fe38a04ad7a1badbf595b996 ;; \ + *) echo "Unsupported 7-Zip architecture: $TARGETARCH" >&2; exit 1 ;; \ + esac \ + && archive_version=$(printf '%s' "$SEVEN_ZIP_VERSION" | tr -d .) \ + && curl -fsSL -o /tmp/7z.tar.xz "https://github.com/ip7z/7zip/releases/download/${SEVEN_ZIP_VERSION}/7z${archive_version}-linux-${archive_arch}.tar.xz" \ + && echo "${archive_sha256} /tmp/7z.tar.xz" | sha256sum -c - \ + && mkdir -p /out /tmp/7z \ + && bsdtar -xJf /tmp/7z.tar.xz -C /tmp/7z \ + && install -m 0755 "$(find /tmp/7z -type f -name 7zzs -print -quit)" /out/7z + # --------------------------------------------------------------------------- +# ffmpeg, built here rather than taken from anywhere. +# +# Alpine's `ffmpeg` package is a full build: 106 MB of codec libraries behind a +# 0.6 MB binary, and almost all of that weight is *encoding* — x264, x265, aom, +# vpx, lame, opus, theora, ass. This application only ever decodes: one frame +# for a video thumbnail, one still out of a HEIC, and ffprobe for metadata. +# +# So the shape of this build is deliberately the opposite of the obvious one. +# `--disable-everything` would be smaller still and is the wrong tool: it is +# opt-in, and the way it fails is a format quietly losing its previews with +# nothing logged anywhere. Instead every native decoder, demuxer and parser is +# kept — they are small — and only the encoders and muxers are cut back to the +# two we write. Nothing this application can open stops being openable. +# +# `--disable-autodetect` means the build cannot pick up a library by accident, +# so what is linked is exactly what is listed. libdav1d is the one external +# decoder worth having: ffmpeg's native AV1 decoder works but is much slower. +# No `--enable-gpl`, because nothing here needs a GPL component once the +# encoders are gone — the result is LGPL. +# +# The checksum is pinned the way the 7-Zip download above is. ffmpeg.org also +# publishes a detached signature (ffmpeg-.tar.xz.asc) for anyone who +# wants to go further than pinning the bytes. +# --------------------------------------------------------------------------- +FROM alpine:3.24 AS ffmpeg_build +ARG FFMPEG_VERSION +ARG FFMPEG_SHA256 + +# `ffmpeg` here is a build dependency and never ships: the verification below +# uses it to synthesise a clip per format, which the binary we build then has +# to decode. +# No `-static` variants: this links against Alpine's shared libraries, which +# keeps the binary small and leaves security updates to apk rather than to a +# rebuild. `dav1d-static` does not exist in Alpine 3.23 in any case. +RUN apk add --no-cache \ + build-base coreutils curl xz pkgconf nasm yasm \ + zlib-dev bzip2-dev dav1d-dev \ + ffmpeg + +RUN curl -fsSL -o /tmp/ffmpeg.tar.xz \ + "https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz" \ + && echo "${FFMPEG_SHA256} /tmp/ffmpeg.tar.xz" | sha256sum -c - \ + && mkdir -p /tmp/ffmpeg-src \ + && tar -xJf /tmp/ffmpeg.tar.xz -C /tmp/ffmpeg-src --strip-components=1 + +WORKDIR /tmp/ffmpeg-src +RUN ./configure \ + --prefix=/out \ + --disable-autodetect \ + --disable-doc \ + --disable-debug \ + --disable-network \ + --disable-ffplay \ + --enable-zlib \ + --enable-bzlib \ + --enable-libdav1d \ + --disable-encoders \ + --enable-encoder=mjpeg \ + --enable-encoder=png \ + --enable-encoder=webvtt \ + --disable-muxers \ + --enable-muxer=image2 \ + --enable-muxer=image2pipe \ + --enable-muxer=rawvideo \ + --enable-muxer=webvtt \ + --enable-small \ + && make -j"$(nproc)" \ + && make install + +# Fails the build if any format the explorer offers previews for cannot be +# decoded. See docker/verify-ffmpeg.sh for what that means and why. +COPY docker/verify-ffmpeg.sh /usr/local/bin/verify-ffmpeg.sh +RUN chmod +x /usr/local/bin/verify-ffmpeg.sh \ + && /usr/local/bin/verify-ffmpeg.sh /out/bin/ffmpeg /out/bin/ffprobe \ + && strip /out/bin/ffmpeg /out/bin/ffprobe \ + && ls -la /out/bin + + +# --------------------------------------------------------------------------- +# ffmpeg for the full image: Alpine's own build, at the version pinned above +# +# The full image takes ffmpeg from Alpine for everything that build carries — +# VA-API, VDPAU, Vulkan, QSV on amd64, and every external decoder — and Alpine +# can be days or weeks behind a security release. 8.1.3 closed three CVEs in +# decoders this image hands uploads to (CVE-2026-66038, CVE-2026-70629, +# CVE-2026-70631) on 21 September 2026, and the 3.24 branch was still on +# 8.1.2-r0; it took Alpine ten days to take 8.1.2 onto the stable branch. +# +# So this builds Alpine's package itself, from its recipe at a pinned aports +# commit, and changes one thing: the version. Same configure line, same +# patches, same libraries, split into the same packages — what Alpine would +# publish, a release earlier. The runtime stage installs whichever is newer, +# this or Alpine's, so the weekly rebuild goes back to Alpine's package the +# day it catches up, without anybody having to remember to. +# +# The recipe is fetched by commit and its hash checked — from Alpine's mirror +# on GitHub, because their GitLab answers a build runner with a 418 meant for +# robots; the commit and the bytes are the same. The patches it lists are +# checked by abuild against the sums inside it; the tarball is checked +# against FFMPEG_SHA256 before its sha512 is written into the recipe. `abuild +# -r` installs the build dependencies and takes them away again, so what is +# left of this stage is the packages. +# --------------------------------------------------------------------------- +FROM alpine:3.24 AS ffmpeg_recipe +ARG FFMPEG_VERSION +ARG FFMPEG_SHA256 +ARG APORTS_COMMIT=d2c3ca384892f415fc1b92abd87f78c1d5d0cbba +ARG APKBUILD_SHA256=76c1c842c47b25fbb8133a86979f642a86545f7e505f46b13d46aa13764a9a1d + +RUN set -eu; \ + apk add --no-cache alpine-sdk curl; \ + SUDO= abuild-keygen -a -i -n; \ + mkdir -p /recipe/community/ffmpeg /var/cache/distfiles; \ + cd /recipe/community/ffmpeg; \ + aports="https://raw.githubusercontent.com/alpinelinux/aports/${APORTS_COMMIT}/community/ffmpeg"; \ + for file in APKBUILD add-av_stream_get_first_dts-for-chromium.patch posix-ioctl.patch; do \ + curl -fsSL -o "$file" "$aports/$file"; \ + done; \ + echo "${APKBUILD_SHA256} APKBUILD" | sha256sum -c -; \ + tarball="/var/cache/distfiles/ffmpeg-${FFMPEG_VERSION}.tar.xz"; \ + curl -fsSL -o "$tarball" "https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz"; \ + echo "${FFMPEG_SHA256} $tarball" | sha256sum -c -; \ + sha512="$(sha512sum "$tarball" | cut -d ' ' -f 1)"; \ + sed -i \ + -e "s/^pkgver=.*/pkgver=${FFMPEG_VERSION}/" \ + -e "s/^pkgrel=.*/pkgrel=0/" \ + -e "s/^[0-9a-f]\{128\} ffmpeg-[0-9.]*\.tar\.xz\$/${sha512} ffmpeg-${FFMPEG_VERSION}.tar.xz/" \ + APKBUILD; \ + grep -qx "pkgver=${FFMPEG_VERSION}" APKBUILD; \ + grep -qx "${sha512} ffmpeg-${FFMPEG_VERSION}.tar.xz" APKBUILD; \ + apk update --quiet; \ + abuild -F -r -P /out; \ + ls /out/community/*/ffmpeg-"${FFMPEG_VERSION}"-r0.apk; \ + rm -rf /recipe /var/cache/distfiles/* /var/cache/apk/* + +# Every format the explorer previews has to decode with what was just built, +# as it does for the lean image — installed here and nowhere else, to check. +COPY docker/verify-ffmpeg.sh /usr/local/bin/verify-ffmpeg.sh +RUN set -eu; \ + dir="$(dirname "$(ls /out/community/*/ffmpeg-"${FFMPEG_VERSION}"-r0.apk)")"; \ + apk add --no-cache --allow-untrusted "$dir/ffmpeg-${FFMPEG_VERSION}-r0.apk" \ + "$dir"/ffmpeg-libav*-"${FFMPEG_VERSION}"-r0.apk \ + "$dir"/ffmpeg-libsw*-"${FFMPEG_VERSION}"-r0.apk; \ + ffmpeg -version | head -n 1 | grep -q "^ffmpeg version ${FFMPEG_VERSION} "; \ + chmod +x /usr/local/bin/verify-ffmpeg.sh; \ + /usr/local/bin/verify-ffmpeg.sh "$(command -v ffmpeg)" "$(command -v ffprobe)" + + +# --------------------------------------------------------------------------- +# Which ffmpeg the runtime gets, decided before anything is built +# +# The runtime mounts a stage rather than copying from it, so no bytes of the +# source build reach a layer it does not use. But a mount is a dependency: +# naming `ffmpeg_build` directly made BuildKit compile it for every image, +# including the ones that go on to install Alpine's package and never read it. +# +# On amd64 that was a quiet waste. On arm64 under emulation it was the whole +# build: the release job spent over forty minutes compiling an ffmpeg the full +# image discards, and was cut off at its hour limit having published nothing. +# +# Selecting the stage here means the compile happens only for the variant that +# asked for it: `source` is the lean image's build, `apk` the packages built +# from Alpine's recipe — just the packages, so the mount carries nothing of +# the stage that made them. +# --------------------------------------------------------------------------- +FROM scratch AS ffmpeg_apk +COPY --from=ffmpeg_recipe /out /out + +FROM ffmpeg_build AS ffmpeg_source + +# An unknown value fails here, by name, rather than silently building neither. +FROM ffmpeg_${FFMPEG_VARIANT} AS ffmpeg_selected + FROM base AS runtime ENV NODE_ENV=production +# Enlarge the libuv thread pool so directory-listing fs.stat calls are not +# starved by concurrent thumbnail-generation fs operations (keeps navigation +# responsive while a large media folder is being processed). Tunable at runtime. +ENV UV_THREADPOOL_SIZE=16 # Create the baseline app user; UID/GID may be mutated at runtime via entrypoint.sh. # Alpine uses busybox addgroup/adduser instead of Debian's groupadd/useradd. @@ -51,49 +272,107 @@ RUN addgroup -S appuser && \ # Runtime packages only. # # Core (always installed): -# ffmpeg – video thumbnail extraction & software transcoding +# ffmpeg – video thumbnails and metadata, and HEIC stills: since 7.1 +# its HEIF demuxer reconstructs the tile grid a phone photo +# is made of, which is why ImageMagick is no longer here. # gosu – UID/GID remapping in entrypoint # ripgrep – fast file-content search -# imagemagick – HEIC → PNG thumbnail conversion +# poppler-utils – pdftotext, so a search can read the words in a PDF. Only +# ones with a text layer; a scan needs OCR, which is +# seconds per page and does not belong in a request. # openssh-client – optional SSH remote access (terminal only) -# unzip – demo mode sample extraction (downloadSamples.js) +# 7zzs – official static 7-Zip binary, copied below; supports +# encrypted ZIP/7z/RAR archives and the RAR codec # bash – entrypoint.sh is a bash script # shadow – provides usermod/groupmod for UID/GID remapping # curl – For terminal users +# rsync – native, cancellable local copies with byte progress # # Optional (see INCLUDE_RAW / INCLUDE_VAAPI build args below): # perl – required by exiftool-vendored for RAW image previews # libva – core VA-API runtime (includes libva-drm) -# mesa-va-gallium – Mesa VA-API GPU drivers (pulls Mesa + LLVM, ~80 MB) +# mesa-va-gallium – Mesa VA-API GPU drivers (pulls Mesa + LLVM: 211 MB, measured +# as the marginal cost of libva + mesa-va-gallium over the +# rest of this list, against the Alpine 3.23 package index) # Optional feature stacks — toggled at build time. Defaults keep the FULL image # byte-for-byte identical to before. # INCLUDE_RAW=false drops perl + the exiftool-vendored node module: removes -# RAW-photo previews only (normal EXIF still works via exifr). -# INCLUDE_VAAPI=false drops libva + mesa-va-gallium (Mesa + LLVM, ~80 MB): ffmpeg +# RAW-photo previews only; ordinary EXIF is read from the +# block sharp hands back, which costs nothing extra. +# INCLUDE_VAAPI=false drops libva + mesa-va-gallium (Mesa + LLVM, 211 MB): ffmpeg # still decodes video in software. VA-API is opt-in anyway, # used only when FFMPEG_HWACCEL is set with a GPU passed in. +# This is by far the largest thing in the image, and it is +# inert on any host that does not pass a GPU to the +# container: the -lean variant exists mainly to drop it. +# FFMPEG_VARIANT=source builds ffmpeg from source with the encoders stripped +# out (see the ffmpeg_build stage). Every decoder, +# demuxer and parser is kept, so nothing stops being +# previewable. Requires INCLUDE_VAAPI=false: that build +# has no VA-API, and enabling it would pull back most of +# what this removes. ARG INCLUDE_RAW=true ARG INCLUDE_VAAPI=true +ARG FFMPEG_VARIANT=apk RUN apk add --no-cache \ - ffmpeg \ gosu \ ripgrep \ - rsync \ - p7zip \ poppler-utils \ - imagemagick \ openssh-client \ - unzip \ bash \ shadow \ curl \ + rsync \ && if [ "$INCLUDE_RAW" = "true" ]; then apk add --no-cache perl; fi \ && if [ "$INCLUDE_VAAPI" = "true" ]; then apk add --no-cache libva mesa-va-gallium; fi \ && rm -rf /tmp/* /var/cache/apk/* +# ffmpeg, from one source or the other. The build stage is mounted rather than +# copied, so none of its bytes reach a layer they are not installed into. +# +# The runtime libraries have to come with it: both builds link against the +# Alpine ones rather than being static, which keeps them small and keeps the +# security updates of those libraries coming from apk rather than from a +# rebuild. +# +# For `apk`, the newer of two wins: Alpine's own package, or the one built +# from its recipe at FFMPEG_VERSION. Alpine's wins as soon as it is at least as +# new, which is what lets the weekly rebuild — this stage is never taken from +# the cache there — move on without an edit. A version apk cannot read fails +# the build rather than choosing by accident. +RUN --mount=from=ffmpeg_selected,target=/ffmpeg-built \ + set -eu; \ + if [ "$FFMPEG_VARIANT" = "source" ]; then \ + if [ "$INCLUDE_VAAPI" = "true" ]; then \ + echo "FFMPEG_VARIANT=source has no VA-API; build with INCLUDE_VAAPI=false" >&2; \ + exit 1; \ + fi; \ + apk add --no-cache dav1d libbz2; \ + install -m 0755 /ffmpeg-built/out/bin/ffmpeg /ffmpeg-built/out/bin/ffprobe /usr/local/bin/; \ + else \ + built="$(ls /ffmpeg-built/out/community/*/ffmpeg-[0-9]*.apk)"; \ + dir="$(dirname "$built")"; \ + ours="$(basename "$built" .apk)"; \ + ours="${ours#ffmpeg-}"; \ + apk update --quiet; \ + theirs="$(apk search -x ffmpeg | sed -n 's/^ffmpeg-//p')"; \ + apk version -c "$ours" "$theirs"; \ + if [ "$(apk version -t "$theirs" "$ours")" = "<" ]; then \ + apk add --no-cache --allow-untrusted "$built" \ + "$dir"/ffmpeg-libav*-"$ours".apk "$dir"/ffmpeg-libsw*-"$ours".apk; \ + echo "ffmpeg $ours, built from Alpine's recipe; Alpine has $theirs"; \ + else \ + apk add --no-cache ffmpeg; \ + echo "ffmpeg $theirs, Alpine's own; the recipe build is $ours"; \ + fi; \ + fi; \ + rm -rf /var/cache/apk/*; \ + ffmpeg -version >/dev/null; \ + ffprobe -version >/dev/null + WORKDIR /app # Make git metadata available at runtime for backend /api/features endpoint. @@ -106,15 +385,39 @@ ENV REPO_URL=${REPO_URL} # Bring in backend production node_modules (pre-compiled for Alpine musl). # Build tools from backend_deps stage are NOT included — only the output. -COPY --from=backend_deps /app/node_modules ./node_modules -COPY --from=backend_deps /app/package.json ./ - -# When RAW support is disabled, drop the vendored ExifTool (~20 MB) from the -# runtime node_modules. rawPreviewService.js already degrades gracefully when the -# module is absent (the require is wrapped in try/catch). -RUN if [ "$INCLUDE_RAW" != "true" ]; then \ +# +# Mounted and copied in one step rather than COPY'd, so that a build without RAW +# support can drop the vendored ExifTool before the layer is committed. Deleting +# it afterwards, which is what this did, removes it from the filesystem and from +# nothing else: the bytes stay in the earlier layer, get pulled on every pull, +# and are still counted in the image size. That was 23 MB of Perl in the lean +# image, with no interpreter present to run it. +# +# The same step drops any `coverage/` a dependency published by accident — 11 MB +# of it, almost entirely fluent-ffmpeg, whose npm tarball carries its own V8 +# coverage dumps beside a lib/ of 110 KB. Nothing requires its own coverage +# output at runtime, so the rule is safe to apply across the tree. +RUN --mount=from=backend_deps,source=/app,target=/deps \ + set -eu; \ + cp -a /deps/node_modules ./node_modules; \ + cp /deps/package.json ./; \ + if [ "$INCLUDE_RAW" != "true" ]; then \ rm -rf node_modules/exiftool-vendored node_modules/exiftool-vendored.pl; \ - fi + fi; \ + find node_modules -type d \( -name coverage -o -name .nyc_output \) \ + -prune -exec rm -rf {} +; \ + rm -rf node_modules/@types node_modules/@redis node_modules/ioredis node_modules/@babel +COPY --from=seven_zip /out/7z /usr/local/bin/7z +COPY docker/verify-7zip-password.js ./verify-7zip-password.js +# Verify both the RAR codec and the non-interactive password flow through the +# same PTY mechanism used by the backend. The sentinel password is build-only. +RUN 7z i | grep -qi 'rar' \ + && mkdir -p /tmp/7z-password-check/input /tmp/7z-password-check/output \ + && printf 'ok' > /tmp/7z-password-check/input/check.txt \ + && (cd /tmp/7z-password-check/input && 7z a -t7z -y -pbuild-check ../archive.7z check.txt >/dev/null) \ + && node ./verify-7zip-password.js /tmp/7z-password-check/archive.7z /tmp/7z-password-check/output build-check \ + && test "$(cat /tmp/7z-password-check/output/check.txt)" = 'ok' \ + && rm -rf /tmp/7z-password-check ./verify-7zip-password.js # Copy backend source and healthcheck. COPY backend/src ./src diff --git a/Dockerfile.dev b/Dockerfile.dev index bed2194bd..ca4b52b46 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,20 @@ -FROM public.ecr.aws/docker/library/node:24-alpine +FROM alpine:3.24 AS seven_zip +ARG TARGETARCH +ARG SEVEN_ZIP_VERSION=26.03 +RUN apk add --no-cache curl libarchive-tools \ + && case "$TARGETARCH" in \ + amd64) archive_arch=x64; archive_sha256=dc99eff5008f1ab79bd7084c68513701547a808a89502bf4133683535ab3c695 ;; \ + arm64) archive_arch=arm64; archive_sha256=2389ba20e4d8295e8709c20b6263b69bd1ec4972fe38a04ad7a1badbf595b996 ;; \ + *) echo "Unsupported 7-Zip architecture: $TARGETARCH" >&2; exit 1 ;; \ + esac \ + && archive_version=$(printf '%s' "$SEVEN_ZIP_VERSION" | tr -d .) \ + && curl -fsSL -o /tmp/7z.tar.xz "https://github.com/ip7z/7zip/releases/download/${SEVEN_ZIP_VERSION}/7z${archive_version}-linux-${archive_arch}.tar.xz" \ + && echo "${archive_sha256} /tmp/7z.tar.xz" | sha256sum -c - \ + && mkdir -p /out /tmp/7z \ + && bsdtar -xJf /tmp/7z.tar.xz -C /tmp/7z \ + && install -m 0755 "$(find /tmp/7z -type f -name 7zzs -print -quit)" /out/7z + +FROM public.ecr.aws/docker/library/node:24.21-alpine3.24 WORKDIR /repo ENV NODE_ENV=development @@ -12,15 +28,22 @@ RUN apk add --no-cache \ curl \ unzip \ openssh-client \ + rsync \ + shadow \ + perl \ libva \ mesa-va-gallium \ bash \ && rm -rf /tmp/* /var/cache/apk/* +COPY --from=seven_zip /out/7z /usr/local/bin/7z +RUN 7z i | grep -qi 'rar' + # Install workspace dependencies (seeded into the named volume on first run). COPY package.json package-lock.json ./ COPY backend/package.json ./backend/package.json COPY frontend/package.json ./frontend/package.json +COPY docs/package.json ./docs/package.json # Use npm install instead of npm ci for dev to properly handle optional dependencies # This avoids ARM64 rollup dependency issues in Docker diff --git a/backend/.eslintrc.cjs b/backend/.eslintrc.cjs deleted file mode 100644 index 0214979b6..000000000 --- a/backend/.eslintrc.cjs +++ /dev/null @@ -1,6 +0,0 @@ -/* eslint-env node */ -module.exports = { - env: { - node: true, - }, -}; diff --git a/backend/Dockerfile b/backend/Dockerfile index 8c3aee054..e1b719846 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM public.ecr.aws/docker/library/node:20-bookworm +FROM public.ecr.aws/docker/library/node:24.21-bookworm RUN apt-get update && apt-get install -y --no-install-recommends \ ffmpeg \ ripgrep \ @@ -16,4 +16,4 @@ ENV NODE_ENV=development EXPOSE 3001 HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ CMD wget -q -T 2 -t 1 -O /dev/null "http://127.0.0.1:${PORT:-3001}/healthz" || exit 1 -CMD ["bash", "-lc", "npm ci && npm run start"] \ No newline at end of file +CMD ["bash", "-lc", "npm ci && npm run start"] diff --git a/backend/package.json b/backend/package.json index bfd6f12e4..04a0fc5e0 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,60 +1,58 @@ { "name": "finder", - "version": "3.1.0", + "version": "3.11.0", "description": "explorer", "main": "src/server.js", + "engines": { + "node": ">=24 <25" + }, "scripts": { "download_samples": "node src/scripts/downloadSamples.js", + "openapi": "node src/scripts/writeOpenApi.js", "dev": "node --watch src/server.js", "format": "prettier --config ../prettier.config.cjs --write .", "format:check": "prettier --config ../prettier.config.cjs --check .", "test": "vitest run", "test:watch": "vitest", "test:ui": "vitest --ui", - "test:coverage": "vitest run --coverage", - "start": "node --watch src/server.js", - "openapi": "node src/scripts/writeOpenApi.js" + "test:coverage": "vitest run --coverage --coverage.reporter=text-summary --coverage.reporter=json-summary", + "start": "node --watch src/server.js" }, "author": "Vikram Soni", "license": "ISC", "dependencies": { - "@homebridge/node-pty-prebuilt-multiarch": "^0.13.1", + "@homebridge/node-pty-prebuilt-multiarch": "^0.14.1", "@tus/file-store": "^2.1.0", "@tus/server": "^2.4.1", - "adm-zip": "^0.5.16", - "archiver": "^6.0.2", - "axios": "^1.7.7", + "adm-zip": "^0.6.1", + "archiver": "^8.0.0", + "axios": "^1.18.1", "bcryptjs": "^2.4.3", - "better-sqlite3": "^12.5.0", - "body-parser": "^1.20.2", - "connect-sqlite3": "^0.9.16", + "better-sqlite3": "^12.11.1", "cookie-parser": "^1.4.7", "cors": "^2.8.5", "exif-reader": "^2.0.3", - "exiftool-vendored": "^34.1.0", + "exiftool-vendored": "^37.0.0", "express": "^5.2.1", - "express-openid-connect": "^2.19.2", + "express-openid-connect": "^2.20.2", "express-rate-limit": "^7.2.0", "express-session": "^1.17.3", - "fluent-ffmpeg": "^2.1.2", "jsonwebtoken": "^9.0.2", - "memorystore": "^1.6.7", - "multer": "^2.0.2", + "multer": "^2.4.0", "p-limit": "^3.1.0", "p-queue": "^7.4.1", "pino": "^10.1.0", "pino-http": "^11.0.0", "pino-pretty": "^13.1.2", - "sharp": "^0.34.4", - "uuid": "^13.0.0", - "ws": "^8.17.0", - "yauzl": "^2.10.0" + "sharp": "^0.35.4", + "uuid": "^13.0.2", + "ws": "^8.21.1" }, "devDependencies": { - "@vitest/coverage-v8": "^4.0.18", - "@vitest/ui": "^4.0.18", + "@vitest/coverage-v8": "^4.1.11", + "@vitest/ui": "^4.1.11", "prettier": "^3.6.2", "supertest": "^7.1.4", - "vitest": "^4.0.18" + "vitest": "^4.1.11" } } diff --git a/backend/src/app.js b/backend/src/app.js index f7d1c0e31..023c7a6ea 100644 --- a/backend/src/app.js +++ b/backend/src/app.js @@ -9,21 +9,22 @@ const express = require('express'); const cookieParser = require('cookie-parser'); const { configureTrustProxy } = require('./middleware/trustProxy'); +const { forwardedAddressWarning } = require('./utils/clientAddress'); const { configureSecurityHeaders } = require('./middleware/securityHeaders'); +const { requestContextMiddleware } = require('./utils/requestContext'); +const { uploads } = require('./config/index'); const { configureHttpLogging } = require('./middleware/logging'); const { configureCors } = require('./middleware/cors'); const { configureOidc } = require('./middleware/oidc'); const { configureHttpsWarning } = require('./middleware/httpsWarning'); -const { requestContextMiddleware } = require('./utils/requestContext'); -const { forwardedAddressWarning } = require('./utils/clientAddress'); const authMiddleware = require('./middleware/authMiddleware'); +const { heldRequestLogger } = require('./middleware/heldRequests'); const registerRoutes = require('./routes'); const { configureStaticFiles } = require('./utils/staticServer'); const { bootstrap } = require('./utils/bootstrap'); const { configureSession } = require('./middleware/session'); const logger = require('./utils/logger'); const { errorHandler, notFoundHandler } = require('./middleware/errorHandler'); -const { uploads } = require('./config'); /** * Creates and configures the Express application. @@ -51,16 +52,46 @@ const createApp = async (options = {}) => { // a client this server was not told to believe: without it every recorded // address is the proxy's and nothing anywhere says why. app.use(forwardedAddressWarning); + // Opens the per-request scratch space early, so everything downstream can + // memoize work that must not be reused by the next request. + app.use(requestContextMiddleware); configureSecurityHeaders(app); configureHttpLogging(app); configureCors(app); - // Large enough to carry back whatever the text editor was allowed to open; - // see the reasoning beside the two limits in the configuration. + + // Before everything that could hold a request, so that what it reports is + // the whole of the chain below it. + app.use(heldRequestLogger); + + // Liveness, before anything that could hold a request. + // + // These were mounted with the rest of the routes, which put them behind the + // session store, the OpenID Connect middleware and the authorization layer. + // A probe that travels through all of that does not answer "is this + // container alive" — it answers "is the identity provider reachable, and is + // the session store responding", and a container was reported unhealthy for + // ten minutes while the application it runs was serving pages perfectly. + // + // Nothing here reads a cookie, a database or the network, so there is no + // state it could wait on. + app.use('/', require('./routes/health')); + + // A selection of a few thousand files is a normal request here, and its list + // of paths outgrows the 100 kB Express allows by default. app.use(express.json({ limit: uploads.maxJsonBodyBytes })); app.use(express.urlencoded({ extended: true, limit: uploads.maxJsonBodyBytes })); + + // Express 5 leaves `req.body` undefined when no parser above matched the + // request's content type, where Express 4 left an empty object. Every route + // in this application was written against the empty object — and the ones + // asking `'field' in req.body` do not fail politely, they throw a TypeError + // and answer 500 to a request whose only fault is a missing header. + app.use((req, _res, next) => { + if (req.body === undefined) req.body = {}; + next(); + }); app.use(cookieParser()); - app.use(requestContextMiddleware); logger.debug('Mounted cookie parser middleware'); if (!skipBootstrap) { diff --git a/backend/src/config/env.js b/backend/src/config/env.js index 988d9bf83..b8a5ed132 100644 --- a/backend/src/config/env.js +++ b/backend/src/config/env.js @@ -1,4 +1,4 @@ -const { normalizeBoolean } = require('../utils/env'); +const { normalizeBoolean, readSecret } = require('../utils/env'); /** * Single source of truth for ALL environment variables. @@ -11,6 +11,34 @@ module.exports = { // Node.js HTTP server request timeout (ms). Set to 0 to disable. // Node defaults to 300000ms (5 minutes) on modern versions, which can abort large uploads. HTTP_TIMEOUT: process.env.HTTP_TIMEOUT != null ? Number(process.env.HTTP_TIMEOUT) : 0, + UPLOAD_INACTIVITY_TIMEOUT: + process.env.UPLOAD_INACTIVITY_TIMEOUT != null + ? Number(process.env.UPLOAD_INACTIVITY_TIMEOUT) + : 120000, + // rsync preserves permissions when copying, which means a chmod on the + // destination. Where that is always refused — a ZFS dataset with + // aclmode=restricted, where new files must inherit the directory's ACL — set + // this to false and skip straight to a copy that does not try, rather than + // paying for the failed attempt and its retry on every single copy. + COPY_PRESERVE_PERMISSIONS: normalizeBoolean(process.env.COPY_PRESERVE_PERMISSIONS) ?? true, + UPLOAD_CHUNKED_ENABLED: normalizeBoolean(process.env.UPLOAD_CHUNKED_ENABLED) ?? false, + // When direct (XHR) upload is used, automatically fall back to chunked uploads + // if a request fails because a reverse proxy rejects the body size. + UPLOAD_CHUNKED_AUTO_FALLBACK: normalizeBoolean(process.env.UPLOAD_CHUNKED_AUTO_FALLBACK) ?? false, + UPLOAD_CHUNK_SIZE: process.env.UPLOAD_CHUNK_SIZE?.trim() || null, + // Upper bound (MiB) an admin may set for the chunk size; caps the settings + // slider/input and clamps saved values. Hard ceiling of 512 MiB still applies. + MAX_CHUNK_SIZE_MIB: process.env.MAX_CHUNK_SIZE_MIB?.trim() || null, + UPLOAD_STORAGE_RESERVE: process.env.UPLOAD_STORAGE_RESERVE?.trim() || '64M', + TUS_UPLOAD_DIR: process.env.TUS_UPLOAD_DIR?.trim() || null, + TUS_INCOMPLETE_UPLOAD_TTL_MS: + process.env.TUS_INCOMPLETE_UPLOAD_TTL_MS != null + ? Number(process.env.TUS_INCOMPLETE_UPLOAD_TTL_MS) + : 60 * 60 * 1000, + TUS_CLEANUP_INTERVAL_MS: + process.env.TUS_CLEANUP_INTERVAL_MS != null + ? Number(process.env.TUS_CLEANUP_INTERVAL_MS) + : 10 * 60 * 1000, // Paths VOLUME_ROOT: process.env.VOLUME_ROOT || '/mnt', @@ -37,16 +65,52 @@ module.exports = { LOG_LEVEL: process.env.LOG_LEVEL?.trim().toLowerCase() || null, DEBUG: normalizeBoolean(process.env.DEBUG), ENABLE_HTTP_LOGGING: normalizeBoolean(process.env.ENABLE_HTTP_LOGGING) || false, + // Lightweight process/cgroup diagnostics, off by default. When enabled the + // sampler logs only anomalous intervals unless explicitly told otherwise. + PERFORMANCE_DIAGNOSTICS_ENABLED: + normalizeBoolean(process.env.PERFORMANCE_DIAGNOSTICS_ENABLED) ?? false, + PERFORMANCE_DIAGNOSTICS_INTERVAL_MS: + process.env.PERFORMANCE_DIAGNOSTICS_INTERVAL_MS != null + ? Number(process.env.PERFORMANCE_DIAGNOSTICS_INTERVAL_MS) + : 15000, + PERFORMANCE_DIAGNOSTICS_LOG_EVERY_INTERVAL: + normalizeBoolean(process.env.PERFORMANCE_DIAGNOSTICS_LOG_EVERY_INTERVAL) ?? false, + PERFORMANCE_DIAGNOSTICS_CPU_THRESHOLD: + process.env.PERFORMANCE_DIAGNOSTICS_CPU_THRESHOLD != null + ? Number(process.env.PERFORMANCE_DIAGNOSTICS_CPU_THRESHOLD) + : 75, + PERFORMANCE_DIAGNOSTICS_RSS_THRESHOLD_MB: + process.env.PERFORMANCE_DIAGNOSTICS_RSS_THRESHOLD_MB != null + ? Number(process.env.PERFORMANCE_DIAGNOSTICS_RSS_THRESHOLD_MB) + : 768, + PERFORMANCE_DIAGNOSTICS_EVENT_LOOP_DELAY_MS: + process.env.PERFORMANCE_DIAGNOSTICS_EVENT_LOOP_DELAY_MS != null + ? Number(process.env.PERFORMANCE_DIAGNOSTICS_EVENT_LOOP_DELAY_MS) + : 250, // Auth AUTH_ENABLED: normalizeBoolean(process.env.AUTH_ENABLED), AUTH_MODE: process.env.AUTH_MODE?.trim().toLowerCase() || null, - SESSION_SECRET: process.env.SESSION_SECRET || process.env.AUTH_SESSION_SECRET || null, + SESSION_SECRET: readSecret('SESSION_SECRET', 'AUTH_SESSION_SECRET'), SESSION_MAX_AGE_DAYS: Number(process.env.SESSION_MAX_AGE_DAYS) || 30, AUTH_MAX_FAILED: Number(process.env.AUTH_MAX_FAILED) || 5, AUTH_LOCK_MINUTES: Number(process.env.AUTH_LOCK_MINUTES) || 15, AUTH_ADMIN_EMAIL: process.env.AUTH_ADMIN_EMAIL?.trim() || process.env.ADMIN_EMAIL?.trim() || null, - AUTH_ADMIN_PASSWORD: process.env.AUTH_ADMIN_PASSWORD || process.env.ADMIN_PASSWORD || null, + AUTH_ADMIN_PASSWORD: readSecret('AUTH_ADMIN_PASSWORD', 'ADMIN_PASSWORD'), + + // Demo mode. Also read by the entrypoint, which seeds the sample files; the + // two agree on what counts as true. Anything unrecognised normalises to null + // and is treated as off, so the default here is closed. + DEMO_MODE: normalizeBoolean(process.env.DEMO_MODE) ?? false, + // Credentials pre-filled on the sign-in form, for a public demo where the + // login is published anyway and asking visitors to retype it is friction for + // nothing. Serving a password is never acceptable outside that case, so it + // takes both DEMO_MODE and these two variables — deliberately separate from + // the admin ones above, so no flag can ever publish a real password by + // reaching for credentials that were set for another purpose. + DEMO_LOGIN_EMAIL: process.env.DEMO_LOGIN_EMAIL?.trim() || null, + // Not trimmed — a password is whatever it is. + DEMO_LOGIN_PASSWORD: process.env.DEMO_LOGIN_PASSWORD || null, // OIDC OIDC_ENABLED: normalizeBoolean(process.env.OIDC_ENABLED), @@ -56,7 +120,7 @@ module.exports = { OIDC_USERINFO_URL: process.env.OIDC_USERINFO_URL || null, OIDC_LOGOUT_URL: process.env.OIDC_LOGOUT_URL || null, OIDC_CLIENT_ID: process.env.OIDC_CLIENT_ID || null, - OIDC_CLIENT_SECRET: process.env.OIDC_CLIENT_SECRET || null, + OIDC_CLIENT_SECRET: readSecret('OIDC_CLIENT_SECRET'), OIDC_CALLBACK_URL: process.env.OIDC_CALLBACK_URL || process.env.OIDC_REDIRECT_URI || null, // Allowlisted native app redirect URIs for the mobile OIDC bridge (iOS/Android). OIDC_MOBILE_REDIRECT_URIS: process.env.OIDC_MOBILE_REDIRECT_URIS || null, @@ -69,58 +133,42 @@ module.exports = { // Search SEARCH_DEEP: normalizeBoolean(process.env.SEARCH_DEEP), SEARCH_RIPGREP: normalizeBoolean(process.env.SEARCH_RIPGREP), - // --- Archive extraction --- - MAX_EXTRACTED_ARCHIVE_SIZE: process.env.MAX_EXTRACTED_ARCHIVE_SIZE?.trim() || null, - MAX_ARCHIVE_ENTRIES: Number(process.env.MAX_ARCHIVE_ENTRIES) || 100000, - ARCHIVE_EXTENSIONS: process.env.ARCHIVE_EXTENSIONS || '', - // --- Folder size index --- - FOLDER_SIZE_MODE: process.env.FOLDER_SIZE_MODE?.trim().toLowerCase() || 'off', - FOLDER_SIZE_MODE_SET: - typeof process.env.FOLDER_SIZE_MODE === 'string' && process.env.FOLDER_SIZE_MODE.trim() !== '', - FOLDER_SIZE_EXCLUDE_PATHS: process.env.FOLDER_SIZE_EXCLUDE_PATHS || '', - FOLDER_SIZE_CONCURRENCY: Number(process.env.FOLDER_SIZE_CONCURRENCY) || 6, - FOLDER_SIZE_NETWORK_CONCURRENCY: Number(process.env.FOLDER_SIZE_NETWORK_CONCURRENCY) || 2, - FOLDER_SIZE_FLUSH_MS: Number(process.env.FOLDER_SIZE_FLUSH_MS) || 3000, - FOLDER_SIZE_RECONCILE_MS: Number(process.env.FOLDER_SIZE_RECONCILE_MS) || 0, - FOLDER_SIZE_RECONCILE_MIN_MS: Number(process.env.FOLDER_SIZE_RECONCILE_MIN_MS) || 900000, - FOLDER_SIZE_RECONCILE_MAX_MS: Number(process.env.FOLDER_SIZE_RECONCILE_MAX_MS) || 43200000, - FOLDER_SIZE_RECONCILE_BATCH: Number(process.env.FOLDER_SIZE_RECONCILE_BATCH) || 100, - FOLDER_SIZE_REBUILD: normalizeBoolean(process.env.FOLDER_SIZE_REBUILD) || false, - // --- Search index --- + SEARCH_MAX_FILESIZE: process.env.SEARCH_MAX_FILESIZE?.trim() || null, + SEARCH_TIMEOUT_MS: Number(process.env.SEARCH_TIMEOUT_MS) || null, SEARCH_INDEX: normalizeBoolean(process.env.SEARCH_INDEX) ?? false, - // Whether the operator set it at all, as opposed to what it came out as. - // A switch in Settings may decide what nobody decided in the environment; it may - // not overrule what somebody did. + // Whether the environment decided, as against falling to the default. When + // it did, the switch in Settings shows the variable and cannot be moved; + // when it did not, Settings decides. A value that is not a boolean decided + // nothing, the same way a blank one does. SEARCH_INDEX_SET: normalizeBoolean(process.env.SEARCH_INDEX) !== null, SEARCH_INDEX_BATCH: Number(process.env.SEARCH_INDEX_BATCH) || null, SEARCH_INDEX_CPU_PERCENT: Number(process.env.SEARCH_INDEX_CPU_PERCENT) || null, SEARCH_INDEX_MEMORY_MB: Number(process.env.SEARCH_INDEX_MEMORY_MB) || null, SEARCH_INDEX_EXCLUDE: process.env.SEARCH_INDEX_EXCLUDE?.trim() || null, + PREVIEW_MAX_RENDER_SIZE: process.env.PREVIEW_MAX_RENDER_SIZE?.trim() || null, SEARCH_INDEX_REBUILD: normalizeBoolean(process.env.SEARCH_INDEX_REBUILD) ?? false, SEARCH_INDEX_RECONCILE_MS: Number(process.env.SEARCH_INDEX_RECONCILE_MS) || null, - SEARCH_TIMEOUT_MS: Number(process.env.SEARCH_TIMEOUT_MS) || null, - SEARCH_MAX_FILESIZE: process.env.SEARCH_MAX_FILESIZE?.trim() || null, // OnlyOffice ONLYOFFICE_URL: process.env.ONLYOFFICE_URL?.trim() || null, - ONLYOFFICE_SECRET: process.env.ONLYOFFICE_SECRET || null, + ONLYOFFICE_SECRET: readSecret('ONLYOFFICE_SECRET'), + ONLYOFFICE_DOWNLOAD_ORIGINS: process.env.ONLYOFFICE_DOWNLOAD_ORIGINS || '', ONLYOFFICE_LANG: process.env.ONLYOFFICE_LANG?.trim() || 'en', ONLYOFFICE_FORCE_SAVE: normalizeBoolean(process.env.ONLYOFFICE_FORCE_SAVE) || false, ONLYOFFICE_FORCE_SAVE_TIMEOUT_MS: Number(process.env.ONLYOFFICE_FORCE_SAVE_TIMEOUT_MS) || 10000, // 0 disables proactive writes to the external storage. A bounded interval - // keeps the Document Server's internal autosave from becoming a full document + // keeps Document Server's internal autosave from becoming a full document // conversion on every edit. ONLYOFFICE_AUTO_SAVE_INTERVAL_MS: (() => { const value = Number(process.env.ONLYOFFICE_AUTO_SAVE_INTERVAL_MS); return Number.isFinite(value) && value >= 0 ? value : 30000; })(), ONLYOFFICE_FILE_EXTENSIONS: process.env.ONLYOFFICE_FILE_EXTENSIONS || '', - ONLYOFFICE_DOWNLOAD_ORIGINS: process.env.ONLYOFFICE_DOWNLOAD_ORIGINS || '', // Collabora (WOPI) COLLABORA_URL: process.env.COLLABORA_URL?.trim() || null, COLLABORA_DISCOVERY_URL: process.env.COLLABORA_DISCOVERY_URL?.trim() || null, - COLLABORA_SECRET: process.env.COLLABORA_SECRET || null, + COLLABORA_SECRET: readSecret('COLLABORA_SECRET'), COLLABORA_LANG: process.env.COLLABORA_LANG?.trim() || 'en', COLLABORA_FILE_EXTENSIONS: process.env.COLLABORA_FILE_EXTENSIONS || '', @@ -144,6 +192,90 @@ module.exports = { VERSIONS_MAX_PER_FILE: process.env.VERSIONS_MAX_PER_FILE?.trim() || '50', VERSIONS_SESSION_CHECKPOINT_MINUTES: process.env.VERSIONS_SESSION_CHECKPOINT_MINUTES?.trim() || '10', + + // Passkeys. The name a passkey is bound to is the domain it was made on, so + // an installation reached through more than one hostname pins it here rather + // than letting each name hold its own passkeys. Empty means "the name this + // request arrived on", which is what a single-hostname installation wants. + WEBAUTHN_RP_ID: process.env.WEBAUTHN_RP_ID?.trim() || null, + WEBAUTHN_RP_NAME: process.env.WEBAUTHN_RP_NAME?.trim() || null, + + // Activity log: off unless somebody asks for it. It is a record of who did + // what, which is worth having when several people share an installation and + // is only weight when nobody is going to read it. + ACTIVITY_ENABLED: normalizeBoolean(process.env.ACTIVITY_ENABLED) ?? false, + ACTIVITY_RETENTION_DAYS: process.env.ACTIVITY_RETENTION_DAYS?.trim() || '90', + + // Folder size index + // Mode: 'off' (default, feature disabled), 'shallow' (size of a folder's + // direct entries only) or 'full' (recursive size of the whole subtree). + FOLDER_SIZE_MODE: process.env.FOLDER_SIZE_MODE?.trim().toLowerCase() || 'off', + // As SEARCH_INDEX_SET: only one of the three modes counts as a decision. + FOLDER_SIZE_MODE_SET: ['off', 'shallow', 'full'].includes( + process.env.FOLDER_SIZE_MODE?.trim().toLowerCase() + ), + // Comma or newline separated paths, relative to VOLUME_ROOT, that must never + // be traversed by the folder-size indexer. + FOLDER_SIZE_EXCLUDE_PATHS: process.env.FOLDER_SIZE_EXCLUDE_PATHS || '', + // Concurrency of the baseline walk on local vs network-detected mounts. + FOLDER_SIZE_CONCURRENCY: Number(process.env.FOLDER_SIZE_CONCURRENCY) || 6, + FOLDER_SIZE_NETWORK_CONCURRENCY: Number(process.env.FOLDER_SIZE_NETWORK_CONCURRENCY) || 2, + // How often (ms) accumulated dirty directories (on-view refresh, hooks, + // optional watcher) are flushed to the index in one transaction. + FOLDER_SIZE_FLUSH_MS: Number(process.env.FOLDER_SIZE_FLUSH_MS) || 3000, + // mtime-based reconciliation cadence. When 0 (default) the interval is + // adaptive: it accelerates to *MIN when a pass finds external changes and + // backs off (doubling) up to *MAX when idle. Set a non-zero value to force a + // fixed interval instead. + FOLDER_SIZE_RECONCILE_MS: Number(process.env.FOLDER_SIZE_RECONCILE_MS) || 0, + FOLDER_SIZE_RECONCILE_MIN_MS: Number(process.env.FOLDER_SIZE_RECONCILE_MIN_MS) || 900000, + FOLDER_SIZE_RECONCILE_MAX_MS: Number(process.env.FOLDER_SIZE_RECONCILE_MAX_MS) || 43200000, + // Reconciliation is paced so it never spikes CPU/IO on huge volumes: it stat()s + // folders in pages of *_BATCH and sleeps *_PAUSE_MS between pages, scanning as a + // gentle background trickle instead of one burst. + FOLDER_SIZE_RECONCILE_BATCH: Number(process.env.FOLDER_SIZE_RECONCILE_BATCH) || 100, + FOLDER_SIZE_RECONCILE_PAUSE_MS: + process.env.FOLDER_SIZE_RECONCILE_PAUSE_MS !== undefined + ? Number(process.env.FOLDER_SIZE_RECONCILE_PAUSE_MS) + : 200, + // Hard upper bound for one scheduled reconciliation slice. Zero keeps the + // historical full-volume sweep behavior; the default keeps idle work small. + FOLDER_SIZE_RECONCILE_MAX_DIRECTORIES: + process.env.FOLDER_SIZE_RECONCILE_MAX_DIRECTORIES !== undefined + ? Number(process.env.FOLDER_SIZE_RECONCILE_MAX_DIRECTORIES) + : 200, + // Targeted scans repair one incomplete subtree detected after an external + // change. They are serialized to keep ancestor deltas deterministic; file + // stats are processed in small batches and optionally paced. When batch/pause + // are unset they inherit the reconciliation settings. + FOLDER_SIZE_SUBTREE_BATCH: + process.env.FOLDER_SIZE_SUBTREE_BATCH !== undefined + ? Number(process.env.FOLDER_SIZE_SUBTREE_BATCH) + : null, + FOLDER_SIZE_SUBTREE_PAUSE_MS: + process.env.FOLDER_SIZE_SUBTREE_PAUSE_MS !== undefined + ? Number(process.env.FOLDER_SIZE_SUBTREE_PAUSE_MS) + : null, + FOLDER_SIZE_SUBTREE_SLOW_LOG_MS: + process.env.FOLDER_SIZE_SUBTREE_SLOW_LOG_MS !== undefined + ? Number(process.env.FOLDER_SIZE_SUBTREE_SLOW_LOG_MS) + : 5000, + // A filesystem operation that never settles must not block every queued + // targeted refresh forever. Zero disables the deadline for unusual filesystems + // where operators explicitly prefer waiting indefinitely. + FOLDER_SIZE_IO_TIMEOUT_MS: + process.env.FOLDER_SIZE_IO_TIMEOUT_MS !== undefined + ? Number(process.env.FOLDER_SIZE_IO_TIMEOUT_MS) + : 30000, + // Timed-out Node fs calls cannot be force-cancelled. Keep at most this many + // unresolved calls before pausing further folder-size I/O to preserve libuv + // worker capacity for the rest of the application. + FOLDER_SIZE_MAX_STALLED_IO: + process.env.FOLDER_SIZE_MAX_STALLED_IO !== undefined + ? Number(process.env.FOLDER_SIZE_MAX_STALLED_IO) + : 2, + // Force a fresh baseline walk even if the volume is already indexed. + FOLDER_SIZE_REBUILD: normalizeBoolean(process.env.FOLDER_SIZE_REBUILD) || false, USER_DIR_ENABLED: normalizeBoolean(process.env.USER_DIR_ENABLED) || false, USER_VOLUMES: normalizeBoolean(process.env.USER_VOLUMES) || false, SKIP_HOME: normalizeBoolean(process.env.SKIP_HOME) || false, @@ -152,15 +284,30 @@ module.exports = { // Uploads (direct, non-chunked) MAX_DIRECT_UPLOAD_SIZE: process.env.MAX_DIRECT_UPLOAD_SIZE?.trim() || null, - UPLOAD_STORAGE_RESERVE: process.env.UPLOAD_STORAGE_RESERVE?.trim() || '64M', - ACTIVITY_ENABLED: normalizeBoolean(process.env.ACTIVITY_ENABLED) ?? false, - ACTIVITY_RETENTION_DAYS: process.env.ACTIVITY_RETENTION_DAYS, - WEBAUTHN_RP_ID: process.env.WEBAUTHN_RP_ID?.trim() || null, - WEBAUTHN_RP_NAME: process.env.WEBAUTHN_RP_NAME?.trim() || null, - UPLOAD_CHUNK_SIZE: process.env.UPLOAD_CHUNK_SIZE, - UPLOAD_CHUNKED_ENABLED: normalizeBoolean(process.env.UPLOAD_CHUNKED_ENABLED), - MAX_CHUNK_SIZE_MIB: process.env.MAX_CHUNK_SIZE_MIB, - UPLOAD_INACTIVITY_TIMEOUT: process.env.UPLOAD_INACTIVITY_TIMEOUT, + MAX_JSON_BODY_SIZE: process.env.MAX_JSON_BODY_SIZE?.trim() || null, + BULK_DELETE_CONCURRENCY: Number(process.env.BULK_DELETE_CONCURRENCY) || 0, + MAX_FILES_PER_UPLOAD: Number(process.env.MAX_FILES_PER_UPLOAD) || 50, + + // Archives + MAX_EXTRACTED_ARCHIVE_SIZE: process.env.MAX_EXTRACTED_ARCHIVE_SIZE?.trim() || null, + MAX_ARCHIVE_ENTRIES: Number(process.env.MAX_ARCHIVE_ENTRIES) || 100000, + MAX_BROWSABLE_ARCHIVE_SIZE: process.env.MAX_BROWSABLE_ARCHIVE_SIZE?.trim() || null, + ARCHIVE_CACHE_MAX_SIZE: process.env.ARCHIVE_CACHE_MAX_SIZE?.trim() || null, + + // Editor + EDITOR_EXTENSIONS: process.env.EDITOR_EXTENSIONS || '', + EDITOR_MAX_FILESIZE: process.env.EDITOR_MAX_FILESIZE?.trim() || null, + + // Archive extraction (comma-separated; leading '+' extends the defaults + // instead of replacing them) + ARCHIVE_EXTENSIONS: process.env.ARCHIVE_EXTENSIONS || '', + + // FFmpeg + FFMPEG_PATH: process.env.FFMPEG_PATH || null, + FFPROBE_PATH: process.env.FFPROBE_PATH || null, + FFMPEG_HWACCEL: process.env.FFMPEG_HWACCEL?.trim() || null, + FFMPEG_HWACCEL_DEVICE: process.env.FFMPEG_HWACCEL_DEVICE?.trim() || null, + FFMPEG_HWACCEL_OUTPUT_FORMAT: process.env.FFMPEG_HWACCEL_OUTPUT_FORMAT?.trim() || null, THUMBNAILS_ENABLED: normalizeBoolean(process.env.THUMBNAILS_ENABLED) ?? true, THUMBNAIL_CACHE_MAX_FILES: process.env.THUMBNAIL_CACHE_MAX_FILES != null @@ -231,21 +378,6 @@ module.exports = { // their CPU priority so the Node event loop stays responsive during generation). THUMBNAIL_PROCESS_NICE: process.env.THUMBNAIL_PROCESS_NICE != null ? Number(process.env.THUMBNAIL_PROCESS_NICE) : 10, - TUS_UPLOAD_DIR: process.env.TUS_UPLOAD_DIR?.trim() || null, - TUS_INCOMPLETE_UPLOAD_TTL_MS: process.env.TUS_INCOMPLETE_UPLOAD_TTL_MS, - TUS_CLEANUP_INTERVAL_MS: process.env.TUS_CLEANUP_INTERVAL_MS, - MAX_FILES_PER_UPLOAD: Number(process.env.MAX_FILES_PER_UPLOAD) || 50, - - // Editor - EDITOR_EXTENSIONS: process.env.EDITOR_EXTENSIONS || '', - EDITOR_MAX_FILESIZE: process.env.EDITOR_MAX_FILESIZE?.trim() || null, - - // FFmpeg - FFMPEG_PATH: process.env.FFMPEG_PATH || null, - FFPROBE_PATH: process.env.FFPROBE_PATH || null, - FFMPEG_HWACCEL: process.env.FFMPEG_HWACCEL?.trim() || null, - FFMPEG_HWACCEL_DEVICE: process.env.FFMPEG_HWACCEL_DEVICE?.trim() || null, - FFMPEG_HWACCEL_OUTPUT_FORMAT: process.env.FFMPEG_HWACCEL_OUTPUT_FORMAT?.trim() || null, // Favorites FAVORITES_DEFAULT_ICON: process.env.FAVORITES_DEFAULT_ICON || 'outline:StarIcon', diff --git a/backend/src/config/index.js b/backend/src/config/index.js index 4bc9da340..93091e7aa 100644 --- a/backend/src/config/index.js +++ b/backend/src/config/index.js @@ -1,10 +1,10 @@ const path = require('path'); const crypto = require('crypto'); const env = require('./env'); -const { resolveSessionSecret } = require('./sessionSecret'); const constants = require('./constants'); const loggingConfig = require('./logging'); const { parseByteSize } = require('../utils/env'); +const { resolveSessionSecret } = require('./sessionSecret'); // logger reads config/logging, never this file — requiring it here makes no cycle. const logger = require('../utils/logger'); @@ -14,7 +14,10 @@ const parseCommaOrSpaceList = (raw) => { return parts.map((s) => s.trim()).filter(Boolean); }; -const DEFAULT_HIDDEN_FILE_PATTERNS = ['.']; +// Keep the artifacts of a transfer in progress under the same configurable +// policy as other hidden files: `.download` while one is being fetched, +// `.uploading` while one is being written. +const DEFAULT_HIDDEN_FILE_PATTERNS = ['.', 'regex:\\.download$', 'regex:\\.uploading$']; const parseRegexPattern = (token) => { if (token.startsWith('regex:')) { @@ -34,7 +37,7 @@ const parseRegexPattern = (token) => { return null; }; -const escapeRipgrepGlob = (value) => String(value).replace(/[\\*?\[\]{}]/g, '\\$&'); +const escapeRipgrepGlob = (value) => String(value).replace(/[\\*?[\]{}]/g, '\\$&'); const parseHiddenFilePatterns = (raw) => { const tokens = raw == null ? DEFAULT_HIDDEN_FILE_PATTERNS : parseCommaOrSpaceList(raw); @@ -144,7 +147,6 @@ const directories = { config: configDir, cache: cacheDir, thumbnails: path.join(cacheDir, 'thumbnails'), - extensions: path.join(configDir, 'extensions'), userRoot: userRootDir, userRootWithSep: userRootDir.endsWith(path.sep) ? userRootDir : `${userRootDir}${path.sep}`, }; @@ -157,7 +159,7 @@ if (env.PUBLIC_URL) { const url = new URL(env.PUBLIC_URL); publicUrl = url.href.replace(/\/$/, ''); publicOrigin = url.origin; - } catch (err) { + } catch (_) { console.warn(`[Config] Invalid PUBLIC_URL: ${env.PUBLIC_URL}`); } } @@ -174,7 +176,7 @@ const parseOriginList = (value, variableName = 'INTERNAL_URL') => .map((entry) => { try { return new URL(entry).origin; - } catch (err) { + } catch (_) { console.warn(`[Config] Invalid ${variableName} entry: ${entry}`); return null; } @@ -198,13 +200,11 @@ const buildCorsConfig = () => { } if (knownOrigins.length) return { allowAll: false, origins: [...knownOrigins] }; // Nothing configured: allow no cross-origin caller rather than reflecting - // whatever origin asks, which combined with credentials:true let any page - // on the same site — another port of the same host, a sibling subdomain, - // where the SameSite=Lax session cookie still goes — read authenticated - // responses. Same-origin requests carry no Origin (or are permitted by the - // browser's own policy), so the normal setup — frontend and API on one host - // — is unaffected. Declare CORS_ORIGINS, PUBLIC_URL or INTERNAL_URL to allow - // a real cross-origin client, or CORS_ORIGINS=* to reflect any origin. + // whatever origin asks, which combined with credentials:true would let any + // site read authenticated responses. Same-origin requests carry no Origin + // (or are permitted by the browser's own policy), so the normal setup — + // frontend and API on one host — is unaffected. Declare CORS_ORIGINS, + // PUBLIC_URL or INTERNAL_URL to allow a real cross-origin client. return { allowAll: false, origins: [] }; }; @@ -225,6 +225,17 @@ const corsOptions = { credentials: true, optionsSuccessStatus: 200, methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], + // Ten minutes of not asking the same question again. + // + // The client sends `X-Requested-With` so that an authenticating proxy answers + // an expired session with a 401 instead of redirecting a fetch to a sign-in + // page it cannot follow. That header is not one of the few CORS treats as + // safe, so on a cross-origin deployment every plain GET now needs a preflight + // first — and without a lifetime on the answer, every single one of them. + // + // Nothing changes for the usual deployment, where the image serves the + // application and the API from one origin and CORS never enters into it. + maxAge: 600, }; // --- HTTP server timeouts --- @@ -233,6 +244,30 @@ const requestTimeoutMs = (() => { return Number.isFinite(value) && value >= 0 ? value : 0; })(); +const uploadInactivityTimeoutMs = (() => { + const value = env.UPLOAD_INACTIVITY_TIMEOUT; + return Number.isFinite(value) && value >= 0 ? value : 120000; +})(); + +const uploadStorageReserveBytes = (() => { + const value = parseByteSize(env.UPLOAD_STORAGE_RESERVE); + return Number.isFinite(value) && value >= 0 ? value : 64 * 1024 * 1024; +})(); + +const tusUploadDir = env.TUS_UPLOAD_DIR + ? path.resolve(env.TUS_UPLOAD_DIR) + : path.join(cacheDir, 'tus-uploads'); + +const tusIncompleteUploadTtlMs = (() => { + const value = env.TUS_INCOMPLETE_UPLOAD_TTL_MS; + return Number.isFinite(value) && value >= 0 ? Math.floor(value) : 60 * 60 * 1000; +})(); + +const tusCleanupIntervalMs = (() => { + const value = env.TUS_CLEANUP_INTERVAL_MS; + return Number.isFinite(value) && value >= 0 ? Math.floor(value) : 10 * 60 * 1000; +})(); + // --- Auth --- // Determine auth mode: 'local', 'oidc', 'both', or 'disabled' // If AUTH_MODE is not set, fall back to legacy behavior based on OIDC_ENABLED @@ -252,6 +287,7 @@ const authMode = determineAuthMode(); const auth = { enabled: authMode === 'disabled' ? false : env.AUTH_ENABLED !== false, + // Configured, or generated once and kept in CONFIG_DIR — see sessionSecret.js. sessionSecret: resolveSessionSecret({ configured: env.SESSION_SECRET, configDir }), sessionMaxAgeMs: env.SESSION_MAX_AGE_DAYS * 24 * 60 * 60 * 1000, // Convert days to milliseconds mode: authMode, @@ -273,6 +309,9 @@ const auth = { }, }; +const deriveSecret = (purpose) => + crypto.createHmac('sha256', auth.sessionSecret).update(`nextexplorer:${purpose}`).digest('hex'); + // --- Search --- const searchMaxFileSizeBytes = (() => { const parsed = parseByteSize(env.SEARCH_MAX_FILESIZE); @@ -280,20 +319,23 @@ const searchMaxFileSizeBytes = (() => { })(); // --- Uploads --- -// --- Editor --- +// Ceilings for direct (non-chunked) uploads. They exist so a single request +// cannot stream until the disk is full; they are generous on purpose, since +// large files are a normal use of a file manager. Chunked uploads have their +// own storage guard in the TUS service. /** * What the inline text editor opens, and what a JSON request body may weigh. * - * They are one decision rather than two. The editor sends a file back through a - * JSON body when it saves it, so a body limit under the size the editor opens - * produces a file that opens and cannot be saved — answered with "request - * entity too large", which names neither setting. Express's own default is - * 100 kB, against an editor that opens two megabytes. + * They are one decision rather than two. The editor sends a file back through + * a JSON body when it saves it, so a body limit under the size the editor + * opens produces a file that opens and cannot be saved — answered with + * "request entity too large", which names neither setting. * * Escaping is why the body has to be worth more than the file: in the worst * case every character of the content is a quote, a backslash or a newline and * becomes two, and the path travels in the same body. A file whose bytes would - * expand further than that is one the editor refuses to open anyway, as binary. + * expand further than that is one the editor refuses to open anyway, as + * binary. */ const JSON_ESCAPE_WORST_CASE = 2; const JSON_BODY_OVERHEAD_BYTES = 64 * 1024; @@ -313,8 +355,8 @@ const { editorMaxFileSizeBytes, maxJsonBodyBytes } = (() => { const bodyWasChosen = Number.isFinite(parsedBody) && parsedBody > 0; // A body limit someone set is a ceiling they meant — it is a guard, not a - // detail — so it is never raised from here. The editor is what gives way, and - // it gives way by refusing to open what it could not save back. + // detail — so it is never raised from here. The editor is what gives way, + // and it gives way by refusing to open what it could not save back. if (bodyWasChosen) { const allowed = fileAllowedBy(parsedBody); if (editorAsked > allowed) { @@ -327,8 +369,8 @@ const { editorMaxFileSizeBytes, maxJsonBodyBytes } = (() => { return { editorMaxFileSizeBytes: Math.min(editorAsked, allowed), maxJsonBodyBytes: parsedBody }; } - // Nobody chose the body limit, so the editor's size is the only wish there is - // to honour: the default body limit rises to carry it. + // Nobody chose the body limit, so the editor's size is the only wish there + // is to honour: the default body limit rises to carry it. const needed = bodyNeededFor(editorAsked); if (needed > DEFAULT_JSON_BODY_BYTES) { logger.info( @@ -343,36 +385,6 @@ const { editorMaxFileSizeBytes, maxJsonBodyBytes } = (() => { }; })(); -// Ceilings for direct (non-chunked) uploads. They exist so a single request -// cannot stream until the disk is full; they are generous on purpose, since -// large files are a normal use of a file manager. Chunked uploads have their -// own storage guard in the TUS service. -// How long a direct upload may go without a byte arriving before it is given -// up on. A client that goes away mid-body otherwise holds the request, and the -// half-written file with it, until the socket itself times out. -const uploadInactivityTimeoutMs = (() => { - const value = Number(env.UPLOAD_INACTIVITY_TIMEOUT); - return Number.isFinite(value) && value >= 0 ? value : 120000; -})(); - -// Where a chunked upload's parts live until the whole file is there, and how -// long an unfinished one is kept. Under the cache rather than beside the -// destination: a part file is not a file anybody asked for, and a volume should -// never show one. -const tusUploadDir = env.TUS_UPLOAD_DIR - ? path.resolve(env.TUS_UPLOAD_DIR) - : path.join(cacheDir, 'tus-uploads'); - -const tusIncompleteUploadTtlMs = (() => { - const value = Number(env.TUS_INCOMPLETE_UPLOAD_TTL_MS); - return Number.isFinite(value) && value >= 0 ? Math.floor(value) : 60 * 60 * 1000; -})(); - -const tusCleanupIntervalMs = (() => { - const value = Number(env.TUS_CLEANUP_INTERVAL_MS); - return Number.isFinite(value) && value >= 0 ? Math.floor(value) : 10 * 60 * 1000; -})(); - const uploads = { maxJsonBodyBytes, maxDirectUploadBytes: (() => { @@ -380,31 +392,18 @@ const uploads = { return Number.isFinite(parsed) && parsed > 0 ? parsed : 64 * 1024 * 1024 * 1024; })(), maxFilesPerRequest: env.MAX_FILES_PER_UPLOAD, + // One section for everything about taking an upload, as this repository has + // always had it: what a direct request may carry, how long a stalled one is + // waited for, where the resumable ones are kept, and the free space held back + // so a full volume never takes the database down with it. inactivityTimeoutMs: uploadInactivityTimeoutMs, tusUploadDir, tusIncompleteUploadTtlMs, tusCleanupIntervalMs, - // Free space kept in reserve when accepting writes, so a full volume never - // takes the database down with it. The trash gives space back before this - // floor is crossed. - storageReserveBytes: (() => { - const parsed = parseByteSize(env.UPLOAD_STORAGE_RESERVE); - // 0 is a real value — no reserve — not "unset". - return Number.isFinite(parsed) && parsed >= 0 ? parsed : 64 * 1024 * 1024; - })(), + storageReserveBytes: uploadStorageReserveBytes, }; // --- OnlyOffice --- -/** - * A secret for one purpose, derived from the session secret. - * - * The session secret signs every session cookie, so it is never handed to - * anything else; a purpose-specific value derived from it can be, and knowing - * it tells nothing about the one it came from. - */ -const deriveSecret = (purpose) => - crypto.createHmac('sha256', auth.sessionSecret).update(`nextexplorer:${purpose}`).digest('hex'); - const onlyoffice = { serverUrl: env.ONLYOFFICE_URL?.replace(/\/$/, '') || null, // Never hand the session signing secret to an external service. When no @@ -412,9 +411,12 @@ const onlyoffice = { // with the Document Server cannot be used to forge session cookies. // // This used to fall back to SESSION_SECRET verbatim, so a deployment that set - // the Document Server's JWT secret to that value worked without ever setting + // the Document Server's JWT_SECRET to that value worked without ever setting // ONLYOFFICE_SECRET. It no longer matches — see the warning emitted below. secret: env.ONLYOFFICE_SECRET || deriveSecret('onlyoffice'), + // Extra origins the Document Server may serve saved documents from, for + // deployments where it reports a different host than the one we call. + downloadOrigins: parseOriginList(env.ONLYOFFICE_DOWNLOAD_ORIGINS, 'ONLYOFFICE_DOWNLOAD_ORIGINS'), lang: env.ONLYOFFICE_LANG, forceSave: env.ONLYOFFICE_FORCE_SAVE, forceSaveTimeoutMs: Math.min(30000, Math.max(7000, env.ONLYOFFICE_FORCE_SAVE_TIMEOUT_MS)), @@ -422,52 +424,23 @@ const onlyoffice = { extensions: env.ONLYOFFICE_FILE_EXTENSIONS.split(',') .map((s) => s.trim().toLowerCase()) .filter(Boolean), - // Where a saved document may be fetched from, beyond the Document Server's - // own address: it sometimes reports itself under another host than the one it - // is called on, behind a proxy or inside a container network. - downloadOrigins: parseOriginList(env.ONLYOFFICE_DOWNLOAD_ORIGINS, 'ONLYOFFICE_DOWNLOAD_ORIGINS'), }; // Silent JWT mismatches surface to the user as "Document security token is not // correctly configured", with nothing in the logs pointing at the cause. if (onlyoffice.serverUrl && !env.ONLYOFFICE_SECRET) { console.warn( - '[Config] ONLYOFFICE_URL is set without ONLYOFFICE_SECRET. A derived secret is used, ' + - 'which will not match the Document Server unless its JWT secret is set to the same ' + + '[config] ONLYOFFICE_URL is set without ONLYOFFICE_SECRET. A derived secret is used, ' + + 'which will not match the Document Server unless its JWT_SECRET is set to the same ' + 'value. Set ONLYOFFICE_SECRET on both sides.' ); } -// --- Thumbnail access --- -// Thumbnails are served from /static, outside the authentication middleware, so -// the URL has to carry its own proof that somebody was cleared to see it. -// Derived from the session secret, so it lasts as long as that does; when even -// that could not be stored, a restart only means already-loaded pages fetch -// their thumbnails again through the API, which re-runs the access check. -// --- Activity log --- -// Defaults only, like the trash's. Off: on a machine one person uses, a record -// of what that person did all day is weight without a reader. -const activity = (() => { - const retentionDays = Number(env.ACTIVITY_RETENTION_DAYS); - return { - enabled: env.ACTIVITY_ENABLED === true, - retentionDays: - Number.isFinite(retentionDays) && retentionDays >= 1 - ? Math.min(3650, Math.round(retentionDays)) - : 90, - }; -})(); - -// --- Passkeys (WebAuthn) --- -// The relying party: the name a browser shows when it asks for a passkey, and -// the domain the key is bound to. The domain is left unset by default and taken -// from the request's own origin — a deployment reached by several names would -// otherwise bind every key to one of them. -const webauthn = { - rpId: env.WEBAUTHN_RP_ID, - rpName: env.WEBAUTHN_RP_NAME || 'NextExplorer', -}; - +// --- Thumbnails served outside /api --- +// Its own secret, so a leaked thumbnail URL cannot be turned into anything +// else. Derived from the session secret, so it lasts as long as that does; when +// even that could not be stored, a restart only means already-loaded pages +// refetch their thumbnails. const thumbnailAccess = { secret: deriveSecret('thumbnails'), }; @@ -488,41 +461,37 @@ const collabora = { .filter(Boolean), }; +// --- Editor --- const editor = { extensions: parseExtensionList(env.EDITOR_EXTENSIONS), maxFileSizeBytes: editorMaxFileSizeBytes, }; -// --- Terminal --- -const terminal = { - extensions: parseExtensionList(env.TERMINAL_FILE_EXTENSIONS), -}; - -// --- Favorites --- -const favorites = { - defaultIcon: env.FAVORITES_DEFAULT_ICON, -}; - -// --- Personal folders --- -const personal = { - userFolderNameOrder: parseUserFolderNameOrder(env.USER_FOLDER_NAME_ORDER), -}; - -// --- Hidden file patterns --- -const hiddenFiles = parseHiddenFilePatterns(env.HIDDEN_FILE_PATTERNS); - -// --- Shares --- -const shares = { - enabled: env.SHARES_ENABLED, - tokenLength: env.SHARES_TOKEN_LENGTH, - maxSharesPerUser: env.SHARES_MAX_PER_USER, - defaultExpiryDays: env.SHARES_DEFAULT_EXPIRY_DAYS, - guestSessionHours: env.SHARES_GUEST_SESSION_HOURS, - allowPasswordProtection: env.SHARES_ALLOW_PASSWORD, - allowAnonymous: env.SHARES_ALLOW_ANONYMOUS, -}; - -// --- Main Export --- +/** + * How much of a document the preview will render. + * + * Not the same question as what the editor will open, and the difference is + * why this is a setting of its own. The editor streams text into a code view; + * the preview parses the document, sanitises the HTML it produces and then + * hands the browser every node to lay out — all on the one thread the + * interface has. A six-megabyte markdown file opens in the editor and freezes + * the tab in the preview, on the same machine, from the same file. + * + * It was hard-coded before this, which meant someone who raised + * EDITOR_MAX_FILESIZE in good faith was refused at a number that appeared in + * no setting and no document. + * + * Generous by default because freezing is no longer the failure mode: the + * preview renders in slices of a frame and hands the browser back between + * them. What is left is the weight of the document in the tab, which is a + * reader's problem rather than an application's. And the preview reads through + * the editor's endpoint, so EDITOR_MAX_FILESIZE already caps what can reach + * it — this only bites when it is set lower than that. + */ +const previewMaxRenderBytes = (() => { + const parsed = parseByteSize(env.PREVIEW_MAX_RENDER_SIZE); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 16 * 1024 * 1024; +})(); // --- Archive extraction --- // Extensions the app is willing to offer for extraction, provided the local @@ -562,6 +531,23 @@ const archives = (() => { return Number.isFinite(parsed) && parsed > 0 ? parsed : 32 * 1024 * 1024 * 1024; })(), maxEntries: env.MAX_ARCHIVE_ENTRIES, + // A compound archive — a .tar.gz and its family — is two archives, and the + // inner one has to be decompressed before anything inside it can be named. + // Above this it is not: browsing a backup by unpacking it first would + // betray the whole point, and extracting it is the operation that exists + // for that. The number is the inner archive's own declared size, so the + // refusal comes before anything is written. + browseMaxBytes: (() => { + const parsed = parseByteSize(env.MAX_BROWSABLE_ARCHIVE_SIZE); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 2 * 1024 * 1024 * 1024; + })(), + // What those decompressed copies may take up altogether. They are a + // convenience and are made again whenever they are missing, so the least + // recently opened goes first when this is passed. + cacheMaxBytes: (() => { + const parsed = parseByteSize(env.ARCHIVE_CACHE_MAX_SIZE); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 8 * 1024 * 1024 * 1024; + })(), }; if (!raw) return { extensions: DEFAULT_ARCHIVE_EXTENSIONS, ...limits }; // 'zip,iso' replaces the default list; '+udf,squashfs' extends it. @@ -573,6 +559,24 @@ const archives = (() => { }; })(); +// --- Terminal --- +const terminal = { + extensions: parseExtensionList(env.TERMINAL_FILE_EXTENSIONS), +}; + +// --- Favorites --- +const favorites = { + defaultIcon: env.FAVORITES_DEFAULT_ICON, +}; + +// --- Personal folders --- +const personal = { + userFolderNameOrder: parseUserFolderNameOrder(env.USER_FOLDER_NAME_ORDER), +}; + +// --- Hidden file patterns --- +const hiddenFiles = parseHiddenFilePatterns(env.HIDDEN_FILE_PATTERNS); + // --- Trash --- // Defaults only: the values in force are the system settings, which start from // these. Out-of-range values fall back rather than failing the start. @@ -618,6 +622,30 @@ const versions = (() => { ), }; })(); +// --- Passkeys --- +// The relying party: the name a passkey is bound to. Left unset, each request +// answers for the hostname it arrived on, which is right until an installation +// is reached through two names — a passkey made on one is refused on the other, +// by design, and WEBAUTHN_RP_ID is how an operator settles which name counts. +const webauthn = { + rpId: env.WEBAUTHN_RP_ID, + rpName: env.WEBAUTHN_RP_NAME || 'NextExplorer', +}; + +// --- Activity log --- +// Defaults only, like the trash's. Off: on a machine one person uses, a record +// of what that person did all day is weight without a reader. +const activity = (() => { + const retentionDays = Number(env.ACTIVITY_RETENTION_DAYS); + return { + enabled: env.ACTIVITY_ENABLED === true, + retentionDays: + Number.isFinite(retentionDays) && retentionDays >= 1 + ? Math.min(3650, Math.round(retentionDays)) + : 90, + }; +})(); + // --- Folder size index --- const VALID_FOLDER_SIZE_MODES = new Set(['off', 'shallow', 'full']); const folderSizeMode = VALID_FOLDER_SIZE_MODES.has(env.FOLDER_SIZE_MODE) @@ -660,12 +688,57 @@ const folderSize = { : 2, rebuild: env.FOLDER_SIZE_REBUILD, }; +// --- Runtime diagnostics --- +const atLeast = (value, minimum, fallback) => + Number.isFinite(value) && value >= minimum ? value : fallback; + +const performanceDiagnostics = { + enabled: env.PERFORMANCE_DIAGNOSTICS_ENABLED, + intervalMs: atLeast(env.PERFORMANCE_DIAGNOSTICS_INTERVAL_MS, 5000, 15000), + logEveryInterval: env.PERFORMANCE_DIAGNOSTICS_LOG_EVERY_INTERVAL, + cpuThreshold: atLeast(env.PERFORMANCE_DIAGNOSTICS_CPU_THRESHOLD, 1, 75), + rssThresholdMb: atLeast(env.PERFORMANCE_DIAGNOSTICS_RSS_THRESHOLD_MB, 1, 768), + eventLoopDelayThresholdMs: atLeast(env.PERFORMANCE_DIAGNOSTICS_EVENT_LOOP_DELAY_MS, 1, 250), +}; +// --- Shares --- +const shares = { + enabled: env.SHARES_ENABLED, + tokenLength: env.SHARES_TOKEN_LENGTH, + maxSharesPerUser: env.SHARES_MAX_PER_USER, + defaultExpiryDays: env.SHARES_DEFAULT_EXPIRY_DAYS, + guestSessionHours: env.SHARES_GUEST_SESSION_HOURS, + allowPasswordProtection: env.SHARES_ALLOW_PASSWORD, + allowAnonymous: env.SHARES_ALLOW_ANONYMOUS, +}; + +// --- Demo sign-in --- +// A password served to anyone who loads the sign-in page. That is right for a +// public demo and wrong everywhere else, so it takes three deliberate things at +// once: demo mode on, and both halves of a credential named for this purpose. +// Any one of them missing and nothing is published — no partial state, no way +// to arrive here by setting something that meant something else. +const demoLoginRequested = Boolean(env.DEMO_LOGIN_EMAIL || env.DEMO_LOGIN_PASSWORD); +const demoLogin = + env.DEMO_MODE && env.DEMO_LOGIN_EMAIL && env.DEMO_LOGIN_PASSWORD + ? { email: env.DEMO_LOGIN_EMAIL, password: env.DEMO_LOGIN_PASSWORD } + : null; + +if (demoLogin) { + console.warn( + `[Config] Demo sign-in enabled: ${demoLogin.email} and its password are served to anyone who opens the sign-in page.` + ); +} else if (demoLoginRequested && !env.DEMO_MODE) { + console.warn( + '[Config] DEMO_LOGIN_EMAIL/DEMO_LOGIN_PASSWORD are set but DEMO_MODE is not enabled; no credentials are published and the sign-in form is not pre-filled.' + ); +} else if (demoLoginRequested) { + console.warn( + '[Config] DEMO_LOGIN_EMAIL and DEMO_LOGIN_PASSWORD must both be set; the sign-in form will not be pre-filled.' + ); +} +// --- Main Export --- module.exports = { - folderSize, - webauthn, - activity, - archives, port: env.PORT, address: env.ADDRESS, http: { @@ -693,6 +766,8 @@ module.exports = { corsOptions, auth, + demoLogin, + preview: { maxRenderBytes: previewMaxRenderBytes }, search: { deep: env.SEARCH_DEEP ?? true, @@ -765,17 +840,22 @@ module.exports = { }, thumbnails: { size: 200, quality: 70 }, - thumbnailAccess, uploads, onlyoffice, + thumbnailAccess, collabora, editor, terminal, favorites, shares, hiddenFiles, + folderSize, + performanceDiagnostics, + archives, trash, versions, + webauthn, + activity, VERSION_BOUNDS, features: { diff --git a/backend/src/errors/AppError.js b/backend/src/errors/AppError.js index e34a41fd2..f88da2be7 100644 --- a/backend/src/errors/AppError.js +++ b/backend/src/errors/AppError.js @@ -102,16 +102,6 @@ class RateLimitError extends AppError { } } -/** - * 500 Internal Server Error - for unexpected server errors - */ -class InternalError extends AppError { - constructor(message = 'Internal server error') { - super(message, 500); - this.name = 'InternalError'; - } -} - /** * 415 Unsupported Media Type - for unsupported file types */ @@ -122,7 +112,6 @@ class UnsupportedMediaTypeError extends AppError { } } -/** 507: the storage cannot hold what is being sent. */ /** * 503 Service Unavailable - something this server depends on did not answer * @@ -137,9 +126,12 @@ class ServiceUnavailableError extends AppError { } } +/** + * 507 Insufficient Storage - the destination cannot hold what was sent + */ class InsufficientStorageError extends AppError { constructor(message = 'Insufficient storage') { - super(message, 507, 'INSUFFICIENT_STORAGE'); + super(message, 507); this.name = 'InsufficientStorageError'; } } @@ -152,7 +144,6 @@ module.exports = { NotFoundError, ConflictError, RateLimitError, - InternalError, UnsupportedMediaTypeError, ServiceUnavailableError, InsufficientStorageError, diff --git a/backend/src/errors/errorCodes.js b/backend/src/errors/errorCodes.js index 33cef687b..4579836a0 100644 --- a/backend/src/errors/errorCodes.js +++ b/backend/src/errors/errorCodes.js @@ -6,10 +6,31 @@ const ErrorCodes = { // Authentication (401) AUTH_REQUIRED: 'AUTH_REQUIRED', AUTH_INVALID_CREDENTIALS: 'AUTH_INVALID_CREDENTIALS', + // The second step of a sign-in, so a screen can say "that code" rather than + // "those credentials" — the password was right, and saying otherwise sends + // somebody looking for the wrong mistake. + AUTH_INVALID_TOTP_CODE: 'AUTH_INVALID_TOTP_CODE', AUTH_ACCOUNT_LOCKED: 'AUTH_ACCOUNT_LOCKED', AUTH_PASSWORD_INCORRECT: 'AUTH_PASSWORD_INCORRECT', - AUTH_INVALID_TOTP_CODE: 'AUTH_INVALID_TOTP_CODE', + // A passkey that did not open anything: the wrong site, a stale question, a + // signature that does not hold, or a credential this server has never seen. + // One code for all of them, because telling them apart would answer which + // passkeys exist here to whoever asks. AUTH_PASSKEY_REJECTED: 'AUTH_PASSKEY_REJECTED', + // The browser cannot do this: no passkey support, or a page that is not on a + // secure origin, which is a setup problem rather than a wrong answer. + AUTH_PASSKEY_UNAVAILABLE: 'AUTH_PASSKEY_UNAVAILABLE', + + // An API token: not valid at all, or valid and reaching for something no + // token may reach. One code for every way the first can happen — see + // middleware/apiTokenAuth.js for why the reason is written in the log and + // never in the answer. + AUTH_TOKEN_INVALID: 'AUTH_TOKEN_INVALID', + AUTH_TOKEN_NOT_ALLOWED: 'AUTH_TOKEN_NOT_ALLOWED', + AUTH_TOKEN_READ_ONLY: 'AUTH_TOKEN_READ_ONLY', + + // Signing in at an identity provider. The two are told apart on purpose: one + // is answered in the configuration, the other by looking at the provider. AUTH_OIDC_NOT_CONFIGURED: 'AUTH_OIDC_NOT_CONFIGURED', AUTH_OIDC_PROVIDER_UNAVAILABLE: 'AUTH_OIDC_PROVIDER_UNAVAILABLE', diff --git a/backend/src/errors/example-usage.js b/backend/src/errors/example-usage.js deleted file mode 100644 index 7e47cfd8f..000000000 --- a/backend/src/errors/example-usage.js +++ /dev/null @@ -1,248 +0,0 @@ -/** - * Example Usage of Error Handling System - * - * This file demonstrates how to use the error handling system in your routes. - * DO NOT import this file - it's just for reference/documentation. - */ - -const express = require('express'); -const asyncHandler = require('../utils/asyncHandler'); -const { - ValidationError, - UnauthorizedError, - ForbiddenError, - NotFoundError, - ConflictError, - RateLimitError, - InternalError, - UnsupportedMediaTypeError, -} = require('./AppError'); - -const router = express.Router(); - -// Example 1: Basic async route with validation -router.post( - '/example/create', - asyncHandler(async (req, res) => { - const { name, email } = req.body; - - // Validation - if (!name || !email) { - throw new ValidationError('Name and email are required'); - } - - if (!email.includes('@')) { - throw new ValidationError('Invalid email format'); - } - - // Simulate creation - const item = { id: 1, name, email }; - res.status(201).json({ success: true, item }); - }) -); - -// Example 2: Authentication check -router.get( - '/example/protected', - asyncHandler(async (req, res) => { - const user = req.user; - - if (!user) { - throw new UnauthorizedError('Please login to access this resource'); - } - - res.json({ message: 'Protected data', user }); - }) -); - -// Example 3: Permission check -router.delete( - '/example/admin-only', - asyncHandler(async (req, res) => { - const user = req.user; - - if (!user) { - throw new UnauthorizedError('Authentication required'); - } - - if (!user.roles?.includes('admin')) { - throw new ForbiddenError('Admin access required'); - } - - res.json({ message: 'Admin action completed' }); - }) -); - -// Example 4: Resource lookup with 404 -router.get( - '/example/:id', - asyncHandler(async (req, res) => { - const { id } = req.params; - - // Simulate database lookup - const item = null; // await db.findById(id); - - if (!item) { - throw new NotFoundError(`Item with id ${id} not found`); - } - - res.json(item); - }) -); - -// Example 5: Duplicate resource handling -router.post( - '/example/register', - asyncHandler(async (req, res) => { - // eslint-disable-next-line no-unused-vars - const { email } = req.body; - - // Simulate checking for existing user - const existingUser = true; // await db.findByEmail(email); - - if (existingUser) { - throw new ConflictError('User with this email already exists'); - } - - res.status(201).json({ message: 'User created' }); - }) -); - -// Example 6: Rate limiting -router.post( - '/example/login', - asyncHandler(async (req, res) => { - const attempts = 11; // Simulate too many attempts - - if (attempts > 10) { - throw new RateLimitError('Too many login attempts. Please try again later.', 900); // retry after 15 minutes - } - - res.json({ message: 'Login successful' }); - }) -); - -// Example 7: File upload validation -router.post( - '/example/upload', - asyncHandler(async (req, res) => { - const fileType = req.file?.mimetype; - - const allowedTypes = ['image/jpeg', 'image/png', 'image/gif']; - - if (!allowedTypes.includes(fileType)) { - throw new UnsupportedMediaTypeError('Only JPEG, PNG, and GIF images are supported'); - } - - res.json({ message: 'File uploaded successfully' }); - }) -); - -// Example 8: Validation with field-specific details -router.post( - '/example/complex-validation', - asyncHandler(async (req, res) => { - const { username, password, email } = req.body; - const errors = {}; - - if (!username || username.length < 3) { - errors.username = 'Username must be at least 3 characters'; - } - - if (!password || password.length < 8) { - errors.password = 'Password must be at least 8 characters'; - } - - if (!email || !email.includes('@')) { - errors.email = 'Valid email is required'; - } - - if (Object.keys(errors).length > 0) { - throw new ValidationError('Validation failed', errors); - } - - res.json({ message: 'Validation passed' }); - }) -); - -// Example 9: Catching specific errors and re-throwing as custom errors -router.post( - '/example/database-operation', - asyncHandler(async () => { - try { - // Simulate database operation - // await db.query('INSERT INTO users...'); - throw new Error('DUPLICATE_KEY'); - } catch (error) { - if (error.message === 'DUPLICATE_KEY') { - throw new ConflictError('Record already exists'); - } - if (error.message === 'CONNECTION_FAILED') { - throw new InternalError('Database connection failed'); - } - // Re-throw unknown errors - throw error; - } - }) -); - -// Example 10: Manual error handling (when you need more control) -router.post('/example/manual', async (req, res, next) => { - try { - // eslint-disable-next-line no-undef - const result = await someComplexOperation(); - - // Custom response format - res.json({ - success: true, - data: result, - metadata: { timestamp: new Date() }, - }); - } catch (error) { - // Transform error if needed - if (error.code === 'CUSTOM_ERROR') { - return next(new ValidationError(error.message)); - } - // Pass through to error handler - next(error); - } -}); - -// Example of what the frontend receives: - -/* -Success Response: -{ - "success": true, - "item": { ... } -} - -Error Response (ValidationError): -{ - "success": false, - "error": { - "message": "Validation failed", - "statusCode": 400, - "requestId": "a1b2c3d4-e5f6-4789-a0b1-c2d3e4f5g6h7", - "timestamp": "2025-01-18T12:34:56.789Z", - "details": { - "email": "Invalid format", - "password": "Too short" - } - } -} - -Error Response (RateLimitError): -{ - "success": false, - "error": { - "message": "Too many login attempts. Please try again later.", - "statusCode": 429, - "requestId": "a1b2c3d4-e5f6-4789-a0b1-c2d3e4f5g6h7", - "timestamp": "2025-01-18T12:34:56.789Z", - "retryAfter": 900 - } -} -*/ - -module.exports = router; diff --git a/backend/src/middleware/authMiddleware.js b/backend/src/middleware/authMiddleware.js index 11fe50db9..f92168c5f 100644 --- a/backend/src/middleware/authMiddleware.js +++ b/backend/src/middleware/authMiddleware.js @@ -5,6 +5,156 @@ const logger = require('../utils/logger'); const { applyApiToken } = require('./apiTokenAuth'); const apiTokens = require('../services/apiTokens'); +/** + * Whoever a request arrives as, when authentication is switched off. + * + * A deployment behind its own front door runs with no accounts at all, and the + * rest of the application asks who is calling — so it has to be told + * something. + */ +const ANONYMOUS_USER = { + id: 'anonymous', + username: 'anonymous', + email: 'anonymous@local', + displayName: 'Anonymous User', + roles: ['admin'], +}; + +/** + * A share link is opened by someone with no account, which is the point of it. + * + * Browsing *inside* one is deliberately not on that list: it needs either an + * account or a guest session, which is the only proof the password was typed. + */ +const isPublicShareRoute = (requestPath) => + requestPath.startsWith('/api/share/') && !requestPath.includes('/browse/'); + +/** + * The paths an editor's server calls back on, which carry their own signed + * token and are checked by the route. + * + * Open only while the integration is configured — otherwise they are + * unauthenticated endpoints for no reason. + */ +const isConfiguredIntegrationCallback = (requestPath) => { + try { + const { onlyoffice, collabora } = require('../config/index'); + + if ( + onlyoffice?.serverUrl && + (requestPath.startsWith('/api/onlyoffice/file') || + requestPath.startsWith('/api/onlyoffice/callback')) + ) { + return true; + } + + if (collabora?.url && collabora?.secret && requestPath.startsWith('/api/collabora/wopi/')) { + return true; + } + } catch (_) { + /* an integration that cannot be read about is an integration that is off */ + } + + return false; +}; + +/** What is answered before anyone is asked to identify themselves. */ +const needsNoIdentity = (req, requestPath) => { + if (!requestPath.startsWith('/api')) return true; + if (req.method === 'OPTIONS') return true; + // Feature flags are needed for plugin registration, and the login page draws + // its branding before anyone has signed in. Neither carries anything private. + if (requestPath.startsWith('/api/features')) return true; + if (requestPath === '/api/branding') return true; + // The API's description says what the published documentation says. + if (requestPath === '/api/openapi.json') return true; + return isConfiguredIntegrationCallback(requestPath); +}; + +/** + * Attach the visitor's guest session, when they have a valid one. + * + * Enrichment and not a gate: it decides nothing on its own, and every access + * check downstream reads it for itself. + */ +const attachGuestSession = async (req, requestPath) => { + const guestSessionId = req.headers['x-guest-session'] || req.cookies?.guestSession; + + if (!guestSessionId) { + if (requestPath.startsWith('/api/preview') || requestPath.startsWith('/api/thumbnails')) { + logger.debug( + { + path: requestPath, + hasHeader: Boolean(req.headers['x-guest-session']), + hasCookie: Boolean(req.cookies?.guestSession), + cookies: Object.keys(req.cookies || {}), + }, + 'No guest session for preview/thumbnail request' + ); + } + return; + } + + logger.debug( + { + source: req.headers['x-guest-session'] ? 'header' : 'cookie', + sessionId: guestSessionId, + path: requestPath, + cookies: Object.keys(req.cookies || {}), + }, + 'Guest session found' + ); + + try { + const { + getGuestSession, + isGuestSessionValid, + updateGuestSessionActivity, + } = require('../services/guestSessionService'); + + if (!(await isGuestSessionValid(guestSessionId))) { + logger.debug({ sessionId: guestSessionId }, 'Guest session invalid or expired'); + return; + } + + req.guestSession = await getGuestSession(guestSessionId); + await updateGuestSessionActivity(guestSessionId); + logger.debug( + { sessionId: guestSessionId, shareId: req.guestSession.shareId, path: requestPath }, + 'Guest session validated' + ); + } catch (err) { + logger.debug({ err }, 'Guest session validation failed'); + } +}; + +/** Whether this request carries a session, and of which kind. */ +const sessionsOn = (req) => ({ + throughIdentityProvider: Boolean( + req.oidc && typeof req.oidc.isAuthenticated === 'function' && req.oidc.isAuthenticated() + ), + throughLocalAccount: Boolean(req.session && req.session.localUserId), +}); + +/** + * Attach the account this session belongs to. + * + * @throws {ForbiddenError} when the identity provider vouches for somebody this + * installation has no account for and does not create accounts on the fly. + * A guest or a share link is let through without one; anything else is not, + * because every check below asks what this user may do. + */ +const attachAuthenticatedUser = async (req, { throughIdentityProvider, requestPath }) => { + const user = await getRequestUser(req); + if (user) req.user = user; + if (user || !throughIdentityProvider) return; + + if ((auth?.oidc?.autoCreateUsers ?? true) !== false) return; + if (isPublicShareRoute(requestPath) || req.guestSession) return; + + throw new ForbiddenError('Profile does not exist.'); +}; + /** * Say, once in a while, that a token this server knows was refused. * @@ -18,7 +168,6 @@ const reportRefusal = async (req, refused) => { if (!refused?.tokenId) return; if (!apiTokens.shouldReportRefusal(refused.tokenId)) return; try { - // eslint-disable-next-line global-require const activityLog = require('../services/activityLog'); await activityLog.record({ action: 'sign-in', @@ -82,216 +231,92 @@ const authenticateWithToken = async (req) => { return { authenticated: true }; }; +/** + * The path, spelled the one way every decision below is written for. + * + * Express matches routes without regard to case, so `/API/Files/list` reaches + * the same handler `/api/files/list` does. Every check in this file compares a + * prefix, and a prefix compared as it arrived does not match that — so + * `/API/anything` answered the very first question ("does this need an + * identity?") with *no*, and walked past authentication entirely. It was not a + * way in: no session was attached either, so the routes that ask who is + * calling refused. It was worse than that — it was a gate that did not hold, + * in front of routes that are entitled to assume it did. + * + * Folded once, here, rather than at each of the comparisons below, because the + * one that gets forgotten is the one that matters. + */ +const pathForDecisions = (req) => (req.path || '').toLowerCase(); + +/** + * Who is calling, and whether they may be here at all. + * + * Four questions in order, each answerable on its own: what needs no identity, + * what runs with no accounts at all, what the visitor's guest session says, + * and finally what account this session belongs to. Anything that reaches the + * end is refused. + * + * It was one function of forty-six paths, in front of every request the + * application serves. + */ const authMiddleware = async (req, res, next) => { - // Express matches routes without regard to case, so `/API/volumes` reaches - // the same handler as `/api/volumes`. Every decision below compares the path - // with a lower-case prefix, and a path compared as it arrived was answered - // "not an API route" and waved through with no identity at all. Folded once, - // here, rather than at each comparison, because the one that gets forgotten - // is the one that matters. - const requestPath = (req.path || '').toLowerCase(); - const apiRoute = requestPath.startsWith('/api'); - const isAuthRoute = requestPath.startsWith('/api/auth'); - // Allow public share access routes (single share with token: /api/share/:token/*) - // Note: /api/shares/* are management endpoints and require authentication. - // Important: exclude /api/share/:token/browse/* so browse requests still require - // an authenticated user and/or a valid guest session. - const isPublicShareRoute = - requestPath.startsWith('/api/share/') && !requestPath.includes('/browse/'); - - if (!apiRoute) { - next(); - return; - } + const requestPath = pathForDecisions(req); - if (req.method === 'OPTIONS') { + if (needsNoIdentity(req, requestPath)) { next(); return; } if (auth.enabled === false) { - // Inject a synthetic anonymous user for features that require user context - req.user = { - id: 'anonymous', - username: 'anonymous', - email: 'anonymous@local', - displayName: 'Anonymous User', - roles: ['admin'], - }; - next(); - return; - } - - // Allow public feature flags endpoint (contains no sensitive data) - // its needed for plugin registrations - if (requestPath === '/api/features' || requestPath.startsWith('/api/features')) { - next(); - return; - } - - // The API's description says what the published documentation says, and - // carries nothing private: a client reads it before it has a token. - if (requestPath === '/api/openapi.json') { - next(); - return; - } - - // Allow public branding endpoint (no sensitive data, used on login page) - if (requestPath === '/api/branding') { - next(); - return; - } - - // Allow ONLYOFFICE server callbacks and file fetches (token-guarded in route) - // Only when ONLYOFFICE integration is enabled - let isOnlyofficeGuest = false; - try { - const { onlyoffice } = require('../config/index'); - if (onlyoffice && onlyoffice.serverUrl) { - isOnlyofficeGuest = - requestPath.startsWith('/api/onlyoffice/file') || - requestPath.startsWith('/api/onlyoffice/callback'); - } - } catch (_) { - /* ignore */ - } - - if (isOnlyofficeGuest) { + req.user = { ...ANONYMOUS_USER }; next(); return; } - // Allow Collabora WOPI endpoints (token-guarded in route) - // Only when Collabora integration is enabled - let isCollaboraGuest = false; - try { - const { collabora } = require('../config/index'); - if (collabora && collabora.url && collabora.secret) { - isCollaboraGuest = requestPath.startsWith('/api/collabora/wopi/'); - } - } catch (_) { - /* ignore */ - } - - if (isCollaboraGuest) { - next(); + // A script's credential, before anything a browser carries. A request that + // presents a token is answered as that token, with that token's scope — a + // session cookie that happened to ride along does not widen it. + const withToken = await authenticateWithToken(req); + if (withToken.error) { + next(withToken.error); return; } - - // A token, if one was presented. Asked before the session, because a script - // sending one is saying which credential it wants judged: a stale cookie in - // the same jar must not be what answers for it. - const byToken = await authenticateWithToken(req); - if (byToken.error) { - next(byToken.error); - return; - } - if (byToken.authenticated) { + if (withToken.authenticated) { next(); return; } - // Check for guest session (on all routes) - const guestSessionId = req.headers['x-guest-session'] || req.cookies?.guestSession; - if (guestSessionId) { - logger.debug( - { - source: req.headers['x-guest-session'] ? 'header' : 'cookie', - sessionId: guestSessionId, - path: requestPath, - cookies: Object.keys(req.cookies || {}), - }, - 'Guest session found' - ); + await attachGuestSession(req, requestPath); - try { - const { - getGuestSession, - isGuestSessionValid, - updateGuestSessionActivity, - } = require('../services/guestSessionService'); - if (await isGuestSessionValid(guestSessionId)) { - const session = await getGuestSession(guestSessionId); - req.guestSession = session; - // Update activity timestamp - await updateGuestSessionActivity(guestSessionId); - - logger.debug( - { - sessionId: guestSessionId, - shareId: session.shareId, - path: requestPath, - }, - 'Guest session validated' - ); - } else { - logger.debug({ sessionId: guestSessionId }, 'Guest session invalid or expired'); - } - } catch (err) { - logger.debug({ err }, 'Guest session validation failed'); - } - } else if (requestPath.startsWith('/api/preview') || requestPath.startsWith('/api/thumbnails')) { - logger.debug( - { - path: requestPath, - hasHeader: !!req.headers['x-guest-session'], - hasCookie: !!req.cookies?.guestSession, - cookies: Object.keys(req.cookies || {}), - }, - 'No guest session for preview/thumbnail request' - ); - } - - if (isAuthRoute) { + if (requestPath.startsWith('/api/auth')) { next(); return; } - // Accept either EOC session or local session - const isEocAuthenticated = Boolean( - req.oidc && typeof req.oidc.isAuthenticated === 'function' && req.oidc.isAuthenticated() - ); - const hasLocalSession = Boolean(req.session && req.session.localUserId); - if (isEocAuthenticated || hasLocalSession) { + const { throughIdentityProvider, throughLocalAccount } = sessionsOn(req); + if (throughIdentityProvider || throughLocalAccount) { try { - const user = await getRequestUser(req); - if (user) req.user = user; - if (isEocAuthenticated && !user && (auth?.oidc?.autoCreateUsers ?? true) === false) { - // Allow guest/public access without an app user profile. - if (isPublicShareRoute || req.guestSession) { - next(); - return; - } - throw new ForbiddenError('Profile does not exist.'); - } + await attachAuthenticatedUser(req, { throughIdentityProvider, requestPath }); } catch (err) { if (err && err.isOperational) { next(err); return; } - /* ignore */ + /* anything else is a lookup that failed; the checks below still apply */ } - // A guest session stays beside the user. For a password-protected share it - // is the proof that this account typed the password, and dropping it here - // refused a signed-in visitor on every request after they had. Every - // access check already prefers the user when both are present. - - next(); - return; - } - - // Public share routes should be accessible without authentication, but we still - // tried to attach an authenticated user above (if present) to support - // user-specific shares opened via share links. - if (isPublicShareRoute) { + // The guest session is kept alongside the user: it is the only proof that + // this visitor typed the password of a protected share. Dropping it here + // made that check unsatisfiable for signed-in visitors. Each access check + // decides on its own, and every one of them prefers the user when both are + // present, so a stale guest session can no longer shadow user access. next(); return; } - // Allow access if valid guest session exists (for share paths on browse, files, etc.) - if (req.guestSession) { - logger.debug('Allowing request with guest session'); + // Reached with no account: a share link is still open to anyone, and a guest + // session is what a visitor to a protected one carries instead. + if (isPublicShareRoute(requestPath) || req.guestSession) { next(); return; } diff --git a/backend/src/middleware/errorHandler.js b/backend/src/middleware/errorHandler.js index f57a7bee7..27b1cb18b 100644 --- a/backend/src/middleware/errorHandler.js +++ b/backend/src/middleware/errorHandler.js @@ -2,7 +2,7 @@ const multer = require('multer'); const logger = require('../utils/logger'); const { v4: uuidv4 } = require('uuid'); const { sanitizeLogUrl } = require('../utils/logSanitizer'); -const { directories } = require('../config/index'); +const { directories, uploads } = require('../config/index'); /** * Strip server-side absolute paths out of a message shown to a client. @@ -24,60 +24,17 @@ const REDACTED_ROOTS = [ const sanitizeClientMessage = (message) => { if (typeof message !== 'string' || !message) return message; - return REDACTED_ROOTS.reduce((text, root) => text.split(root).join('…'), message).replace( - // Any remaining absolute path (a temporary directory, say) keeps only its name. - /(^|[\s'"(])\/(?:[\w.@ -]+\/)+([\w.@ -]+)/g, - '$1…/$2' - ); -}; - -/** - * What an upload that met one of multer's limits is told. - * - * These are refusals of what the client sent, not failures of the server: a - * file over the size ceiling, more files than one request may carry. Without - * this they reached the client as a 500. - */ -const MULTIPART_REFUSALS = { - LIMIT_FILE_SIZE: [413, 'The file is larger than this server accepts.'], - LIMIT_FILE_COUNT: [413, 'The request carries more files than this server accepts at once.'], - LIMIT_PART_COUNT: [413, 'The request carries more parts than this server accepts at once.'], - LIMIT_FIELD_COUNT: [400, 'The request carries more form fields than this server reads.'], - LIMIT_FIELD_KEY: [400, 'A form field name is longer than this server reads.'], - LIMIT_FIELD_VALUE: [400, 'A form field is longer than this server reads.'], - LIMIT_UNEXPECTED_FILE: [400, 'A file was sent in a field this request does not take.'], -}; - -const multipartRefusal = (err) => { - if (!(err instanceof multer.MulterError)) return null; - const [statusCode, sentence] = MULTIPART_REFUSALS[err.code] || [400, err.message]; - // A route that knows its own limit says so: `explainMultipartRefusals` puts that - // sentence on the error, and it was being built and then thrown away — "the file - // is larger than this server accepts" where "a logo can be at most 2 MB" was - // ready to be said. - return [statusCode, err.clientMessage || sentence]; -}; - -/** - * A write the storage itself refused: the mount is read-only, or the folder - * belongs to somebody the server does not run as. - * - * Both surfaced as a 500 carrying the system's own words — `EROFS: read-only - * file system, mkdir`, `EACCES: permission denied, mkdir` — for what is - * neither a fault of the server nor something a retry would change, and with - * an absolute path from inside the container in the message. The listing now - * says beforehand that such a folder cannot be written in; this is the answer - * for whatever still tries. - */ -const STORAGE_REFUSALS = { - EROFS: 'This storage is read-only: nothing can be written here.', - EACCES: 'The server is not allowed to write in this folder.', - EPERM: 'The server is not allowed to write in this folder.', -}; - -const storageRefusal = (err) => { - const sentence = STORAGE_REFUSALS[err?.code]; - return sentence ? [403, sentence] : null; + return REDACTED_ROOTS.reduce( + (text, root) => text.split(root).join('…'), + message + // Any remaining absolute path (e.g. a temp dir) keeps only its basename. + // + // Directory segments must not contain spaces: allowing them let a single + // match run from one path, across the words between, and into the next — + // "copy /srv/a.txt to /srv/b.txt" came back as "copy …/b.txt". What has to + // stay hidden is the server's directory layout, not the file name the user + // typed themselves, so the basename still allows them. + ).replace(/(?<=^|[\s'"(])\/(?:[\w.@-]+\/)+([\w.@ -]+)/g, '…/$1'); }; /** @@ -124,6 +81,80 @@ const clearOidcSessionCookies = (req, res) => { * * Handles both operational errors (AppError instances) and unexpected errors */ +/** + * What multer refuses, and what kind of refusal it is. + * + * Its errors carry a code and no status, so every one of them reached the + * client as a 500 and the log as a server error: a logo of two megabytes and a + * byte, an upload over MAX_DIRECT_UPLOAD_SIZE. They are the request's doing. + * + * 413 where the request carries more than the server takes — a file too large, + * more files or parts than one request may hold. That is something the sender + * can act on by sending less, and it is what a proxy refusing a body answers + * too. 400 where the form is not the one the route reads: a file in a field it + * does not take, a field without a name, more or longer fields than the + * application ever sends. Nothing the interface sends comes near those limits, + * so meeting one is a malformed request rather than a large one. + * + * The sentence here is the general one; a route that knows its limits gives a + * better one through `explainMultipartRefusals`. + */ +const MULTIPART_REFUSALS = { + LIMIT_FILE_SIZE: [413, 'The file is larger than this server accepts.'], + LIMIT_FILE_COUNT: [413, 'The request carries more files than this server accepts at once.'], + LIMIT_PART_COUNT: [413, 'The request carries more parts than this server accepts at once.'], + LIMIT_FIELD_COUNT: [400, 'The request carries more form fields than this server reads.'], + LIMIT_FIELD_KEY: [400, 'A form field name is longer than this server reads.'], + LIMIT_FIELD_VALUE: [400, 'A form field is longer than this server reads.'], + LIMIT_FIELD_NESTING: [400, 'A form field name is nested deeper than this server reads.'], + LIMIT_UNEXPECTED_FILE: [400, 'A file was sent in a field this request does not take.'], + MISSING_FIELD_NAME: [400, 'A form field was sent without a name.'], +}; + +/** + * A write the storage itself refused: the mount is read-only, or the folder + * belongs to somebody the server does not run as. + * + * Both surfaced as a 500 carrying the system's own words — `EROFS: read-only + * file system, mkdir`, `EACCES: permission denied, mkdir` — for what is + * neither a fault of the server nor something a retry would change, and with + * an absolute path from inside the container in the message. The listing now + * says beforehand that such a folder cannot be written in; this is the answer + * for whatever still tries. + */ +const STORAGE_REFUSALS = { + EROFS: 'This storage is read-only: nothing can be written here.', + EACCES: 'The server is not allowed to write in this folder.', + EPERM: 'The server is not allowed to write in this folder.', +}; + +const storageRefusal = (err) => { + const message = STORAGE_REFUSALS[err?.code]; + return message ? { statusCode: 403, message } : null; +}; + +const multipartRefusal = (err) => { + if (!(err instanceof multer.MulterError)) return null; + const [statusCode, sentence] = MULTIPART_REFUSALS[err.code] || [400, err.message]; + return { statusCode, message: err.clientMessage || sentence }; +}; + +const describeError = (err) => { + const refusal = multipartRefusal(err) || storageRefusal(err); + if (refusal) return refusal.message; + + if (err?.type === 'entity.too.large') { + const megabytes = (uploads?.maxJsonBodyBytes ?? 0) / (1024 * 1024); + const limit = + megabytes >= 1 + ? `${Math.round(megabytes * 10) / 10} MB` + : `${uploads?.maxJsonBodyBytes} bytes`; + return `This request is larger than the ${limit} this server accepts. Raise MAX_JSON_BODY_SIZE to accept more.`; + } + + return err?.message || 'An unexpected error occurred'; +}; + // Express only recognizes error middleware when it has 4 args: (err, req, res, next) // eslint-disable-next-line no-unused-vars const errorHandler = (err, req, res, next) => { @@ -131,23 +162,27 @@ const errorHandler = (err, req, res, next) => { const requestId = uuidv4(); // Determine if this is an operational error (expected) or programmer error (unexpected) - const refusal = multipartRefusal(err) || storageRefusal(err); - const isOperational = Boolean(refusal) || err.isOperational || false; - const statusCode = refusal ? refusal[0] : err.statusCode || err.status || 500; - const message = refusal ? refusal[1] : err.message || 'An unexpected error occurred'; + const isOperational = err.isOperational || false; + const statusCode = + multipartRefusal(err)?.statusCode || + storageRefusal(err)?.statusCode || + err.statusCode || + err.status || + 500; + const message = describeError(err); // For OIDC callback navigations, redirect back into the SPA so the login screen can show the error. // Otherwise, the browser will render the JSON payload as a standalone error page. if (!res.headersSent && isOidcDocumentRequest(req)) { clearOidcSessionCookies(req, res); - // Same redaction as the JSON body: this one lands in the address bar, browser - // history and every proxy log along the way, so a raw server path here travels - // further than it would in a response body. + // Same redaction as the JSON body: this one lands in the address bar, + // browser history and every proxy log along the way, so a raw server path + // here travels further than it would in a response body. const query = new URLSearchParams({ error: sanitizeClientMessage(message) }); - // The code travels beside the sentence, never instead of it. The screen says - // what the codes it knows mean in the reader's own language — which is how "the - // provider could not be reached" stops reading as "OIDC is not configured" — - // and falls back to the sentence for the ones it does not. + // The code travels beside the sentence, never instead of it. The screen + // says what the codes it knows mean in the reader's own language — which is + // how "the provider could not be reached" stops reading as "OIDC is not + // configured" — and falls back to the sentence for the ones it does not. if (err.code) query.set('error_code', String(err.code)); res.setHeader('Cache-Control', 'no-store'); res.redirect(302, `/auth/login?${query.toString()}`); @@ -158,7 +193,6 @@ const errorHandler = (err, req, res, next) => { const errorContext = { requestId, method: req.method, - // The address as the log may keep it: a token in the query string is not. url: sanitizeLogUrl(req.originalUrl), statusCode, isOperational, @@ -221,14 +255,11 @@ const errorHandler = (err, req, res, next) => { */ const notFoundHandler = (req, res, next) => { const NotFoundError = require('../errors/AppError').NotFoundError; - next(new NotFoundError(`Route ${req.method} ${req.originalUrl} not found`)); + next(new NotFoundError(`Route ${req.method} ${sanitizeLogUrl(req.originalUrl)} not found`)); }; module.exports = { + sanitizeClientMessage, errorHandler, notFoundHandler, - // Used by the routes that stream their progress: an NDJSON stream has already - // answered 200 by the time something fails, so it says why in a line of its - // own rather than through the error handler — and says it the same way. - sanitizeClientMessage, }; diff --git a/backend/src/middleware/logging.js b/backend/src/middleware/logging.js index 354a19645..c872ff5ed 100644 --- a/backend/src/middleware/logging.js +++ b/backend/src/middleware/logging.js @@ -12,9 +12,6 @@ const configureHttpLogging = (app) => { app.use( pinoHttp({ logger: logger.child({ context: 'http' }), - // pino-http logs the whole request by default: the URL with whatever - // token its query string carries, and every header, the session cookie - // included. serializers: { req: sanitizeHttpRequestForLog, }, diff --git a/backend/src/openapi/paths/files.js b/backend/src/openapi/paths/files.js index 33ba89711..7ca673aeb 100644 --- a/backend/src/openapi/paths/files.js +++ b/backend/src/openapi/paths/files.js @@ -113,7 +113,7 @@ module.exports = { access: 'account', body: body(obj({ items: arrayOf(itemRef), destination: str() }, ['items', 'destination'])), responses: { - 200: json(transferDone, 'What landed where, once it has all landed.'), + 200: ndjson(transferDone, 'Progress as it goes, and what landed where.'), ...errors(400, 401, 403, 404, 507), }, }), @@ -127,11 +127,21 @@ module.exports = { access: 'account', body: body(obj({ items: arrayOf(itemRef), destination: str() }, ['items', 'destination'])), responses: { - 200: json(transferDone, 'What landed where, once it has all landed.'), + 200: ndjson(transferDone, 'Progress as it goes, and what landed where.'), ...errors(400, 401, 403, 404, 507), }, }), }, + '/api/files/recent-destinations': { + get: op({ + id: 'listRecentDestinations', + summary: 'Folders this account recently copied or moved into', + description: 'Only those it can still reach.', + tag: TAG, + access: 'account', + responses: { 200: json(obj({ items: arrayOf(str()) }, ['items'])), ...errors(401) }, + }), + }, '/api/files/delete-impact': { post: op({ id: 'describeDeletion', diff --git a/backend/src/openapi/paths/integrations.js b/backend/src/openapi/paths/integrations.js index 1b29a54ea..7cab69ee9 100644 --- a/backend/src/openapi/paths/integrations.js +++ b/backend/src/openapi/paths/integrations.js @@ -58,8 +58,7 @@ module.exports = { documentServerUrl: str(), config: loose('What `DocsAPI.DocEditor` takes, `token` included.'), forceSaveSessionId: str(), - // The editing session this open belongs to, which every later - // call about the document carries back. + // The same value under the name this answer has always carried. editorSessionId: str(), autoSaveIntervalMs: num(), }, @@ -179,6 +178,16 @@ module.exports = { }, }), }, + '/api/onlyoffice/storage-file': { + post: op({ + id: 'pickFileForOnlyoffice', + summary: 'A file the editor inserts or compares, signed for it', + tag: OO, + access: 'session', + body: body(obj({ path, c: str('The editor’s command.') }, ['path'])), + responses: { 200: json(loose()), ...errors(400, 401, 403, 404) }, + }), + }, '/api/onlyoffice/users': { get: op({ id: 'listOnlyofficeUsers', @@ -358,14 +367,4 @@ module.exports = { }, }), }, - '/api/onlyoffice/storage-file': { - post: op({ - id: 'pickFileForOnlyoffice', - summary: 'A file the editor inserts or compares, signed for it', - tag: OO, - access: 'session', - body: body(obj({ path, c: str('The editor’s command.') }, ['path'])), - responses: { 200: json(loose()), ...errors(400, 401, 403, 404) }, - }), - }, }; diff --git a/backend/src/routes/archive.js b/backend/src/routes/archive.js index 8698c2b19..3d898707b 100644 --- a/backend/src/routes/archive.js +++ b/backend/src/routes/archive.js @@ -289,7 +289,7 @@ router.post( // The staging directory is this route's own, created a moment ago under // the cache: it never holds anything anybody put there, so it does not // go through the trash. - // eslint-disable-next-line no-restricted-properties + await fs.rm(stagingAbsolutePath, { recursive: true, force: true }); writeEvent({ type: 'done', @@ -302,7 +302,7 @@ router.post( { err: error, archive: archive.relativePath }, 'Extracting from an archive failed' ); - // eslint-disable-next-line no-restricted-properties + await fs.rm(stagingAbsolutePath, { recursive: true, force: true }); // What this placed, and only that: a file somebody saved into a placed // folder in the meantime stays, with the folders holding it. diff --git a/backend/src/routes/auth.js b/backend/src/routes/auth.js index 1df37e6ca..03af9b840 100644 --- a/backend/src/routes/auth.js +++ b/backend/src/routes/auth.js @@ -14,6 +14,7 @@ const logger = require('../utils/logger'); const { countUsers, createLocalUser, + getById, attemptLocalLogin, changeLocalPassword, addLocalPassword, @@ -28,15 +29,13 @@ const { twoFactorStatus, verifySecondFactor, } = require('../services/users'); -const { incrementFailedAttempts, clearLock, isLocked } = require('../services/users/lockout'); -const { issueCode, redeemCode, isValidChallenge } = require('../services/oidcMobileBridge'); -const rateLimit = require('express-rate-limit'); -const asyncHandler = require('../utils/asyncHandler'); -const { startAuthenticatedSession } = require('../utils/authenticatedSession'); const passkeys = require('../services/users/passkeys'); -const activityLog = require('../services/activityLog'); const apiTokens = require('../services/apiTokens'); +const activityLog = require('../services/activityLog'); const { WebAuthnError } = require('../utils/webauthn'); +const { issueCode, redeemCode, isValidChallenge } = require('../services/oidcMobileBridge'); +const rateLimit = require('express-rate-limit'); +const asyncHandler = require('../utils/asyncHandler'); const { ValidationError, UnauthorizedError, @@ -46,116 +45,9 @@ const { ServiceUnavailableError, } = require('../errors/AppError'); const { ErrorCodes } = require('../errors/errorCodes'); - -/** - * The relying party: who is asking for a passkey, and where from. - * - * A key is bound to a domain, and the browser refuses to use it anywhere else. - * The domain is taken from the public address when one is configured and from - * the request otherwise, because a deployment reached by several names would - * otherwise bind every key to whichever one was written down. - */ -const relyingParty = (req) => { - const known = uniqueOrigins(publicConfig.origins || []); - const origins = known.length - ? known - : uniqueOrigins([`${req.protocol}://${req.get('host') || ''}`]); - - let rpId = webauthnConfig.rpId; - if (!rpId) { - try { - rpId = new URL(publicConfig.url || origins[0] || '').hostname; - } catch (_) { - rpId = null; - } - } - if (!rpId) rpId = req.hostname; - return { rpId, rpName: webauthnConfig.rpName, origins }; -}; - -/** - * Keep the question until the answer arrives, and spend it then. - * - * In the session, not in a table: it belongs to one browser and one moment. - * Spending it means taking it away — an answer is worth one sign-in, and a - * challenge still lying about is one somebody else can answer with a recording - * of the first. - */ -const rememberChallenge = (req, purpose, challenge) => - new Promise((resolve, reject) => { - if (!req.session) { - reject(new Error('A passkey needs a session to ask its question in.')); - return; - } - req.session.webauthn = { purpose, challenge, at: Date.now() }; - req.session.save((error) => (error ? reject(error) : resolve())); - }); - -const spendChallenge = (req, purpose) => { - const held = req.session?.webauthn; - if (req.session) delete req.session.webauthn; - if (!held || held.purpose !== purpose) return null; - if (Date.now() - (Number(held.at) || 0) > passkeys.CEREMONY_TIMEOUT_MS) return null; - return held.challenge; -}; - -/** Every refusal reads the same from outside, and says what happened in the log. */ -const refusePasskey = (error) => { - if (!(error instanceof WebAuthnError)) throw error; - if (error.status === 409) throw new ValidationError(error.message); - logger.warn({ reason: error.message }, 'A passkey was refused'); - throw new UnauthorizedError('That passkey was not accepted.', ErrorCodes.AUTH_PASSKEY_REJECTED); -}; - -/** - * How long the second step stays open. - * - * Long enough to find a phone, pick the app and read the digits; short enough - * that a machine walked away from is not a sign-in waiting to be finished by - * whoever sits down next. - */ -const SECOND_STEP_MS = 5 * 60 * 1000; - -/** - * The password was right, and the account wants a code as well. - * - * Deliberately not a signed-in session with a flag on it: nothing but - * `localUserId` signs anybody in, and this state does not set it. The session - * is regenerated here for the same reason it is regenerated at the end — an id - * somebody planted in the browser must not be the one that finishes the - * sign-in. - */ -const startSecondStep = (req, userId) => - new Promise((resolve, reject) => { - if (!req.session) { - reject(new Error('No session to hold the second step in.')); - return; - } - req.session.regenerate((error) => { - if (error) { - reject(error); - return; - } - req.session.pendingTotpUserId = userId; - req.session.pendingTotpSince = Date.now(); - req.session.save((saveError) => (saveError ? reject(saveError) : resolve())); - }); - }); - -/** The account halfway through signing in here, or null. */ -const secondStepUserId = (req) => { - const userId = req.session?.pendingTotpUserId; - if (!userId) return null; - const since = Number(req.session.pendingTotpSince) || 0; - if (Date.now() - since > SECOND_STEP_MS) return null; - return userId; -}; - -const forgetSecondStep = (req) => { - if (!req.session) return; - delete req.session.pendingTotpUserId; - delete req.session.pendingTotpSince; -}; +const { startAuthenticatedSession } = require('../utils/authenticatedSession'); +const { incrementFailedAttempts, clearLock, isLocked } = require('../services/users/lockout'); +const { clientAddress } = require('../utils/clientAddress'); const rateLimitHandler = (req, res, next, options) => { const retryAfterSeconds = Math.ceil(options.windowMs / 1000); @@ -218,6 +110,56 @@ const oneSetupAtATime = (task) => { return run; }; +/** + * How long the second step stays open. + * + * Long enough to find a phone, pick the app and read the digits; short enough + * that a machine walked away from is not a sign-in waiting to be finished by + * whoever sits down next. + */ +const SECOND_STEP_MS = 5 * 60 * 1000; + +/** + * The password was right, and the account wants a code as well. + * + * Deliberately not a signed-in session with a flag on it: nothing but + * `localUserId` signs anybody in, and this state does not set it. The session + * is regenerated here for the same reason it is regenerated at the end — an id + * somebody planted in the browser must not be the one that finishes the + * sign-in. + */ +const startSecondStep = (req, userId) => + new Promise((resolve, reject) => { + if (!req.session) { + reject(new Error('A second factor needs a session to wait in.')); + return; + } + req.session.regenerate((error) => { + if (error) { + reject(error); + return; + } + req.session.pendingTotpUserId = userId; + req.session.pendingTotpSince = Date.now(); + req.session.save((saveError) => (saveError ? reject(saveError) : resolve())); + }); + }); + +/** The account halfway through signing in here, or null. */ +const secondStepUserId = (req) => { + const userId = req.session?.pendingTotpUserId; + if (!userId) return null; + const since = Number(req.session.pendingTotpSince) || 0; + if (Date.now() - since > SECOND_STEP_MS) return null; + return userId; +}; + +const forgetSecondStep = (req) => { + if (!req.session) return; + delete req.session.pendingTotpUserId; + delete req.session.pendingTotpSince; +}; + const respondWithUser = async (req, res) => { const user = await getRequestUser(req); res.json({ user }); @@ -251,10 +193,10 @@ router.get('/status', async (req, res) => { res.json({ requiresSetup, + strategies, // A reload in the middle of signing in lands back on the code, rather than // on a password screen that would start the whole thing again. totpPending: Boolean(secondStepUserId(req)), - strategies, authEnabled: auth.enabled, authMode, authenticated: auth.enabled ? Boolean(isEoc || hasLocal) : true, @@ -290,9 +232,8 @@ router.post( await startAuthenticatedSession(req, user.id); // Clear guest session cookie when user sets up account - // Both paths: the cookie has been set on `/api` and on `/`, and a guest - // session left behind on the other one outlives the sign-in that should - // have ended it. + // Clear both scopes: the cookie used to be set on /api, and browsers + // still holding that one would otherwise keep it. res.clearCookie('guestSession', { path: '/' }); res.clearCookie('guestSession', { path: '/api' }); @@ -300,22 +241,33 @@ router.post( }) ); -// Local login with email + password +// Local login with an email address or a username, and a password router.post( '/login', loginLimiter, asyncHandler(async (req, res) => { refuseWithoutPasswordSignIn(); - const { email, password, username } = req.body || {}; - // Support both email and username (backward compatibility) - const emailOrUsername = email || username; + const { identifier, email, password, username } = req.body || {}; + // `email` and `username` are the older field names; both carried whatever + // was typed into the one box on the sign-in screen. + const typed = identifier || email || username; - let user = null; + let user; try { - user = await attemptLocalLogin({ email: emailOrUsername, password }); + user = await attemptLocalLogin({ identifier: typed, password }); } catch (e) { if (e?.status === 423) { - throw new RateLimitError(e.message, e.until); + // Seconds in `retryAfter`, which is what the interface reads, with the + // deadline itself beside it, under a code of its own so the message + // translates. The ISO date used to sit in `retryAfter` under a generic + // code, and the sign-in screen could say neither how long nor in what + // language. + const lockedUntil = e.until || null; + const msLeft = lockedUntil ? Date.parse(lockedUntil) - Date.now() : NaN; + const retryAfter = Number.isFinite(msLeft) ? Math.max(1, Math.ceil(msLeft / 1000)) : null; + const locked = new RateLimitError(e.message, retryAfter, ErrorCodes.AUTH_ACCOUNT_LOCKED); + if (lockedUntil) locked.details = { ...locked.details, lockedUntil }; + throw locked; } throw e; } @@ -324,7 +276,7 @@ router.post( action: 'sign-in', outcome: 'refused', // The name that was typed, not one this server confirmed exists. - actor: String(emailOrUsername || '').slice(0, 200) || 'unknown', + actor: String(typed || '').slice(0, 200) || 'unknown', detail: { method: 'password' }, req, }); @@ -332,9 +284,9 @@ router.post( } // The password was right and the account asks for a code as well. Nothing - // about who they are is answered here: that an account has a second factor - // is not something to tell whoever guessed a password correctly, so the - // answer carries the question and nothing else. + // about who they are is answered here: an account that has a second factor + // is not something to tell anybody who guessed a password correctly, so + // the answer carries the question and nothing else. if (await twoFactorRequired(user.id)) { await startSecondStep(req, user.id); res.json({ totpRequired: true }); @@ -345,9 +297,8 @@ router.post( await activityLog.record({ action: 'sign-in', user, detail: { method: 'password' }, req }); // Clear guest session cookie when user logs in - // Both paths: the cookie has been set on `/api` and on `/`, and a guest - // session left behind on the other one outlives the sign-in that should - // have ended it. + // Clear both scopes: the cookie used to be set on /api, and browsers + // still holding that one would otherwise keep it. res.clearCookie('guestSession', { path: '/' }); res.clearCookie('guestSession', { path: '/api' }); @@ -355,56 +306,305 @@ router.post( }) ); -/** The passkeys on this account, so they can be named and taken away. */ +/** + * The second step: the code from the phone, or one off the paper. + * + * Wrong codes count against the same lockout a wrong password does, so the + * second factor is not a place to guess a million times at six digits while + * the first one is bounded. + */ +router.post( + '/login/totp', + loginLimiter, + asyncHandler(async (req, res) => { + refuseWithoutPasswordSignIn(); + const userId = secondStepUserId(req); + if (!userId) { + forgetSecondStep(req); + throw new UnauthorizedError( + 'That sign-in is no longer waiting for a code. Sign in again.', + ErrorCodes.AUTH_INVALID_CREDENTIALS + ); + } + + if (await isLocked(userId)) { + throw new RateLimitError( + 'Account is temporarily locked due to failed login attempts.', + null, + ErrorCodes.AUTH_ACCOUNT_LOCKED + ); + } + + const { code } = req.body || {}; + const outcome = await verifySecondFactor({ userId, code }); + if (!outcome.ok) { + await incrementFailedAttempts(userId); + await activityLog.record({ + action: 'sign-in', + outcome: 'refused', + userId, + actor: (await getById(userId))?.username || userId, + detail: { method: 'code' }, + req, + }); + throw new UnauthorizedError('That code is not right.', ErrorCodes.AUTH_INVALID_TOTP_CODE); + } + + await clearLock(userId); + forgetSecondStep(req); + await startAuthenticatedSession(req, userId); + + res.clearCookie('guestSession', { path: '/' }); + res.clearCookie('guestSession', { path: '/api' }); + + const user = await getRequestUser(req); + await activityLog.record({ + action: 'sign-in', + user, + detail: { method: outcome.usedRecoveryCode ? 'recovery code' : 'code' }, + req, + }); + res.json({ + user, + usedRecoveryCode: Boolean(outcome.usedRecoveryCode), + recoveryCodesLeft: outcome.recoveryCodesLeft ?? null, + }); + }) +); + +/** Whether this account asks for a code, and how many recovery codes are left. */ router.get( - '/passkeys', + '/totp', asyncHandler(async (req, res) => { const me = await getRequestUser(req); if (!me) throw new UnauthorizedError('Authentication required.'); - res.json({ passkeys: await passkeys.listPasskeys(me.id) }); + res.json(await twoFactorStatus(me.id)); }) ); /** - * Start making one. + * Draw a secret and show it, which turns nothing on. * - * Signed in here with this account, like the second factor: a session the - * identity provider opened is not one that adds a local way in. + * What comes back is shown once and never again: the phone keeps it, and the + * copy here is unreadable the moment it is written. */ router.post( - '/passkeys/register/start', + '/totp/start', passwordLimiter, asyncHandler(async (req, res) => { refuseWithoutPasswordSignIn(); const me = await getRequestUser(req); if (!me) throw new UnauthorizedError('Authentication required.'); if (!req.session || req.session.localUserId !== me.id) { - throw new ForbiddenError('Sign in with your password to add a passkey.'); + throw new ForbiddenError('Sign in with your password to set up a second factor.'); } - const { rpId, rpName, origins } = relyingParty(req); - const options = await passkeys.beginRegistration({ + const enrolment = await beginTwoFactorEnrolment({ userId: me.id, account: me.email || me.username || me.id, - displayName: me.displayName || me.username || me.email || me.id, - rpId, - rpName, }); - await rememberChallenge(req, 'register', options.challenge); - res.json({ options, origins }); + res.json(enrolment); }) ); -/** Keep it, if it answers the question this browser was just asked. */ +/** Turn it on, once a code proves the phone holds the same secret. */ router.post( - '/passkeys/register/finish', + '/totp/confirm', passwordLimiter, asyncHandler(async (req, res) => { - refuseWithoutPasswordSignIn(); const me = await getRequestUser(req); if (!me) throw new UnauthorizedError('Authentication required.'); - if (!req.session || req.session.localUserId !== me.id) { - throw new ForbiddenError('Sign in with your password to add a passkey.'); + + const confirmed = await confirmTwoFactorEnrolment({ userId: me.id, code: req.body?.code }); + if (!confirmed) { + throw new UnauthorizedError('That code is not right.', ErrorCodes.AUTH_INVALID_TOTP_CODE); + } + logger.info({ userId: me.id }, 'Two-factor authentication turned on'); + await activityLog.record({ action: 'account.two-factor', user: me, detail: { on: true }, req }); + res.json(confirmed); + }) +); + +/** + * New recovery codes, and the password to prove it is still the same person. + * + * A browser left unlocked is the case this is about: drawing new codes throws + * the old ones away, and somebody who sat down at a signed-in screen should + * not be able to leave with the only working set. + */ +router.post( + '/totp/recovery-codes', + passwordLimiter, + asyncHandler(async (req, res) => { + const me = await getRequestUser(req); + if (!me) throw new UnauthorizedError('Authentication required.'); + if (!(await verifyLocalPassword({ userId: me.id, password: req.body?.password }))) { + throw new UnauthorizedError( + 'That password is not right.', + ErrorCodes.AUTH_PASSWORD_INCORRECT + ); + } + + res.json({ recoveryCodes: await replaceRecoveryCodes(me.id) }); + }) +); + +/** Off, with the password for the same reason. */ +router.delete( + '/totp', + passwordLimiter, + asyncHandler(async (req, res) => { + const me = await getRequestUser(req); + if (!me) throw new UnauthorizedError('Authentication required.'); + if (!(await verifyLocalPassword({ userId: me.id, password: req.body?.password }))) { + throw new UnauthorizedError( + 'That password is not right.', + ErrorCodes.AUTH_PASSWORD_INCORRECT + ); + } + + await disableTwoFactor(me.id); + logger.info({ userId: me.id }, 'Two-factor authentication turned off'); + await activityLog.record({ + action: 'account.two-factor', + user: me, + detail: { on: false }, + req, + }); + res.status(204).end(); + }) +); + +/** + * The site a passkey is bound to, and the pages allowed to use one. + * + * A passkey is made for a name and signs only for that name — which is the + * phishing resistance, and also the reason an installation reached through two + * hostnames has to pick one. PUBLIC_URL answers it where it is set; where it + * is not, the name this request arrived on is the answer, which is right for + * the single-hostname installation that never configured anything. An operator + * who needs to settle it sets WEBAUTHN_RP_ID. + * + * Browsers refuse a passkey on an address that is not a name, and on a page + * that is not secure: a LAN IP or plain http offers nothing to bind to. That + * refusal happens in the browser, before this is reached. + */ +const relyingParty = (req) => { + const known = uniqueOrigins(publicConfig.origins || []); + const origins = known.length + ? known + : uniqueOrigins([`${req.protocol}://${req.get('host') || ''}`]); + + let rpId = webauthnConfig.rpId; + if (!rpId) { + try { + rpId = new URL(publicConfig.url || origins[0] || '').hostname; + } catch (_) { + rpId = null; + } + } + if (!rpId) rpId = req.hostname; + return { rpId, rpName: webauthnConfig.rpName, origins }; +}; + +/** + * Keep the question until the answer arrives, and spend it then. + * + * In the session, not in a table: it belongs to one browser and one moment. + * Spending it means taking it away — an answer is worth one sign-in, and a + * challenge still lying about is one somebody else can answer with a recording + * of the first. + */ +const rememberChallenge = (req, purpose, challenge) => + new Promise((resolve, reject) => { + if (!req.session) { + reject(new Error('A passkey needs a session to ask its question in.')); + return; + } + req.session.webauthn = { purpose, challenge, at: Date.now() }; + req.session.save((error) => (error ? reject(error) : resolve())); + }); + +const spendChallenge = (req, purpose) => { + const held = req.session?.webauthn; + if (req.session) delete req.session.webauthn; + if (!held || held.purpose !== purpose) return null; + if (Date.now() - (Number(held.at) || 0) > passkeys.CEREMONY_TIMEOUT_MS) return null; + return held.challenge; +}; + +/** Every refusal reads the same from outside, and says what happened in the log. */ +const refusePasskey = (error, req) => { + if (!(error instanceof WebAuthnError)) throw error; + if (error.status === 409) { + throw new ValidationError(error.message); + } + logger.warn({ reason: error.message, ip: clientAddress(req) }, 'A passkey was refused'); + // Not awaited: this is the throwing path, and a log line is not worth + // holding a refusal for. `record` never rejects. + activityLog.record({ + action: 'sign-in', + outcome: 'refused', + actor: 'unknown', + detail: { method: 'passkey' }, + req, + }); + throw new UnauthorizedError( + 'That passkey did not open anything here.', + ErrorCodes.AUTH_PASSKEY_REJECTED + ); +}; + +/** The passkeys on this account. */ +router.get( + '/passkeys', + asyncHandler(async (req, res) => { + const me = await getRequestUser(req); + if (!me) throw new UnauthorizedError('Authentication required.'); + res.json({ passkeys: await passkeys.listPasskeys(me.id) }); + }) +); + +/** + * Start making one. + * + * Signed in here with this account, like the second factor: a session the + * identity provider opened is not one that adds a local way in. + */ +router.post( + '/passkeys/register/start', + passwordLimiter, + asyncHandler(async (req, res) => { + refuseWithoutPasswordSignIn(); + const me = await getRequestUser(req); + if (!me) throw new UnauthorizedError('Authentication required.'); + if (!req.session || req.session.localUserId !== me.id) { + throw new ForbiddenError('Sign in with your password to add a passkey.'); + } + + const { rpId, rpName, origins } = relyingParty(req); + const options = await passkeys.beginRegistration({ + userId: me.id, + account: me.email || me.username || me.id, + displayName: me.displayName || me.username || me.email || me.id, + rpId, + rpName, + }); + await rememberChallenge(req, 'register', options.challenge); + res.json({ options, origins }); + }) +); + +/** Keep it, if it answers the question this browser was just asked. */ +router.post( + '/passkeys/register/finish', + passwordLimiter, + asyncHandler(async (req, res) => { + refuseWithoutPasswordSignIn(); + const me = await getRequestUser(req); + if (!me) throw new UnauthorizedError('Authentication required.'); + if (!req.session || req.session.localUserId !== me.id) { + throw new ForbiddenError('Sign in with your password to add a passkey.'); } const challenge = spendChallenge(req, 'register'); @@ -429,7 +629,7 @@ router.post( }); res.status(201).json({ passkey }); } catch (error) { - refusePasskey(error); + refusePasskey(error, req); } }) ); @@ -483,6 +683,12 @@ router.delete( 'This is the only way into this account. Add a password, or another passkey, before removing it.' ); } + await activityLog.record({ + action: 'account.passkey', + user: me, + detail: { added: false }, + req, + }); res.status(204).end(); }) ); @@ -511,8 +717,8 @@ router.post( * * A passkey that was unlocked — a fingerprint, a face, a PIN — is already two * things: the device, and whoever can open it. That is why it satisfies an - * account that asks for a second factor, and why one that was not unlocked does - * not: that proves only that the device was there. + * account that asks for a second factor, and why one that was not unlocked + * does not: it proves only that the device was there. */ router.post( '/login/passkey/finish', @@ -535,7 +741,7 @@ router.post( expected: { challenge, origins, rpId }, }); } catch (error) { - refusePasskey(error); + refusePasskey(error, req); } if (await isLocked(outcome.userId)) { @@ -554,9 +760,6 @@ router.post( } await startAuthenticatedSession(req, outcome.userId); - // Both paths: the cookie has been set on `/api` and on `/`, and a guest - // session left behind on the other one outlives the sign-in that should - // have ended it. res.clearCookie('guestSession', { path: '/' }); res.clearCookie('guestSession', { path: '/api' }); @@ -564,185 +767,14 @@ router.post( { userId: outcome.userId, passkeyId: outcome.passkeyId }, 'Signed in with a passkey' ); - const signedIn = await getRequestUser(req); + const user = await getRequestUser(req); await activityLog.record({ action: 'sign-in', - user: signedIn, + user, detail: { method: 'passkey', passkey: outcome.name }, req, }); - res.json({ user: signedIn }); - }) -); - -/** - * The second step: the code from the phone, or one off the paper. - * - * Wrong codes count against the same lockout a wrong password does, so the - * second factor is not a place to guess a million times at six digits while - * the first one is bounded. - */ -router.post( - '/login/totp', - loginLimiter, - asyncHandler(async (req, res) => { - refuseWithoutPasswordSignIn(); - const userId = secondStepUserId(req); - if (!userId) { - forgetSecondStep(req); - throw new UnauthorizedError( - 'That sign-in is no longer waiting for a code. Sign in again.', - ErrorCodes.AUTH_INVALID_CREDENTIALS - ); - } - - if (await isLocked(userId)) { - throw new RateLimitError( - 'Account is temporarily locked due to failed login attempts.', - null, - ErrorCodes.AUTH_ACCOUNT_LOCKED - ); - } - - const { code } = req.body || {}; - const outcome = await verifySecondFactor({ userId, code }); - if (!outcome.ok) { - await incrementFailedAttempts(userId); - await activityLog.record({ - action: 'sign-in', - outcome: 'refused', - userId, - detail: { method: 'code' }, - req, - }); - throw new UnauthorizedError('That code is not right.', ErrorCodes.AUTH_INVALID_TOTP_CODE); - } - - await clearLock(userId); - forgetSecondStep(req); - await startAuthenticatedSession(req, userId); - // Both paths: the cookie has been set on `/api` and on `/`, and a guest - // session left behind on the other one outlives the sign-in that should - // have ended it. - res.clearCookie('guestSession', { path: '/' }); - res.clearCookie('guestSession', { path: '/api' }); - - const signedIn = await getRequestUser(req); - await activityLog.record({ - action: 'sign-in', - user: signedIn, - detail: { method: outcome.usedRecoveryCode ? 'recovery code' : 'code' }, - req, - }); - res.json({ - user: signedIn, - usedRecoveryCode: Boolean(outcome.usedRecoveryCode), - recoveryCodesLeft: outcome.recoveryCodesLeft ?? null, - }); - }) -); - -/** Whether this account asks for a code, and how many recovery codes are left. */ -router.get( - '/totp', - asyncHandler(async (req, res) => { - const me = await getRequestUser(req); - if (!me) throw new UnauthorizedError('Authentication required.'); - res.json(await twoFactorStatus(me.id)); - }) -); - -/** - * Draw a secret and show it, which turns nothing on. - * - * What comes back is shown once and never again: the phone keeps it, and the - * copy here is unreadable the moment it is written. - */ -router.post( - '/totp/start', - passwordLimiter, - asyncHandler(async (req, res) => { - refuseWithoutPasswordSignIn(); - const me = await getRequestUser(req); - if (!me) throw new UnauthorizedError('Authentication required.'); - if (!req.session || req.session.localUserId !== me.id) { - throw new ForbiddenError('Sign in with your password to set up a second factor.'); - } - - res.json( - await beginTwoFactorEnrolment({ - userId: me.id, - account: me.email || me.username || me.id, - }) - ); - }) -); - -/** Turn it on, once a code proves the phone holds the same secret. */ -router.post( - '/totp/confirm', - passwordLimiter, - asyncHandler(async (req, res) => { - const me = await getRequestUser(req); - if (!me) throw new UnauthorizedError('Authentication required.'); - - const confirmed = await confirmTwoFactorEnrolment({ userId: me.id, code: req.body?.code }); - if (!confirmed) { - throw new UnauthorizedError('That code is not right.', ErrorCodes.AUTH_INVALID_TOTP_CODE); - } - logger.info({ userId: me.id }, 'Two-factor authentication turned on'); - await activityLog.record({ action: 'account.two-factor', user: me, detail: { on: true }, req }); - res.json(confirmed); - }) -); - -/** - * New recovery codes, and the password to prove it is still the same person. - * - * A browser left unlocked is the case this is about: drawing new codes throws - * the old ones away, and somebody who sat down at a signed-in screen should not - * be able to leave with the only working set. - */ -router.post( - '/totp/recovery-codes', - passwordLimiter, - asyncHandler(async (req, res) => { - const me = await getRequestUser(req); - if (!me) throw new UnauthorizedError('Authentication required.'); - if (!(await verifyLocalPassword({ userId: me.id, password: req.body?.password }))) { - throw new UnauthorizedError( - 'That password is not right.', - ErrorCodes.AUTH_PASSWORD_INCORRECT - ); - } - - res.json({ recoveryCodes: await replaceRecoveryCodes(me.id) }); - }) -); - -/** Off, with the password for the same reason. */ -router.delete( - '/totp', - passwordLimiter, - asyncHandler(async (req, res) => { - const me = await getRequestUser(req); - if (!me) throw new UnauthorizedError('Authentication required.'); - if (!(await verifyLocalPassword({ userId: me.id, password: req.body?.password }))) { - throw new UnauthorizedError( - 'That password is not right.', - ErrorCodes.AUTH_PASSWORD_INCORRECT - ); - } - - await disableTwoFactor(me.id); - logger.info({ userId: me.id }, 'Two-factor authentication turned off'); - await activityLog.record({ - action: 'account.two-factor', - user: me, - detail: { on: false }, - req, - }); - res.status(204).end(); + res.json({ user }); }) ); @@ -767,6 +799,7 @@ router.post( newPassword, keepSessionId: signedInHere ? req.sessionID : null, }); + if (signedInHere) await startAuthenticatedSession(req, me.id); await activityLog.record({ action: 'account.password', user: me, req }); res.status(204).end(); }) @@ -1174,9 +1207,8 @@ router.post( throw new UnauthorizedError('User no longer exists.', ErrorCodes.AUTH_INVALID_CREDENTIALS); } - // Both paths: the cookie has been set on `/api` and on `/`, and a guest - // session left behind on the other one outlives the sign-in that should - // have ended it. + // Clear both scopes: the cookie used to be set on /api, and browsers still + // holding that one would otherwise keep it. res.clearCookie('guestSession', { path: '/' }); res.clearCookie('guestSession', { path: '/api' }); res.json({ user }); diff --git a/backend/src/routes/browse.js b/backend/src/routes/browse.js index 1038237c8..c930a426c 100644 --- a/backend/src/routes/browse.js +++ b/backend/src/routes/browse.js @@ -2,6 +2,7 @@ const express = require('express'); const { normalizeRelativePath } = require('../utils/pathUtils'); const { pathExists } = require('../utils/fsUtils'); +const env = require('../config/env'); const { withStorage } = require('../services/storageWritability'); const { getSettings, getUserSettings } = require('../services/settingsService'); const logger = require('../utils/logger'); @@ -53,7 +54,10 @@ router.get( asyncHandler(async (req, res) => { const settings = await getSettings(); const userSettings = req.user?.id ? await getUserSettings(req.user.id) : {}; - const thumbsEnabled = settings?.thumbnails?.enabled !== false; + // Off for the whole installation, or off in the settings: either one means + // the listing must not promise a thumbnail the server will never make. + const thumbsEnabled = + env.THUMBNAILS_ENABLED !== false && settings?.thumbnails?.enabled !== false; const includeHiddenFiles = userSettings?.showHiddenFiles === true; const rawPath = (req.params.splat || []).join('/'); const inputRelativePath = normalizeRelativePath(rawPath); @@ -83,7 +87,6 @@ router.get( parentLogicalPath: relativePath, context, thumbsEnabled, - excludeDownloadArtifacts: true, includeHiddenFiles, access: settings?.access || null, itemExtras: await versionMarks(directoryPath, userSettings), @@ -101,6 +104,9 @@ router.get( canUpload: access.canUpload, canDelete: access.canDelete, canCreateFolder: access.canCreateFolder, + // A share may permit folders and refuse files, or the other way + // round, so the two are answered apart. + canCreateFile: access.canCreateFile, canShare: access.canShare, canDownload: access.canDownload, // Whether the files here show their history, which a share hands out @@ -126,6 +132,9 @@ router.get( sourceFolderName: pathParts[pathParts.length - 1] || '', }; } + // Listings carry transient information such as active OnlyOffice sessions. + // Keep browser and proxy caches from serving an out-of-date directory view. + res.setHeader('Cache-Control', 'private, no-store'); res.json(response); }) diff --git a/backend/src/routes/collabora.js b/backend/src/routes/collabora.js index 28ec3e7e1..2b3103eaf 100644 --- a/backend/src/routes/collabora.js +++ b/backend/src/routes/collabora.js @@ -5,7 +5,8 @@ const fsp = require('fs/promises'); const crypto = require('crypto'); const jwt = require('jsonwebtoken'); -const { collabora, public: publicConfig, mimeTypes } = require('../config/index'); +const { collabora, public: publicConfig } = require('../config/index'); +const { toExtension, resolveMimeType } = require('../utils/fileTypes'); const { normalizeRelativePath } = require('../utils/pathUtils'); const { ensureDir } = require('../utils/fsUtils'); const { resolvePathWithAccess } = require('../services/accessManager'); @@ -13,8 +14,9 @@ const asyncHandler = require('../utils/asyncHandler'); const logger = require('../utils/logger'); const { getDiscoveryActionsByExt } = require('../services/collaboraDiscoveryService'); const lockService = require('../services/wopiLockService'); -const versions = require('../services/versions/operations'); const { ValidationError, UnauthorizedError, ForbiddenError } = require('../errors/AppError'); +const folderSizeHooks = require('../services/folderSizeHooks'); +const versions = require('../services/versions/operations'); /** A Collabora save header, under its current name or the one older servers still send. */ const wopiSaveHeader = (req, name) => @@ -24,14 +26,6 @@ const wopiSaveHeader = (req, name) => const router = express.Router(); -const toExt = (filename = '') => { - const base = path.basename(String(filename)); - const idx = base.lastIndexOf('.'); - return idx > 0 ? base.slice(idx + 1).toLowerCase() : ''; -}; - -const resolveMime = (ext) => mimeTypes[ext] || 'application/octet-stream'; - const toBase64Url = (buf) => Buffer.from(buf).toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, ''); @@ -54,6 +48,9 @@ const getAccessTokenFromReq = (req) => { return null; }; +// Signed with the same secret as other tokens, so it says what it is. +const WOPI_TOKEN_TYPE = 'nextexplorer-wopi'; + const verifyWopiToken = (req, fileId) => { if (!collabora?.secret) { throw new UnauthorizedError('COLLABORA_SECRET is not configured.'); @@ -64,7 +61,7 @@ const verifyWopiToken = (req, fileId) => { throw new UnauthorizedError('Missing access_token.'); } - let payload = null; + let payload; try { payload = jwt.verify(token, collabora.secret, { algorithms: ['HS256'] }); } catch (_e) { @@ -83,6 +80,12 @@ const verifyWopiToken = (req, fileId) => { throw new UnauthorizedError('access_token missing absolutePath.'); } + // Older tokens predate the type claim; anything that declares another type + // is not a WOPI token and must not stand in for one. + if (payload.typ && payload.typ !== WOPI_TOKEN_TYPE) { + throw new UnauthorizedError('access_token type mismatch.'); + } + return payload; }; @@ -128,27 +131,51 @@ router.post( throw new ForbiddenError(accessInfo?.denialReason || 'Access denied.'); } - const abs = resolved.absolutePath; - const stat = await fsp.stat(abs); + const stat = await fsp.stat(resolved.absolutePath); if (stat.isDirectory()) { throw new ValidationError('Cannot open a directory in Collabora.'); } + // An earlier version, opened to be read: its own content under the file's + // name, never writable, and a file id of its own so that it takes no part in + // the locks of the document open beside it. + const versionHistory = require('../services/versions'); + const requestedVersion = typeof req.body?.versionId === 'string' ? req.body.versionId : ''; + const version = requestedVersion + ? await versionHistory.locateVersion(context, relativePath, requestedVersion, { + download: false, + }) + : null; + const abs = version ? version.absolutePath : resolved.absolutePath; + const isReadonlyShare = resolved.shareInfo && resolved.shareInfo.accessMode === 'readonly'; - const userCanWrite = Boolean(accessInfo.canWrite) && !isReadonlyShare && mode !== 'view'; + const userCanWrite = + !version && Boolean(accessInfo.canWrite) && !isReadonlyShare && mode !== 'view'; - const filename = path.basename(abs); - const ext = toExt(filename); + const filename = version ? version.name : path.basename(abs); + const ext = toExtension(filename); if (!ext) { throw new ValidationError('Unknown file extension.'); } - const fileId = buildFileId({ space: resolved.space, relativePath: resolved.relativePath }); + const fileId = buildFileId({ + space: resolved.space, + relativePath: version + ? `${resolved.relativePath}@version:${version.version.id}` + : resolved.relativePath, + }); + + // The editor's Revision history entry opens NextExplorer's own history, so + // it is only shown where there is one to show. + const { getVersionSettings } = require('../services/versions/settings'); + const offerHistory = + !version && (await getVersionSettings()).enabled && versionHistory.rightsFrom(accessInfo).see; const tokenTtlSeconds = 6 * 60 * 60; // 6 hours const tokenExpiresAtMs = Date.now() + tokenTtlSeconds * 1000; const accessToken = jwt.sign( { + typ: WOPI_TOKEN_TYPE, fileId, absolutePath: abs, logicalPath: resolved.relativePath, @@ -159,6 +186,10 @@ router.post( req.user?.displayName || req.user?.username || (req.guestSession ? 'Guest User' : null), guestSessionId: req.guestSession?.id || null, shareToken: resolved.shareInfo?.shareToken || null, + // A version's content is stored under its id: the name it is shown + // under is the file's. + baseName: filename, + versionId: version ? version.version.id : null, }, collabora.secret, { @@ -192,6 +223,16 @@ router.post( if (collabora.lang) { iframeUrl.searchParams.set('lang', collabora.lang); } + // Shows File > Revision history, which posts UI_FileVersions to the page. + if (offerHistory) { + iframeUrl.searchParams.set('revisionhistory', '1'); + } + // Draws the editor's own close button, which posts UI_Close to the page + // rather than closing anything itself. Without it the only way out of a + // full-screen editor was a button the page floated over the toolbar, which + // sat there looking like something Collabora had not quite finished drawing + // (nxzai/NextExplorer#303). ONLYOFFICE is asked the same thing. + iframeUrl.searchParams.set('closebutton', '1'); res.json({ urlSrc: iframeUrl.toString(), @@ -215,7 +256,9 @@ router.get( const stat = await fsp.stat(abs); if (stat.isDirectory()) throw new ValidationError('Cannot open a directory.'); - const baseName = path.basename(abs); + // A version's content is stored under its id; the token carries the name + // it is shown under. + const baseName = tokenPayload.baseName || path.basename(abs); const version = versionFromStat(stat); res.json({ @@ -231,6 +274,8 @@ router.get( SupportsGetLock: true, // Required for PostMessage API (enables features like @ mentions) PostMessageOrigin: publicConfig?.url || '*', + // An earlier version is read, not saved elsewhere under a new name either. + ...(tokenPayload.versionId ? { UserCanNotWriteRelative: true } : {}), }); }) ); @@ -248,8 +293,8 @@ router.get( const stat = await fsp.stat(abs); if (stat.isDirectory()) throw new ValidationError('Cannot fetch a directory.'); - const ext = toExt(abs); - const mime = resolveMime(ext); + const ext = toExtension(abs); + const mime = resolveMimeType(ext); res.writeHead(200, { 'Content-Type': mime, 'Content-Length': stat.size, @@ -267,6 +312,23 @@ router.get( }) ); +/** + * A token lives for hours; the share it was issued for may not. + * + * The token stands in for a permission check, so before writing we confirm the + * share still exists and has not expired. Deleting or expiring a share now + * ends the editing session instead of leaving it writable until the token + * runs out. + */ +const assertShareStillValid = async (tokenPayload) => { + if (!tokenPayload?.shareToken) return; + const { getShareByToken, isShareExpired } = require('../services/sharesService'); + const share = await getShareByToken(tokenPayload.shareToken); + if (!share || isShareExpired(share)) { + throw new UnauthorizedError('The share for this editing session is no longer available.'); + } +}; + // WOPI: PutFile (save) router.post( '/collabora/wopi/files/:fileId/contents', @@ -275,12 +337,23 @@ router.post( if (!fileId) throw new ValidationError('fileId is required.'); const tokenPayload = verifyWopiToken(req, fileId); + await assertShareStillValid(tokenPayload); + if (!tokenPayload.canWrite) { throw new ForbiddenError('This file is read-only.'); } const abs = tokenPayload.absolutePath; await ensureDir(path.dirname(abs)); + let previousSize = 0; + let existed = false; + try { + const previous = await fsp.stat(abs); + existed = previous.isFile(); + previousSize = existed ? previous.size : 0; + } catch { + // A newly-created document is valid. + } const requestLock = (req.headers['x-wopi-lock'] || '').toString(); const currentLock = lockService.getLock(fileId); @@ -304,19 +377,22 @@ router.post( author: { id: tokenPayload.userId || null, label: tokenPayload.userName || null }, source: 'collabora', session: { - // One lock per open document, held by everyone editing it together: - // the session its saves belong to. Without one every save would stand - // as a state of its own. + // One lock per open document, shared by everyone editing it: the + // session its saves belong to. Without one, every save counts. key: requestLock ? `wopi:${requestLock}` : null, startedAt: Number.isFinite(tokenPayload.iat) ? tokenPayload.iat * 1000 : null, }, - // Collabora saves on its own every few minutes; the ones somebody asked - // for, and the one made on closing the document, are states worth - // keeping. + // Collabora saves on its own every few minutes; a save someone asked + // for, or the one made on closing, is a state worth keeping. explicit: wopiSaveHeader(req, 'isautosave') !== 'true', } ); const stat = await fsp.stat(abs); + if (existed) { + await folderSizeHooks.onFileReplaced(abs, previousSize, stat.size); + } else { + await folderSizeHooks.onFileWritten(abs, stat.size); + } res.setHeader('Cache-Control', 'no-store'); res.setHeader('X-WOPI-ItemVersion', versionFromStat(stat)); @@ -332,6 +408,8 @@ router.post( if (!fileId) throw new ValidationError('fileId is required.'); const tokenPayload = verifyWopiToken(req, fileId); + await assertShareStillValid(tokenPayload); + if (!tokenPayload.canWrite) { throw new ForbiddenError('This file is read-only.'); } diff --git a/backend/src/routes/editor.js b/backend/src/routes/editor.js index b49eb1b89..8aafaabfd 100644 --- a/backend/src/routes/editor.js +++ b/backend/src/routes/editor.js @@ -13,6 +13,7 @@ const { ValidationError, ForbiddenError, NotFoundError } = require('../errors/Ap const { readFileEncoding, encodeText, + textFileEtag, MAX_EDITOR_FILE_SIZE, } = require('../services/textEditorService'); @@ -157,9 +158,13 @@ router.put( // a stop halfway through left it truncated and the state it replaced was // gone. Somebody pressed Save, so it is a state worth keeping — there is no // session here to group it with, as there is in the office editors. + let written = null; await versions.saveFile( absolutePath, - (temporaryPath) => fs.writeFile(temporaryPath, payload, { flag: 'wx' }), + async (temporaryPath) => { + await fs.writeFile(temporaryPath, payload, { flag: 'wx' }); + written = await fs.stat(temporaryPath, { bigint: true }); + }, { purpose: 'editor', author: versions.authorOf({ user: req.user, guestSession: req.guestSession }), @@ -169,14 +174,28 @@ router.put( ); // The index takes the difference the save made, from the size it can already // see, instead of waiting for the periodic sweep to walk the folder again. - const updated = await fs.stat(absolutePath).catch(() => null); + const updated = await fs.stat(absolutePath, { bigint: true }).catch(() => null); if (updated) { if (existed) { - await folderSizeHooks.onFileReplaced(absolutePath, previousSize, updated.size); + await folderSizeHooks.onFileReplaced(absolutePath, previousSize, Number(updated.size)); } else { - await folderSizeHooks.onFileWritten(absolutePath, updated.size); + await folderSizeHooks.onFileWritten(absolutePath, Number(updated.size)); } } + // The identity the next read of the file will carry — given only when the + // file now at the path is the one this save wrote. A save set aside, or one + // whose content was already there, leaves another file in place; a write in + // place right after the rename changes the modification time. Either way + // this answer would name content it did not send. + if ( + written && + updated && + updated.ino === written.ino && + updated.size === written.size && + updated.mtimeNs === written.mtimeNs + ) { + res.setHeader('ETag', textFileEtag(updated)); + } res.send({ success: true }); }) diff --git a/backend/src/routes/files/delete.js b/backend/src/routes/files/delete.js index 3b356b188..f1c6ae85f 100644 --- a/backend/src/routes/files/delete.js +++ b/backend/src/routes/files/delete.js @@ -35,15 +35,15 @@ router.post( * Read from what each deletion did, and not from the `permanent` flag the * request carried. That flag is what the caller asked for, which is not what * happened: with the trash switched off, the interface asks for nothing in - * particular — there is no trash to choose — so every file it deleted for good - * would be written down as having been moved to a trash that does not exist. - * The service already answers per item, `trashed` or `deleted`, and that is - * the only account of it that cannot be wrong. + * particular — there is no trash to choose — and every file it deleted for + * good was written down as having been moved to the trash that does not + * exist. The service already answers per item, `trashed` or `deleted`, and + * that is the only account of it that cannot be wrong. * - * One line per outcome, so a selection that was partly kept and partly removed - * says both rather than the first one twice. An item that did not go anywhere - * — already missing, or refused by the trash and left where it is — writes - * nothing at all. + * One line per outcome, so a selection that was partly kept and partly + * removed says both rather than the first one twice. An item that did not go + * anywhere — already missing, or refused by the trash and left where it is — + * writes nothing at all, where it used to be counted among the deleted. */ const ACTION_FOR = { trashed: 'file.delete', deleted: 'file.purge' }; @@ -67,9 +67,9 @@ const recordDeletion = async ({ results, req }) => { } // The part of a deletion that nothing on screen showed. A file is one line - // in a folder and its earlier versions are none, so a deletion that took ten - // of them said as much as one that took none — and versions are the half - // that cannot be restored from anywhere. + // in a folder and its earlier versions are none, so a deletion that took + // ten of them said as much as one that took none — and versions are the + // half that cannot be restored from anywhere. const withHistory = (Array.isArray(results) ? results : []).filter( (result) => Number(result?.versionsPurged) > 0 ); diff --git a/backend/src/routes/files/download.js b/backend/src/routes/files/download.js index 0563fde5b..8545a5309 100644 --- a/backend/src/routes/files/download.js +++ b/backend/src/routes/files/download.js @@ -1,16 +1,19 @@ const path = require('path'); const fs = require('fs/promises'); -const archiver = require('archiver'); +const { ZipArchive } = require('archiver'); const { normalizeRelativePath } = require('../../utils/pathUtils'); const { resolvePathWithAccess } = require('../../services/accessManager'); const activityLog = require('../../services/activityLog'); +const { trackShareDownload } = require('../../services/sharesService'); const asyncHandler = require('../../utils/asyncHandler'); +const { mapWithConcurrency } = require('../../utils/mapWithConcurrency'); const { collectArchiveEntries, appendEntries } = require('../../services/archiveTree'); const { ValidationError, ForbiddenError } = require('../../errors/AppError'); const logger = require('../../utils/logger'); const { collectInputPaths, encodeContentDisposition, stripBasePath, toPosix } = require('./utils'); const router = require('express').Router(); +const { clientAddress } = require('../../utils/clientAddress'); const getLogicalSegments = (relativePath = '') => toPosix(relativePath).split('/').filter(Boolean); @@ -44,24 +47,33 @@ const handleDownloadRequest = async (paths, req, res, basePath = '') => { const context = { user: req.user, guestSession: req.guestSession }; - const targets = await Promise.all( - normalizedPaths.map(async (relativePath) => { - const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); - - if ( - !accessInfo || - !accessInfo.canAccess || - !accessInfo.canRead || - !accessInfo.canDownload || - !resolved - ) { - throw new ForbiddenError(accessInfo?.denialReason || 'Download not allowed.'); - } + // The list came in the request, so the number of resolutions in flight is + // not the client's to choose. + const targets = await mapWithConcurrency(normalizedPaths, async (relativePath) => { + const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); + + if ( + !accessInfo || + !accessInfo.canAccess || + !accessInfo.canRead || + !accessInfo.canDownload || + !resolved + ) { + throw new ForbiddenError(accessInfo?.denialReason || 'Download not allowed.'); + } - const { absolutePath, relativePath: logicalPath } = resolved; - const stats = await fs.stat(absolutePath); - return { relativePath: logicalPath, absolutePath, stats }; - }) + const { absolutePath, relativePath: logicalPath } = resolved; + const stats = await fs.stat(absolutePath); + const shareId = resolved.shareInfo?.sharingType === 'anyone' ? resolved.shareInfo.id : null; + return { relativePath: logicalPath, absolutePath, stats, shareId }; + }); + + // A public link's own counter, for a download that came through the files + // route rather than the share one: the same fetch, reached by a different + // address, and a last-downloaded date that skipped it was simply wrong. + const shareDownloadIds = [...new Set(targets.map(({ shareId }) => shareId).filter(Boolean))]; + await mapWithConcurrency(shareDownloadIds, (shareId) => + trackShareDownload(shareId, { ipAddress: clientAddress(req) }) ); // What left, named once for the whole request: a selection is one download @@ -120,7 +132,7 @@ const handleDownloadRequest = async (paths, req, res, basePath = '') => { res.setHeader('Content-Type', 'application/zip'); res.setHeader('Content-Disposition', encodeContentDisposition(archiveName)); - const archive = archiver('zip', { zlib: { level: 1 } }); + const archive = new ZipArchive({ zlib: { level: 1 } }); archive.on('error', (archiveError) => { logger.error({ err: archiveError }, 'Archive creation failed'); if (!res.headersSent) { diff --git a/backend/src/routes/files/rename.js b/backend/src/routes/files/rename.js index a576e128b..980a207b5 100644 --- a/backend/src/routes/files/rename.js +++ b/backend/src/routes/files/rename.js @@ -1,7 +1,7 @@ const { normalizeRelativePath } = require('../../utils/pathUtils'); -const { renameEntry } = require('../../services/renameService'); const asyncHandler = require('../../utils/asyncHandler'); const { buildItemMetadata } = require('./utils'); +const { renameEntry } = require('../../services/renameService'); const router = require('express').Router(); @@ -9,9 +9,10 @@ router.post( '/files/rename', asyncHandler(async (req, res) => { const parentRelative = normalizeRelativePath(req.body?.path ?? ''); + const context = { user: req.user, guestSession: req.guestSession }; const renamed = await renameEntry({ - context: { user: req.user, guestSession: req.guestSession }, + context, parentRelative, currentName: req.body?.name, newName: req.body?.newName, diff --git a/backend/src/routes/files/transfer.js b/backend/src/routes/files/transfer.js index 3da76385c..0f49f9890 100644 --- a/backend/src/routes/files/transfer.js +++ b/backend/src/routes/files/transfer.js @@ -1,30 +1,112 @@ -const { transferItems } = require('../../services/fileTransferService'); +const { sanitizeClientMessage } = require('../../middleware/errorHandler'); +const { prepareTransfer, executeTransfer } = require('../../services/fileTransferService'); +const recentDestinations = require('../../services/recentDestinationsService'); +const { ACTIONS, authorizeAndResolve } = require('../../services/authorizationService'); +const fs = require('node:fs/promises'); const asyncHandler = require('../../utils/asyncHandler'); +const { startNdjsonStream } = require('../../utils/ndjsonStream'); const router = require('express').Router(); -router.post( - '/files/copy', +// Copy/move stream newline-delimited JSON events so the client can render a +// determinate progress bar: +// {type:'start', totalBytes, totalItems, destination} +// {type:'progress', copiedBytes, totalBytes, currentName} (throttled) +// {type:'done', success, destination, items} +// {type:'error', message, code} +// Validation/authorization runs first (prepareTransfer); if it throws, no +// streaming header has been sent yet, so asyncHandler forwards it to the error +// middleware and the client gets a normal HTTP error response. +const runTransfer = (operation) => asyncHandler(async (req, res) => { const { items = [], destination = '' } = req.body || {}; - const result = await transferItems(items, destination, 'copy', { - user: req.user, - guestSession: req.guestSession, - }); - res.json({ success: true, ...result }); - }) -); + const controller = new AbortController(); + const abort = () => controller.abort(); + const onClose = () => { + if (!res.writableEnded) abort(); + }; + req.once('aborted', abort); + const options = { user: req.user, guestSession: req.guestSession, signal: controller.signal }; + let streaming = false; + let writeEvent = () => {}; + + try { + const prep = await prepareTransfer(items, destination, operation, options); + + streaming = true; + writeEvent = startNdjsonStream(res, { onClose }); + + writeEvent({ + type: 'start', + totalBytes: prep.totalBytes, + totalItems: prep.totalItems, + destination: prep.destinationRelative, + }); + + const result = await executeTransfer( + prep, + operation, + (progress) => writeEvent({ type: 'progress', ...progress }), + { signal: controller.signal } + ); + + // Recorded from the transfer itself rather than asked of the client, so + // every route into a folder counts — the picker, a drag onto a favorite, + // a paste — and the list reflects where things really go. + await recentDestinations.record(req.user?.id, prep.destinationRelative); -router.post( - '/files/move', + writeEvent({ type: 'done', success: true, ...result }); + } catch (error) { + // Keep authorization/validation failures as ordinary HTTP errors. Once + // streaming starts, the error belongs to the NDJSON operation stream. + if (!streaming) throw error; + writeEvent({ + type: 'error', + message: sanitizeClientMessage(error.message || 'Transfer failed.'), + code: error.code || 'TRANSFER_FAILED', + }); + } finally { + req.off('aborted', abort); + res.off('close', onClose); + if (streaming && !res.writableEnded) res.end(); + } + }); + +/** + * Where this user has recently moved or copied things. + * + * Filtered against what they can reach right now: a folder can be deleted or + * have its access revoked long after it was last used, and offering it as a + * destination would only produce a failure at the end of the flow. Anything + * gone is forgotten on the way out, so the list heals itself. + */ +router.get( + '/files/recent-destinations', asyncHandler(async (req, res) => { - const { items = [], destination = '' } = req.body || {}; - const result = await transferItems(items, destination, 'move', { - user: req.user, - guestSession: req.guestSession, - }); - res.json({ success: true, ...result }); + const paths = await recentDestinations.list(req.user?.id); + const context = { user: req.user, guestSession: req.guestSession }; + + const reachable = []; + for (const relativePath of paths) { + const { allowed, resolved } = await authorizeAndResolve( + context, + relativePath, + ACTIONS.upload + ); + const stats = resolved ? await fs.stat(resolved.absolutePath).catch(() => null) : null; + + if (allowed && stats?.isDirectory()) { + reachable.push(relativePath); + } else { + await recentDestinations.forget(req.user?.id, relativePath); + } + } + + res.json({ items: reachable }); }) ); +router.post('/files/copy', runTransfer('copy')); +router.post('/files/move', runTransfer('move')); + module.exports = router; diff --git a/backend/src/routes/files/utils.js b/backend/src/routes/files/utils.js index 3e47da5c0..7accb0802 100644 --- a/backend/src/routes/files/utils.js +++ b/backend/src/routes/files/utils.js @@ -47,7 +47,7 @@ const collectInputPaths = (...sources) => { if (typeof value.path === 'string' && typeof value.name === 'string') { try { add(combineRelativePath(value.path, value.name)); - } catch (error) { + } catch (_) { // ignore invalid combined paths and continue collecting } return; diff --git a/backend/src/routes/index.js b/backend/src/routes/index.js index 95d477b10..ab6ea1c97 100644 --- a/backend/src/routes/index.js +++ b/backend/src/routes/index.js @@ -6,6 +6,7 @@ const thumbnailRoutes = require('./thumbnails'); const editorRoutes = require('./editor'); const volumeRoutes = require('./volumes'); const usageRoutes = require('./usage'); +const folderSizeRoutes = require('./folderSize'); const favoritesRoutes = require('./favorites'); const settingsRoutes = require('./settings'); const searchRoutes = require('./search'); @@ -19,20 +20,17 @@ const permissionsRoutes = require('./permissions'); const sharesRoutes = require('./shares'); const zipRoutes = require('./zip'); const archiveRoutes = require('./archive'); -const activityRoutes = require('./activity'); -const openapiRoutes = require('./openapi'); -const capabilitiesRoutes = require('./capabilities'); -const healthRoutes = require('./health'); const userVolumesRoutes = require('./userVolumes'); -const folderSizeRoutes = require('./folderSize'); const trashRoutes = require('./trash'); const versionsRoutes = require('./versions'); const versionsAdminRoutes = require('./versionsAdmin'); +const activityRoutes = require('./activity'); +const capabilitiesRoutes = require('./capabilities'); +const openapiRoutes = require('./openapi'); const { onlyoffice, collabora } = require('../config/index'); const registerRoutes = (app) => { // Health endpoints (no /api prefix, unauthenticated) - app.use('/', healthRoutes); app.use('/api/auth', authRoutes); app.use('/api', uploadRoutes); @@ -41,6 +39,7 @@ const registerRoutes = (app) => { app.use('/api', editorRoutes); app.use('/api', volumeRoutes); app.use('/api', usageRoutes); + app.use('/api', folderSizeRoutes); app.use('/api', favoritesRoutes); app.use('/api', settingsRoutes); app.use('/api', thumbnailRoutes); @@ -50,13 +49,13 @@ const registerRoutes = (app) => { app.use('/api', permissionsRoutes); app.use('/api', zipRoutes); app.use('/api', archiveRoutes); - app.use('/api', activityRoutes); - app.use('/api', openapiRoutes); - app.use('/api', capabilitiesRoutes); - app.use('/api', folderSizeRoutes); app.use('/api', trashRoutes); - app.use('/api', versionsRoutes); + // Before the per-file routes: `/versions/admin/…` must not be read as a + // version id with a suffix. app.use('/api', versionsAdminRoutes); + app.use('/api', versionsRoutes); + app.use('/api', activityRoutes); + app.use('/api', capabilitiesRoutes); // User volumes management (admin only, requires USER_VOLUMES feature) app.use('/api', userVolumesRoutes); // Share routes (supports guest sessions) @@ -64,6 +63,8 @@ const registerRoutes = (app) => { app.use('/api/share', sharesRoutes); // Public features endpoint (always available) app.use('/api', featuresRoutes); + // The API's own description, also answered to anybody + app.use('/api', openapiRoutes); // Admin-only terminal session endpoint app.use('/api', terminalRoutes); // Mount ONLYOFFICE routes only when configured diff --git a/backend/src/routes/onlyoffice.js b/backend/src/routes/onlyoffice.js index 8796a5b82..d49918847 100644 --- a/backend/src/routes/onlyoffice.js +++ b/backend/src/routes/onlyoffice.js @@ -2,41 +2,36 @@ const express = require('express'); const path = require('path'); const fs = require('fs'); const fsp = require('fs/promises'); -const crypto = require('crypto'); const { pipeline } = require('stream/promises'); +const crypto = require('crypto'); const axios = require('axios'); const jwt = require('jsonwebtoken'); -const { onlyoffice, public: publicConfig, mimeTypes } = require('../config/index'); +const { onlyoffice, public: publicConfig } = require('../config/index'); +const { toExtension, resolveMimeType } = require('../utils/fileTypes'); +const { getDocumentType } = require('../utils/onlyofficeDocumentTypes'); const { + normalizeRelativePath, combineRelativePath, ensureValidName, - normalizeRelativePath, } = require('../utils/pathUtils'); const { ensureDir } = require('../utils/fsUtils'); const { placeWithoutOverwrite } = require('../utils/placeWithoutOverwrite'); -const { track: trackInFlight } = require('../services/inFlightFiles'); const { resolvePathWithAccess } = require('../services/accessManager'); const { renameEntry } = require('../services/renameService'); -const versions = require('../services/versions/operations'); -const onlyofficeActivity = require('../services/onlyofficeActivityService'); -const documentKeys = require('../services/onlyofficeDocumentKeyService'); -const editorSessions = require('../services/onlyofficeEditorSessionService'); -const { getDocumentType } = require('../utils/onlyofficeDocumentTypes'); const logger = require('../utils/logger'); const asyncHandler = require('../utils/asyncHandler'); const { ValidationError, UnauthorizedError, ForbiddenError } = require('../errors/AppError'); +const folderSizeHooks = require('../services/folderSizeHooks'); +const onlyofficeActivity = require('../services/onlyofficeActivityService'); +const documentKeys = require('../services/onlyofficeDocumentKeyService'); +const versions = require('../services/versions/operations'); +const { track: trackInFlight } = require('../services/inFlightFiles'); -const router = express.Router(); - -// The backend token is signed with the same secret as the Document Server -// tokens, so it carries a type claim to keep the two apart, and a lifetime long -// enough for an editing session but not indefinite. It used to have neither: a -// token from the Document Server would have been accepted as one of ours, and -// one of ours never expired. -const BACKEND_TOKEN_TYPE = 'nextexplorer-backend'; -const BACKEND_TOKEN_TTL_SECONDS = 12 * 60 * 60; +const editorSessions = require('../services/onlyofficeEditorSessionService'); +const { markLongPoll } = require('../middleware/heldRequests'); +const router = express.Router(); // In-flight force-save requests only: these are meaningless once the process // that issued them is gone, unlike the sessions they refer to. const pendingForceSaves = new Map(); @@ -44,17 +39,40 @@ const pendingForceSavesBySession = new Map(); const FORCE_SAVE_RETRY_DELAYS_MS = [250, 750, 1500, 2500]; -// Helpers -const toExt = (filename = '') => String(filename).split('.').pop().toLowerCase(); +// The backend token is signed with the same secret as the Document Server +// tokens, so it carries a type claim to keep the two apart, and a lifetime +// long enough for an editing session but not indefinite. +const BACKEND_TOKEN_TYPE = 'nextexplorer-backend'; +const BACKEND_TOKEN_TTL_SECONDS = 12 * 60 * 60; -const resolveMime = (ext) => mimeTypes[ext] || 'application/octet-stream'; +/** + * Read a backend token from the query string. + * + * Returns null unless the token is valid, is a backend token (not a Document + * Server one signed with the same secret) and carries an absolute path. + */ +const readBackendToken = (req) => { + const raw = typeof req.query?.backend === 'string' ? req.query.backend : null; + if (!raw || !onlyoffice.secret) return null; + try { + const payload = jwt.verify(raw, onlyoffice.secret, { algorithms: ['HS256'] }); + if (!payload || typeof payload !== 'object') return null; + if (payload.typ !== BACKEND_TOKEN_TYPE) return null; + if (typeof payload.absolutePath !== 'string' || !payload.absolutePath) return null; + return payload; + } catch (e) { + logger.warn({ err: e }, 'ONLYOFFICE backend token verification failed'); + return null; + } +}; /** - * The addresses a saved document may be fetched from. + * Document Server origins we accept a saved document from. * - * The Document Server's own, and any declared beside it: behind a proxy or - * inside a container network it sometimes reports itself under a host other - * than the one it is called on. + * The callback hands us a URL to download the edited file; without this check + * the server would fetch any address an authorized editor asks for. Extra + * origins can be declared when the Document Server reports itself under a + * different host than the one we call it on. */ const buildAllowedDownloadOrigins = () => { const origins = new Set(); @@ -71,12 +89,6 @@ const buildAllowedDownloadOrigins = () => { return origins; }; -/** - * The callback says where to fetch the saved document from, and the server - * fetched whatever it was told to: an address on the machine itself, or inside - * the network the container sits in, reached by anyone who can reach the - * callback. It has to come from the Document Server we sent the document to. - */ const ensureAllowedDownloadUrl = (rawUrl) => { let parsed; try { @@ -99,13 +111,19 @@ const ensureAllowedDownloadUrl = (rawUrl) => { }; /** - * Pull the saved document into a file of its own, next to the document. - * - * Never into the document itself: it used to be truncated to nothing before - * the Document Server had answered, so a slow network, a restart or a refused - * download left an empty file where the work had been. The versions place the - * temporary file over the document once it is whole. + * A backend token lives 12 hours; the share it was issued for may not. + * Confirm the share still exists before honouring the token's write claim. */ +const assertShareStillValid = async (backendCtx) => { + if (!backendCtx?.shareToken) return; + const { getShareByToken, isShareExpired } = require('../services/sharesService'); + const share = await getShareByToken(backendCtx.shareToken); + if (!share || isShareExpired(share)) { + throw new ForbiddenError('The share for this editing session is no longer available.'); + } +}; + +/** Pull a document the Document Server prepared into a new file. */ const fetchDocumentInto = async (downloadUrl, temporaryPath, mode) => { const response = await axios.get(downloadUrl, { responseType: 'stream', timeout: 30000 }); await pipeline(response.data, fs.createWriteStream(temporaryPath, { flags: 'wx', mode })); @@ -114,20 +132,16 @@ const fetchDocumentInto = async (downloadUrl, temporaryPath, mode) => { }; /** - * Who a callback is saving for. - * - * The Document Server names the people whose changes are in this save; the - * last of them is the one to credit. It says nothing when a save carries no - * history — the first one of a session — and then the account the editing - * session was opened for is the answer. Somebody who came through a share link - * is credited as the link: there is no account to name. - */ -/** - * Pull a document the Document Server prepared into a new file in `directory`. + * Pull a document the Document Server prepared into a new file in `directory`, + * under `desiredName` or the first free name after it, "report (1).pdf". + * Answers the name and path it took. * - * Never over anything: a name already taken, before the download or during it, - * gets the same "(1)" treatment as everywhere else, and the caller is told the - * name actually used. + * Written to a temporary name in the directory first, so a slow or failed + * response never leaves a document truncated to nothing, then put under its + * name by a move that never replaces anything. Choosing the name before the + * download and renaming over it afterwards replaced whatever arrived under that + * name while the Document Server was answering. A save over the document itself + * goes through the versions instead. */ const downloadDocumentInto = async (downloadUrl, directory, desiredName, mode = 0o600) => { const temporaryPath = path.join( @@ -145,6 +159,11 @@ const downloadDocumentInto = async (downloadUrl, directory, desiredName, mode = } }; +/** + * Who wrote the state a save callback carries. The Document Server lists the + * changes and who made them, the last one first to know; a visitor through a + * share link is not an account. + */ const authorFromCallback = (body, backendCtx) => { const changes = Array.isArray(body?.history?.changes) ? body.history.changes : []; const user = changes.length ? changes[changes.length - 1]?.user : null; @@ -156,40 +175,43 @@ const authorFromCallback = (body, backendCtx) => { }; /** - * Read a backend token from the query string. + * The key this document is currently open under. * - * Returns null unless the token is valid, is a backend token — not a Document - * Server one signed with the same secret — and carries an absolute path. + * Delegated to the key store: it has to stay the same for everyone who has the + * document open — including across the saves they are making, which change the + * file — and change once the Document Server has let go. Computing it from the + * file alone, as this used to, meant the second person to open a document that + * had just been saved got a different key, and therefore a separate editing + * session on the same file, with no sign that anyone else was in it. */ -const readBackendToken = (req) => { - const raw = typeof req.query?.backend === 'string' ? req.query.backend : null; - if (!raw || !onlyoffice.secret) return null; - try { - const payload = jwt.verify(raw, onlyoffice.secret, { algorithms: ['HS256'] }); - if (!payload || typeof payload !== 'object') return null; - if (payload.typ !== BACKEND_TOKEN_TYPE) return null; - if (typeof payload.absolutePath !== 'string' || !payload.absolutePath) return null; - return payload; - } catch (error) { - logger.warn({ err: error }, 'ONLYOFFICE backend token verification failed'); - return null; - } +const resolveKeyForOpen = async ({ absolutePath, relativePath, stat, documentType }) => { + const presence = onlyofficeActivity.get(absolutePath); + return documentKeys.resolveDocumentKey({ + relativePath, + stat, + documentType, + inUse: Boolean(presence?.active), + }); }; -/** - * A backend token lives twelve hours; the share it was issued for may not. - * Confirm the share still exists before honouring the token's write claim. - */ -const assertShareStillValid = async (backendCtx) => { - if (!backendCtx?.shareToken) return; - // Required here rather than at the top: the shares service reaches back into - // routes for its own helpers. - // eslint-disable-next-line global-require - const { getShareByToken, isShareExpired } = require('../services/sharesService'); - const share = await getShareByToken(backendCtx.shareToken); - if (!share || isShareExpired(share)) { - throw new ForbiddenError('The share for this editing session is no longer available.'); +const getCommandServiceUrl = (key, legacy = false) => { + const commandUrl = new URL( + legacy ? 'coauthoring/CommandService.ashx' : 'command', + `${onlyoffice.serverUrl.replace(/\/+$/, '')}/` + ); + if (!legacy) commandUrl.searchParams.set('shardkey', key); + return commandUrl.toString(); +}; + +const getDsJwtFromReq = (req) => { + const auth = (req.headers['authorization'] || req.headers['authorizationjwt'] || '').toString(); + if (auth.toLowerCase().startsWith('bearer ')) { + return auth.slice(7).trim(); } + const q = req.query || {}; + if (typeof q.token === 'string' && q.token) return q.token; + if (typeof q.jwt === 'string' && q.jwt) return q.jwt; + return null; }; const getSessionOwner = (req) => ({ @@ -202,31 +224,72 @@ const matchesSessionOwner = (session, req) => { return owner.userId === session.userId && owner.guestSessionId === session.guestSessionId; }; +const cleanupExpiredEditorSessions = () => { + void editorSessions.purgeExpired(); +}; + +/** + * Presence is deliberately not recorded here. + * + * This runs when the editor asks for its configuration, which says nothing + * about whether the document will open. A file the editor then refused — a + * drawing announced with the wrong editor, say — was still displayed as being + * edited, by everyone, until the session expired. The client reports presence + * once ONLYOFFICE says the document is ready, through the heartbeat below. + */ +const createEditorSession = async (req, relativePath, key, absolutePath) => { + cleanupExpiredEditorSessions(); + const sessionId = crypto.randomUUID(); + const owner = getSessionOwner(req); + await editorSessions.create({ + sessionId, + key, + relativePath, + // Where the document is *now*. The backend token carries the path as it + // was when the editor opened, and renaming makes that copy wrong; a save + // arriving afterwards would recreate the old name beside the new one. + absolutePath, + ...owner, + }); + return sessionId; +}; + +/** + * Where a save should be written for this token. + * + * The token is minted once and handed to the Document Server, which returns it + * unchanged however long the editing session lasts. The session is what follows + * the document if it is renamed meanwhile — and it is stored, so a restart no + * longer forgets the rename and put the save back under the old name. The token + * remains the fallback for a session that has genuinely expired. + */ +const resolveSaveTarget = async (backendCtx) => { + const session = backendCtx?.sessionId ? await editorSessions.get(backendCtx.sessionId) : null; + return session?.absolutePath || backendCtx.absolutePath; +}; + const describeSessionUser = (req) => { const owner = getSessionOwner(req); return { id: owner.userId || (owner.guestSessionId ? `guest_${owner.guestSessionId}` : null), - name: req.user?.displayName || req.user?.username || (owner.guestSessionId ? 'Guest' : 'User'), + name: + req.user?.displayName || + req.user?.username || + (owner.guestSessionId ? 'Invité' : 'Utilisateur'), }; }; -const getCommandServiceUrl = (key, legacy = false) => { - const commandUrl = new URL( - legacy ? 'coauthoring/CommandService.ashx' : 'command', - `${onlyoffice.serverUrl.replace(/\/+$/, '')}/` - ); - if (!legacy) commandUrl.searchParams.set('shardkey', key); - return commandUrl.toString(); +const getEditorSession = async (req, sessionId, relativePath) => { + const session = await editorSessions.get(sessionId); + if (!session || session.relativePath !== relativePath || !matchesSessionOwner(session, req)) { + throw new ForbiddenError( + 'The ONLYOFFICE editing session is no longer valid. Reopen the document.' + ); + } + await editorSessions.touch(sessionId); + return session; }; -/** - * Ask the Document Server to write what the editor holds, now. - * - * Closing the preview used to rely on the status-2 callback the Document Server - * sends when it decides the document is finished with, which arrives seconds - * later — long enough for the folder to be listed again with the old content, - * and for the tab to be gone before anything was written. - */ const enqueueForceSave = ({ sessionId, key, relativePath, reason }) => { const requestId = `nextexplorer-force-save:${crypto.randomUUID()}`; const timeout = setTimeout( @@ -246,6 +309,10 @@ const enqueueForceSave = ({ sessionId, key, relativePath, reason }) => { }); pendingForceSavesBySession.set(sessionId, requestId); + logger.debug( + { path: relativePath, requestId, reason }, + 'ONLYOFFICE force-save accepted by NextExplorer' + ); setImmediate(() => { void dispatchForceSave({ requestId, key, relativePath, reason }); }); @@ -263,29 +330,46 @@ const finishForceSave = (requestId, result) => { clearTimeout(pending.timeout); if (pending.retryTimer) clearTimeout(pending.retryTimer); logger.debug( - { requestId, reason: pending.reason, elapsedMs: Date.now() - pending.requestedAt, ...result }, + { + requestId, + reason: pending.reason, + elapsedMs: Date.now() - pending.requestedAt, + ...result, + }, 'ONLYOFFICE force-save finished' ); - // A close may arrive while an automatic save is still assembling an earlier - // version. Queue one final command so the most recent edits do not depend on - // the delayed callback. + // A close may arrive while an automatic save is assembling an earlier + // version. Queue one final command so the most recent edits do not rely on + // ONLYOFFICE's delayed status-2 callback. if (pending.followUpReason) { const { sessionId, key, relativePath, followUpReason } = pending; + logger.debug( + { path: relativePath, requestId, reason: followUpReason }, + 'ONLYOFFICE force-save scheduling follow-up request' + ); enqueueForceSave({ sessionId, key, relativePath, reason: followUpReason }); } }; const dispatchForceSave = async ({ requestId, key, relativePath, reason, attempt = 0 }) => { try { - const command = { c: 'forcesave', key, userdata: requestId }; + logger.debug( + { path: relativePath, requestId, reason, attempt }, + 'ONLYOFFICE force-save dispatching' + ); + const command = { + c: 'forcesave', + key, + userdata: requestId, + }; command.token = jwt.sign(command, onlyoffice.secret, { algorithm: 'HS256' }); let response = await axios.post(getCommandServiceUrl(key), command, { timeout: 8000, validateStatus: () => true, }); - // ONLYOFFICE Docs 8.2 introduced /command. Keep older Document Server + // ONLYOFFICE Docs 8.2 introduced /command. Keep legacy Document Server // installations working when they explicitly report the new route absent. if (response.status === 404) { response = await axios.post(getCommandServiceUrl(key, true), command, { @@ -295,10 +379,16 @@ const dispatchForceSave = async ({ requestId, key, relativePath, reason, attempt } const code = Number(response.data?.error ?? 0); - if (response.status >= 200 && response.status < 300 && code === 0) return; + if (response.status >= 200 && response.status < 300 && code === 0) { + logger.debug( + { path: relativePath, requestId, reason, status: response.status }, + 'ONLYOFFICE force-save accepted by Document Server' + ); + return; + } - // Code 4 means the editor has not yet sent its last changes to the Document - // Server. Retried here so that closing the preview stays instant. + // Code 4 means the document editor has not yet sent its last changes to + // Document Server. Retry server-side so closing the preview stays instant. if (code === 4 && attempt < FORCE_SAVE_RETRY_DELAYS_MS.length) { const pending = pendingForceSaves.get(requestId); if (!pending) return; @@ -324,89 +414,237 @@ const dispatchForceSave = async ({ requestId, key, relativePath, reason, attempt }; /** - * The key to hand this editor. + * Which ONLYOFFICE theme to dress the editor in. * - * Whether anyone currently has the document open is what separates "the file - * changed because we are editing it" from "the file changed while nobody was - * looking": the first must keep the key, the second must not. + * The appearance is a client preference, but it has to be decided here: the + * Document Server reads the configuration from the signed token and ignores + * whatever the page sets on the object afterwards. So the client sends what it + * is currently showing, and anything unrecognised falls back to the editor's + * own default rather than being passed through. */ -const resolveKeyForOpen = async ({ absolutePath, relativePath, stat, documentType }) => { - const presence = onlyofficeActivity.get(absolutePath); - return documentKeys.resolveDocumentKey({ - relativePath, - stat, - documentType, - inUse: Boolean(presence?.active), - }); +const resolveUiTheme = (requested) => { + if (requested === 'dark') return 'theme-dark'; + if (requested === 'light') return 'theme-light'; + return null; }; /** - * Presence is deliberately not recorded here. + * Earlier versions of a document, in the editor. * - * This runs when the editor asks for its configuration, which says nothing - * about whether the document will open. A file the editor then refused — a - * drawing announced with the wrong editor, say — would still be displayed as - * being edited, by everyone, until the session expired. The client reports - * presence once ONLYOFFICE says the document is ready, through the heartbeat. + * The Document Server shows a history when the integration hands it one: the + * list, then — version by version — a URL to fetch that version from. Both go + * through the same rights as the Versions panel, and the URL is signed for the + * version's own content through `/onlyoffice/file`, which serves exactly the + * path its token names and nothing else. + * + * Every version keeps its own key, never the document's: a key is what the + * Document Server caches a document under, and the one the document is open + * under must keep meaning the document as it is now. */ -const createEditorSession = async (req, relativePath, key, absolutePath) => { - void editorSessions.purgeExpired(); - const sessionId = crypto.randomUUID(); - await editorSessions.create({ - sessionId, - key, - relativePath, - // Where the document is *now*. The backend token carries the path as it was - // when the editor opened, and renaming makes that copy wrong; a save - // arriving afterwards would recreate the old name beside the new one. - absolutePath, - ...getSessionOwner(req), - }); - return sessionId; +const versionHistory = () => require('../services/versions'); + +const versionKeyFor = (versionId) => `version-${versionId}`; + +const editorUserOf = (req) => + req.user && req.user.id + ? { id: String(req.user.id), name: req.user.displayName || req.user.username || 'User' } + : req.guestSession + ? { id: `guest_${req.guestSession.id}`, name: 'Guest User' } + : undefined; + +/** A token for `/onlyoffice/file` that serves one content, and never writes. */ +const readOnlyFileUrl = (req, relativePath, absolutePath, ttlSeconds) => { + const backendToken = jwt.sign( + { + typ: BACKEND_TOKEN_TYPE, + absolutePath, + logicalPath: relativePath, + canWrite: false, + sessionId: null, + userId: req.user?.id ? String(req.user.id) : null, + guestSessionId: req.guestSession?.id || null, + shareToken: null, + }, + onlyoffice.secret, + { algorithm: 'HS256', expiresIn: ttlSeconds } + ); + const fileUrl = new URL('/api/onlyoffice/file', publicConfig.url); + fileUrl.searchParams.set('path', relativePath); + fileUrl.searchParams.set('backend', backendToken); + return fileUrl.toString(); }; -const getEditorSession = async (req, sessionId, relativePath) => { - const session = await editorSessions.get(sessionId); - if (!session || session.relativePath !== relativePath || !matchesSessionOwner(session, req)) { - throw new ForbiddenError( - 'The ONLYOFFICE editing session is no longer valid. Reopen the document.' +// The token naming a version's content is always signed: without ONLYOFFICE_SECRET +// the configuration derives a secret of its own. +const requireVersionSetup = () => { + if (!publicConfig?.url) { + throw new ValidationError( + 'PUBLIC_URL is required on the server to build absolute URLs for ONLYOFFICE.' ); } - await editorSessions.touch(sessionId); - return session; -}; - -/** - * Where a save should be written for this token. - * - * The token is minted once and handed to the Document Server, which returns it - * unchanged however long the editing session lasts. The session is what follows - * the document if it is renamed meanwhile — and it is stored, so a restart does - * not forget the rename and put the save back under the old name. The token - * remains the fallback for a session that has genuinely expired. - */ -const resolveSaveTarget = async (backendCtx) => { - const session = backendCtx?.sessionId ? await editorSessions.get(backendCtx.sessionId) : null; - return session?.absolutePath || backendCtx.absolutePath; }; -const getDsJwtFromReq = (req) => { - const auth = (req.headers['authorization'] || req.headers['authorizationjwt'] || '').toString(); - if (auth.toLowerCase().startsWith('bearer ')) { - return auth.slice(7).trim(); - } - const q = req.query || {}; - if (typeof q.token === 'string' && q.token) return q.token; - if (typeof q.jwt === 'string' && q.jwt) return q.jwt; - return null; +/** The key the document is open under now, as the configuration hands it out. */ +const currentKeyOf = async (req, relativePath) => { + const context = { user: req.user, guestSession: req.guestSession }; + const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); + if (!accessInfo?.canAccess || !accessInfo.canRead || !resolved) { + throw new ForbiddenError(accessInfo?.denialReason || 'Access denied.'); + } + const stat = await fsp.stat(resolved.absolutePath); + const documentType = getDocumentType(toExtension(resolved.absolutePath)); + const key = await resolveKeyForOpen({ + absolutePath: resolved.absolutePath, + relativePath, + stat, + documentType, + }); + return { key, absolutePath: resolved.absolutePath }; }; -// POST /api/onlyoffice/config { path, mode? } +/** A version opened on its own, to be read: a viewer, with nothing to save. */ +const versionViewConfig = async (req, relativePath, versionId, uiTheme) => { + requireVersionSetup(); + const context = { user: req.user, guestSession: req.guestSession }; + const located = await versionHistory().locateVersion(context, relativePath, versionId, { + download: false, + }); + const ext = toExtension(located.name); + const documentType = getDocumentType(ext); + if (!documentType) { + throw new ValidationError(`ONLYOFFICE has no editor for .${ext} files.`); + } + const mayCopy = located.target.rights.download; + const config = { + documentType, + type: 'desktop', + document: { + fileType: ext, + key: versionKeyFor(located.version.id), + title: located.name, + url: readOnlyFileUrl(req, relativePath, located.absolutePath, BACKEND_TOKEN_TTL_SECONDS), + permissions: { + edit: false, + comment: false, + review: false, + // Printing or downloading a version is taking a copy of it. + download: mayCopy, + print: mayCopy, + }, + }, + // No callback: nothing is saved from a version, and the one the document + // has would release the key everyone editing it now shares. + editorConfig: { + mode: 'view', + customization: { + anonymous: { request: false }, + close: { visible: true }, + ...(uiTheme ? { uiTheme } : {}), + }, + lang: onlyoffice.lang || 'en', + user: editorUserOf(req), + }, + }; + config.token = jwt.sign(config, onlyoffice.secret, { algorithm: 'HS256' }); + return { + documentServerUrl: onlyoffice.serverUrl, + config, + forceSaveSessionId: null, + editorSessionId: null, + autoSaveIntervalMs: 0, + version: { id: located.version.id, modifiedAt: located.version.modifiedAt }, + }; +}; + +// POST /api/onlyoffice/history { path } +router.post( + '/onlyoffice/history', + asyncHandler(async (req, res) => { + const relativePath = normalizeRelativePath(req.body?.path || ''); + if (!relativePath) throw new ValidationError('A valid file path is required.'); + const context = { user: req.user, guestSession: req.guestSession }; + const listed = await versionHistory().listVersions(context, relativePath); + const { key } = await currentKeyOf(req, relativePath); + + // The editor numbers versions from the oldest; the list comes newest first. + const history = [...listed.versions].reverse().map((version, index) => ({ + version: index + 1, + versionId: version.id, + key: versionKeyFor(version.id), + created: version.modifiedAt, + user: { id: version.author?.id || '', name: version.author?.label || '' }, + label: version.label, + available: version.available !== false, + })); + history.push({ + version: history.length + 1, + versionId: null, + key, + created: listed.file.modifiedAt, + user: { id: listed.file.author?.id || '', name: listed.file.author?.label || '' }, + label: null, + available: true, + }); + + res.set('Cache-Control', 'no-store'); + res.json({ + currentVersion: history.length, + history, + canRestore: listed.rights.restore, + }); + }) +); + +// POST /api/onlyoffice/history-data { path, version, versionId? } +router.post( + '/onlyoffice/history-data', + asyncHandler(async (req, res) => { + requireVersionSetup(); + const relativePath = normalizeRelativePath(req.body?.path || ''); + if (!relativePath) throw new ValidationError('A valid file path is required.'); + const version = Number(req.body?.version); + if (!Number.isInteger(version) || version < 1) { + throw new ValidationError('A version number is required.'); + } + const context = { user: req.user, guestSession: req.guestSession }; + const versionId = typeof req.body?.versionId === 'string' ? req.body.versionId : ''; + + let key; + let absolutePath; + let name; + if (versionId) { + const located = await versionHistory().locateVersion(context, relativePath, versionId, { + download: false, + }); + key = versionKeyFor(located.version.id); + absolutePath = located.absolutePath; + name = located.name; + } else { + // The current state, from inside the history: the same rights decide. + await versionHistory().listVersions(context, relativePath); + ({ key, absolutePath } = await currentKeyOf(req, relativePath)); + name = path.basename(absolutePath); + } + + const payload = { + fileType: toExtension(name), + key, + url: readOnlyFileUrl(req, relativePath, absolutePath, STORAGE_FILE_TOKEN_TTL_SECONDS), + version, + }; + payload.token = jwt.sign(payload, onlyoffice.secret, { algorithm: 'HS256' }); + res.set('Cache-Control', 'no-store'); + res.json(payload); + }) +); + +// POST /api/onlyoffice/config { path, mode?, theme?, versionId? } router.post( '/onlyoffice/config', asyncHandler(async (req, res) => { const relativeRaw = req.body?.path || ''; const mode = (req.body?.mode || 'edit').toLowerCase(); + const uiTheme = resolveUiTheme(String(req.body?.theme || '').toLowerCase()); if (!publicConfig?.url) { throw new ValidationError( @@ -423,14 +661,11 @@ router.post( const relativePath = normalizeRelativePath(relativeRaw); - // An earlier version, opened to be read: a viewer with nothing to save, and - // a key of its own so it never touches the one the document is open under. const requestedVersion = typeof req.body?.versionId === 'string' ? req.body.versionId : ''; if (requestedVersion) { - res.json(await versionViewConfig(req, relativePath, requestedVersion, null)); + res.json(await versionViewConfig(req, relativePath, requestedVersion, uiTheme)); return; } - const context = { user: req.user, guestSession: req.guestSession }; const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); @@ -448,19 +683,27 @@ router.post( // Disable editing for readonly shares, readonly locations, or view mode. // Computed before the backend token is signed: the token carries this - // decision, so a viewer never receives one that allows writing. It used to - // ignore the location's own rights, so somebody who could only read a - // folder was handed an editing session on the documents in it. + // decision, so a viewer never receives one that allows writing. const canEdit = mode !== 'view' && !isReadonlyShare && accessInfo.canWrite === true; + /** + * Whether the reader may annotate — which nothing grants separately yet, so + * for now it is exactly whoever may edit. + * + * It exists as its own name because commenting is not a weaker kind of + * editing: a comment leaves the content alone and still rewrites the file, + * since a .docx keeps its comments inside its own OOXML. Granting it will + * mean a third state in the backend token below, not a looser boolean — + * see TODO.md. + */ + const canComment = canEdit; + const filename = path.basename(abs); - const ext = toExt(filename); + const ext = toExtension(filename); const documentType = getDocumentType(ext); if (!documentType) { - // Refused here rather than left for the Document Server to open with the - // wrong editor: everything used to fall back to 'word', so a drawing was - // answered "the file content does not match the file extension" — true, - // unhelpful, and several steps from the setting that caused it. + // Refuse here rather than let the Document Server open it with the wrong + // editor: the answer it gives back names the file, never the setting. throw new ValidationError( `ONLYOFFICE has no editor for .${ext} files. Remove it from ONLYOFFICE_FILE_EXTENSIONS, ` + 'or open it with Collabora instead.' @@ -474,9 +717,7 @@ router.post( callbackUrl.searchParams.set('path', relativePath); // Shared with anyone already in this document, so they edit together rather - // than in two sessions that overwrite each other. It used to be recomputed - // from the file's own state on every open, which changed it under the - // people already editing. + // than in two sessions that overwrite each other. const key = await resolveKeyForOpen({ absolutePath: abs, relativePath, @@ -484,23 +725,25 @@ router.post( documentType, }); - // Only an editing session gets one: it is what a save is written through. - const editorSessionId = canEdit ? await createEditorSession(req, relativePath, key, abs) : null; + // canEdit is decided above, before the backend token is signed. + const forceSaveSessionId = canEdit + ? await createEditorSession(req, relativePath, key, abs) + : null; // Backend context for storage requests (signed separately and passed via query) - let backendToken = null; + let backendToken; if (onlyoffice.secret) { const backendPayload = { typ: BACKEND_TOKEN_TYPE, absolutePath: abs, logicalPath: resolved.relativePath, space: resolved.space, - // The callback trusts this flag instead of re-resolving permissions, so - // it must say what this session is actually allowed to do. + // The callback trusts this flag instead of re-resolving permissions, + // so it must reflect what this session is actually allowed to do. canWrite: canEdit, - // Lets a save find the document again if it was renamed while open; the - // path above is only what it was called when the editor started. - sessionId: editorSessionId, + // Lets a save find the document again if it was renamed while open; + // the path above is only what it was called when the editor started. + sessionId: forceSaveSessionId, userId: req.user && req.user.id ? String(req.user.id) : null, guestSessionId: req.guestSession?.id || null, shareToken: resolved.shareInfo?.shareToken || null, @@ -514,7 +757,7 @@ router.post( } const config = { - documentType, // text | spreadsheet | presentation + documentType, // word | cell | slide | pdf | diagram type: 'desktop', document: { fileType: ext, @@ -523,20 +766,36 @@ router.post( url: fileUrl.toString(), permissions: { edit: canEdit, + // Stated rather than inherited: ONLYOFFICE defaults `comment` to the + // value of `edit`, so leaving it out reads as "we did not think about + // it" and moves with their default if it ever changes. + comment: canComment, download: true, print: true, review: canEdit, }, }, editorConfig: { - mode: canEdit ? 'edit' : 'view', + // 'view' is a viewer, not a read-only editor: it hides the comment UI + // even when `comment` is granted. So the mode follows whether there is + // anything at all to do in the document, and the permissions above say + // what that is. Tying it to `canEdit` is what makes a comment-only + // reader impossible to express. + mode: canEdit || canComment ? 'edit' : 'view', callbackUrl: callbackUrl.toString(), customization: { anonymous: { request: false }, - // Expose ONLYOFFICE's own Save action as a force-save when it has - // been asked for. Closing the document is flushed by the route - // above, whether or not this is on. + // Expose ONLYOFFICE's Save action as a force-save when explicitly + // requested. Closing the editor is handled by the route below. forcesave: Boolean(onlyoffice.forceSave && canEdit), + // Let the editor draw its own close button. NextExplorer used to lay + // one over the toolbar, which meant covering the editor's logo and + // hoping nothing underneath moved; the client closes the preview when + // ONLYOFFICE asks it to instead. + close: { visible: true }, + // Omitted when the client sends no usable preference, so the editor + // keeps its own default instead of being forced light. + ...(uiTheme ? { uiTheme } : {}), }, lang: onlyoffice.lang || 'en', // Optionally attach current user info if available @@ -550,400 +809,110 @@ router.post( ? { id: `guest_${req.guestSession.id}`, name: 'Guest User', - } - : undefined, - }, - }; - - // Sign config for Document Server when ONLYOFFICE JWT is enabled - if (onlyoffice.secret) { - try { - // Important: sign the final config as-is; do not mutate URLs afterwards - const token = jwt.sign(config, onlyoffice.secret, { - algorithm: 'HS256', - }); - config.token = token; - } catch (e) { - logger.warn({ err: e }, 'ONLYOFFICE: failed to sign config token'); - } - } - - res.json({ - documentServerUrl: onlyoffice.serverUrl, - config, - editorSessionId, - autoSaveIntervalMs: canEdit ? onlyoffice.autoSaveIntervalMs : 0, - }); - }) -); - -/** - * The client says the document is really open, and goes on saying so. - * - * Presence starts here rather than when the configuration is handed out: that - * says nothing about whether the document opened, and a file the editor then - * refused was still shown to everybody as being edited until it expired. - */ -router.post( - '/onlyoffice/session-heartbeat', - asyncHandler(async (req, res) => { - const relativePath = normalizeRelativePath(req.body?.path || ''); - const sessionId = req.body?.sessionId || ''; - if (!relativePath || typeof sessionId !== 'string' || !sessionId) { - throw new ValidationError('A valid ONLYOFFICE editing session is required.'); - } - const context = { user: req.user, guestSession: req.guestSession }; - const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); - if (!accessInfo?.canAccess || !accessInfo.canRead) throw new ForbiddenError('Access denied.'); - await getEditorSession(req, sessionId, relativePath); - - const active = onlyofficeActivity.touch({ - absolutePath: resolved.absolutePath, - sessionId, - user: describeSessionUser(req), - }); - res.json({ active }); - }) -); - -/** - * The editor was closed. The session ends, so the document stops being reported - * as open by somebody who has left. - */ -router.post( - '/onlyoffice/session-end', - asyncHandler(async (req, res) => { - const relativePath = normalizeRelativePath(req.body?.path || ''); - const sessionId = req.body?.sessionId || ''; - if (!relativePath || typeof sessionId !== 'string' || !sessionId) { - throw new ValidationError('A valid ONLYOFFICE editing session is required.'); - } - const context = { user: req.user, guestSession: req.guestSession }; - const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); - if (!accessInfo?.canAccess || !accessInfo.canRead) throw new ForbiddenError('Access denied.'); - const session = await getEditorSession(req, sessionId, relativePath); - - // Somebody who was only reading has nothing to flush, and an integration - // with no Document Server has nowhere to ask. Neither is a reason to refuse - // the close — the session still has to end, or the document goes on being - // reported as open by somebody who has left. - let requestId = null; - if (onlyoffice.serverUrl && accessInfo.canWrite) { - requestId = - pendingForceSavesBySession.get(sessionId) || - enqueueForceSave({ sessionId, key: session.key, relativePath, reason: 'close' }); - } - - onlyofficeActivity.close({ absolutePath: resolved.absolutePath, sessionId }); - await editorSessions.remove(sessionId); - res.json({ ended: true, flushed: Boolean(requestId), requestId }); - }) -); - -/** - * Rename the open document from the editor's title bar. - * - * The rename itself is the ordinary one, with the ordinary permission checks. - * What is specific here is keeping the editing session pointed at the file - * afterwards: the Document Server holds a token naming the path as it was when - * the editor opened, and returns it unchanged with every save. Left alone, the - * next autosave would recreate the old name beside the new one. - */ -router.post( - '/onlyoffice/rename', - asyncHandler(async (req, res) => { - const relativePath = normalizeRelativePath(req.body?.path || ''); - const sessionId = req.body?.sessionId || ''; - if (!relativePath || typeof sessionId !== 'string' || !sessionId) { - throw new ValidationError('A valid ONLYOFFICE editing session is required.'); - } - - // Only the session that opened this document may rename it from inside the - // editor, and only sessions allowed to write ever get one. - const session = await getEditorSession(req, sessionId, relativePath); - - const parentPath = path.posix.dirname(relativePath); - const renamed = await renameEntry({ - context: { user: req.user, guestSession: req.guestSession }, - parentRelative: parentPath === '.' ? '' : parentPath, - currentName: path.posix.basename(relativePath), - newName: req.body?.newName, - }); - - if (renamed.changed) { - // Three records follow the file: the session decides where a save lands, - // presence decides which row shows as being edited, and the key decides - // whether the people already in the document stay together. - const previousRelativePath = session.relativePath; - await editorSessions.move(sessionId, { - relativePath: renamed.relativePath, - absolutePath: renamed.absolutePath, - }); - onlyofficeActivity.rename({ - from: renamed.previousAbsolutePath, - to: renamed.absolutePath, - }); - await documentKeys.renameDocumentKey({ - from: previousRelativePath, - to: renamed.relativePath, - }); - } - - res.json({ path: renamed.relativePath, name: renamed.name }); - }) -); - -const versionHistory = () => require('../services/versions'); - -const versionKeyFor = (versionId) => `version-${versionId}`; - -const editorUserOf = (req) => - req.user && req.user.id - ? { id: String(req.user.id), name: req.user.displayName || req.user.username || 'User' } - : req.guestSession - ? { id: `guest_${req.guestSession.id}`, name: 'Guest User' } - : undefined; - -/** A token for `/onlyoffice/file` that serves one content, and never writes. */ -const readOnlyFileUrl = (req, relativePath, absolutePath, ttlSeconds) => { - const backendToken = jwt.sign( - { - typ: BACKEND_TOKEN_TYPE, - absolutePath, - logicalPath: relativePath, - canWrite: false, - sessionId: null, - userId: req.user?.id ? String(req.user.id) : null, - guestSessionId: req.guestSession?.id || null, - shareToken: null, - }, - onlyoffice.secret, - { algorithm: 'HS256', expiresIn: ttlSeconds } - ); - const fileUrl = new URL('/api/onlyoffice/file', publicConfig.url); - fileUrl.searchParams.set('path', relativePath); - fileUrl.searchParams.set('backend', backendToken); - return fileUrl.toString(); -}; - -const requirePublicUrl = () => { - if (!publicConfig?.url) { - throw new ValidationError( - 'PUBLIC_URL is required on the server to build absolute URLs for ONLYOFFICE.' - ); - } -}; - -/** The key the document is open under now, as the configuration hands it out. */ -const currentKeyOf = async (req, relativePath) => { - const context = { user: req.user, guestSession: req.guestSession }; - const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); - if (!accessInfo?.canAccess || !accessInfo.canRead || !resolved) { - throw new ForbiddenError(accessInfo?.denialReason || 'Access denied.'); - } - const stat = await fsp.stat(resolved.absolutePath); - const documentType = getDocumentType(toExt(resolved.absolutePath)); - const key = await resolveKeyForOpen({ - absolutePath: resolved.absolutePath, - relativePath, - stat, - documentType, - }); - return { key, absolutePath: resolved.absolutePath }; -}; - -/** - * A version opened on its own, to be read: a viewer, with nothing to save. - * - * This is what lets an office document's history be looked at at all — the - * panel could only offer a text file until now, because nothing could put an - * earlier .docx in front of anybody. - */ -const versionViewConfig = async (req, relativePath, versionId, uiTheme) => { - requirePublicUrl(); - const context = { user: req.user, guestSession: req.guestSession }; - const located = await versionHistory().locateVersion(context, relativePath, versionId, { - download: false, - }); - const ext = toExt(located.name); - const documentType = getDocumentType(ext); - if (!documentType) { - throw new ValidationError(`ONLYOFFICE has no editor for .${ext} files.`); - } - const mayCopy = located.target.rights.download; - const config = { - documentType, - type: 'desktop', - document: { - fileType: ext, - key: versionKeyFor(located.version.id), - title: located.name, - url: readOnlyFileUrl(req, relativePath, located.absolutePath, BACKEND_TOKEN_TTL_SECONDS), - permissions: { - edit: false, - comment: false, - review: false, - // Printing or downloading a version is taking a copy of it. - download: mayCopy, - print: mayCopy, - }, - }, - // No callback: nothing is saved from a version, and the one the document - // has would release the key everyone editing it now shares. - editorConfig: { - mode: 'view', - customization: { - anonymous: { request: false }, - ...(uiTheme ? { uiTheme } : {}), - }, - lang: onlyoffice.lang || 'en', - user: editorUserOf(req), - }, - }; - config.token = jwt.sign(config, onlyoffice.secret, { algorithm: 'HS256' }); - return { - documentServerUrl: onlyoffice.serverUrl, - config, - editorSessionId: null, - autoSaveIntervalMs: 0, - version: { id: located.version.id, modifiedAt: located.version.modifiedAt }, - }; -}; - -/** - * The document's history, as the editor's own history panel reads it. - * - * The editor numbers versions from the oldest and expects the current state to - * be the last of them; the history this application keeps comes newest first - * and does not count the current state as a version, so the two are reconciled - * here rather than in the editor. - */ -router.post( - '/onlyoffice/history', - asyncHandler(async (req, res) => { - const relativePath = normalizeRelativePath(req.body?.path || ''); - if (!relativePath) throw new ValidationError('A valid file path is required.'); - const context = { user: req.user, guestSession: req.guestSession }; - const listed = await versionHistory().listVersions(context, relativePath); - const { key } = await currentKeyOf(req, relativePath); + } + : undefined, + }, + }; - const history = [...listed.versions].reverse().map((version, index) => ({ - version: index + 1, - versionId: version.id, - key: versionKeyFor(version.id), - created: version.modifiedAt, - user: { id: version.author?.id || '', name: version.author?.label || '' }, - label: version.label, - available: version.available !== false, - })); - history.push({ - version: history.length + 1, - versionId: null, - key, - created: listed.file.modifiedAt, - user: { id: listed.file.author?.id || '', name: listed.file.author?.label || '' }, - label: null, - available: true, - }); + // Sign config for Document Server when ONLYOFFICE JWT is enabled + if (onlyoffice.secret) { + try { + // Important: sign the final config as-is; do not mutate URLs afterwards + const token = jwt.sign(config, onlyoffice.secret, { + algorithm: 'HS256', + }); + config.token = token; + } catch (e) { + logger.warn({ err: e }, 'ONLYOFFICE: failed to sign config token'); + } + } - res.set('Cache-Control', 'no-store'); - res.json({ currentVersion: history.length, history, canRestore: listed.rights.restore }); + res.json({ + documentServerUrl: onlyoffice.serverUrl, + config, + forceSaveSessionId, + // The name this answer has always carried. `forceSaveSessionId` says what + // it is for; `editorSessionId` says what it is, and anything already + // reading it — a client, a script, a test — keeps working. + editorSessionId: forceSaveSessionId, + autoSaveIntervalMs: canEdit ? onlyoffice.autoSaveIntervalMs : 0, + }); }) ); -/** Where one entry of that history is fetched from. */ router.post( - '/onlyoffice/history-data', + '/onlyoffice/session-heartbeat', asyncHandler(async (req, res) => { - requirePublicUrl(); const relativePath = normalizeRelativePath(req.body?.path || ''); - if (!relativePath) throw new ValidationError('A valid file path is required.'); - const version = Number(req.body?.version); - if (!Number.isInteger(version) || version < 1) { - throw new ValidationError('A version number is required.'); + const sessionId = req.body?.sessionId || ''; + if (!relativePath || typeof sessionId !== 'string' || !sessionId) { + throw new ValidationError('A valid ONLYOFFICE editing session is required.'); } const context = { user: req.user, guestSession: req.guestSession }; - const versionId = typeof req.body?.versionId === 'string' ? req.body.versionId : ''; - - let key; - let absolutePath; - let name; - if (versionId) { - const located = await versionHistory().locateVersion(context, relativePath, versionId, { - download: false, - }); - key = versionKeyFor(located.version.id); - absolutePath = located.absolutePath; - name = located.name; - } else { - // The current state, from inside the history: the same rights decide. - await versionHistory().listVersions(context, relativePath); - ({ key, absolutePath } = await currentKeyOf(req, relativePath)); - name = path.basename(absolutePath); - } - - const payload = { - fileType: toExt(name), - key, - url: readOnlyFileUrl(req, relativePath, absolutePath, STORAGE_FILE_TOKEN_TTL_SECONDS), - version, - }; - payload.token = jwt.sign(payload, onlyoffice.secret, { algorithm: 'HS256' }); - res.set('Cache-Control', 'no-store'); - res.json(payload); + const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); + if (!accessInfo?.canAccess || !accessInfo.canRead) throw new ForbiddenError('Access denied.'); + await getEditorSession(req, sessionId, relativePath); + // The client starts beating once ONLYOFFICE reports the document ready, so + // the first beat is what declares the document open. + const active = onlyofficeActivity.touch({ + absolutePath: resolved.absolutePath, + sessionId, + user: describeSessionUser(req), + }); + res.json({ active }); }) ); /** - * A file from the storage, handed to the editor. + * Rename the open document from the editor's title bar. * - * The editor inserts an image, merges a spreadsheet or compares against - * another document by asking its host for one — it never reaches the storage - * itself. So the host answers with a URL the Document Server may fetch once, - * signed, read-only and short-lived, for a file this caller may already read. - * Nothing is ever written back through it. + * The rename itself is the ordinary one, with the ordinary permission checks. + * What is specific here is keeping the editing session pointed at the file + * afterwards: the Document Server holds a token naming the path as it was when + * the editor opened, and returns it unchanged with every save. Left alone, the + * next autosave would recreate the old name beside the new one. */ router.post( - '/onlyoffice/storage-file', + '/onlyoffice/rename', asyncHandler(async (req, res) => { - requirePublicUrl(); - if (!onlyoffice.secret) { - throw new ValidationError('ONLYOFFICE_SECRET is required to hand files to the editor.'); - } - const relativePath = normalizeRelativePath(req.body?.path || ''); - if (!relativePath) { - throw new ValidationError('A valid file path is required.'); + const sessionId = req.body?.sessionId || ''; + if (!relativePath || typeof sessionId !== 'string' || !sessionId) { + throw new ValidationError('A valid ONLYOFFICE editing session is required.'); } - // What the editor is asking for, carried back untouched in the answer it - // recognises. Bounded because it is the caller's own string. - const command = typeof req.body?.c === 'string' ? req.body.c.slice(0, 64) : undefined; - const context = { user: req.user, guestSession: req.guestSession }; - const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); - if (!accessInfo?.canAccess || !accessInfo.canRead) { - throw new ForbiddenError(accessInfo?.denialReason || 'Access denied.'); - } + // Only the session that opened this document may rename it from inside the + // editor, and only sessions allowed to write ever get one. + const session = await getEditorSession(req, sessionId, relativePath); - const stat = await fsp.stat(resolved.absolutePath); - if (stat.isDirectory()) { - throw new ValidationError('A file is required.'); - } + const parentPath = path.posix.dirname(relativePath); + const renamed = await renameEntry({ + context: { user: req.user, guestSession: req.guestSession }, + parentRelative: parentPath === '.' ? '' : parentPath, + currentName: path.posix.basename(relativePath), + newName: req.body?.newName, + }); - const payload = { - ...(command === undefined ? {} : { c: command }), - fileType: toExt(path.basename(resolved.absolutePath)), - url: readOnlyFileUrl( - req, - relativePath, - resolved.absolutePath, - STORAGE_FILE_TOKEN_TTL_SECONDS - ), - }; - payload.token = jwt.sign(payload, onlyoffice.secret, { algorithm: 'HS256' }); + if (renamed.changed) { + // Three records follow the file: the session decides where a save lands, + // presence decides which row shows as being edited, and the key decides + // whether the people already in the document stay together. + const previousRelativePath = session.relativePath; + await editorSessions.move(sessionId, { + relativePath: renamed.relativePath, + absolutePath: renamed.absolutePath, + }); + onlyofficeActivity.rename({ + from: renamed.previousAbsolutePath, + to: renamed.absolutePath, + }); + await documentKeys.renameDocumentKey({ + from: previousRelativePath, + to: renamed.relativePath, + }); + } - res.set('Cache-Control', 'no-store'); - res.json(payload); + res.json({ path: renamed.relativePath, name: renamed.name }); }) ); @@ -953,10 +922,10 @@ router.post( * ONLYOFFICE does not write anything itself: it converts the document, then * hands the integration a URL to fetch the result from. Without a route to * receive it the menu entry is hidden, which left Download as the only way out - * — through the browser, into the person's downloads, not their volume. + * — through the browser, into the user's downloads, not their volume. * * Deliberately not tied to an editing session: saving a copy is not a change to - * the original, so a reader may do it too. What it does require is the right to + * the original, so viewers may do it too. What it does require is the right to * read the document it came from and to write into the folder it lands in, * exactly as an upload would. */ @@ -969,7 +938,7 @@ router.post( } // The URL comes from the editor, so it is only ever fetched when it points - // at the configured Document Server — the same rule as the save callback. + // at the configured Document Server — same rule as the save callback. const downloadUrl = ensureAllowedDownloadUrl(req.body?.url); let desiredName; @@ -985,7 +954,7 @@ router.post( const context = { user: req.user, guestSession: req.guestSession }; - // Reading the source is what entitles somebody to save a copy of it. + // Reading the source is what entitles someone to save a copy of it. const { accessInfo: sourceAccess } = await resolvePathWithAccess(context, relativePath); if (!sourceAccess?.canAccess || !sourceAccess.canRead) { throw new ForbiddenError(sourceAccess?.denialReason || 'Access denied.'); @@ -1001,6 +970,9 @@ router.post( throw new ForbiddenError(folderAccess?.denialReason || 'Access denied.'); } + // A copy never overwrites: a name held, before the download or during it, + // gets the same "(1)" treatment as everywhere else in the app, and the + // answer gives the name actually taken. await ensureDir(folder.absolutePath); const { name, path: absolute } = await downloadDocumentInto( downloadUrl, @@ -1009,6 +981,8 @@ router.post( ); const written = await fsp.stat(absolute); + await folderSizeHooks.onFileWritten(absolute, written.size); + const savedPath = combineRelativePath(targetFolder, name); logger.info( { path: savedPath, size: written.size }, @@ -1019,17 +993,92 @@ router.post( }) ); -// How long a token naming one file for the editor is good for: long enough to -// fetch it, short enough that the link is not worth keeping. +/** + * Hand the editor a file from the user's storage. + * + * ONLYOFFICE inserts images, merges spreadsheets and compares documents by + * being given a URL it fetches itself, so the choice made in NextExplorer has + * to become something the Document Server can download. It gets the same + * `/onlyoffice/file` route the open document uses, with a short-lived token + * naming this file and nothing else. + * + * `c` comes from the event and travels back untouched: the editor uses it to + * match the answer to the request it made, and it is part of what the token + * signs — a token covering a different object is rejected. + * + * Read access to the chosen file is the whole permission check. Writing is + * never involved: the file is copied into the open document, not modified. + */ const STORAGE_FILE_TOKEN_TTL_SECONDS = 15 * 60; +router.post( + '/onlyoffice/storage-file', + asyncHandler(async (req, res) => { + if (!publicConfig?.url) { + throw new ValidationError( + 'PUBLIC_URL is required on the server to build absolute URLs for ONLYOFFICE.' + ); + } + if (!onlyoffice.secret) { + throw new ValidationError('ONLYOFFICE_SECRET is required to hand files to the editor.'); + } + + const relativePath = normalizeRelativePath(req.body?.path || ''); + if (!relativePath) { + throw new ValidationError('A valid file path is required.'); + } + const command = typeof req.body?.c === 'string' ? req.body.c.slice(0, 64) : undefined; + + const context = { user: req.user, guestSession: req.guestSession }; + const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); + if (!accessInfo?.canAccess || !accessInfo.canRead) { + throw new ForbiddenError(accessInfo?.denialReason || 'Access denied.'); + } + + const stat = await fsp.stat(resolved.absolutePath); + if (stat.isDirectory()) { + throw new ValidationError('A file is required.'); + } + + const backendToken = jwt.sign( + { + typ: BACKEND_TOKEN_TYPE, + absolutePath: resolved.absolutePath, + logicalPath: resolved.relativePath, + space: resolved.space, + // Nothing is ever written back through this token. + canWrite: false, + sessionId: null, + userId: req.user?.id ? String(req.user.id) : null, + guestSessionId: req.guestSession?.id || null, + shareToken: resolved.shareInfo?.shareToken || null, + }, + onlyoffice.secret, + { algorithm: 'HS256', expiresIn: STORAGE_FILE_TOKEN_TTL_SECONDS } + ); + + const fileUrl = new URL('/api/onlyoffice/file', publicConfig.url); + fileUrl.searchParams.set('path', relativePath); + fileUrl.searchParams.set('backend', backendToken); + + const payload = { + ...(command === undefined ? {} : { c: command }), + fileType: toExtension(resolved.absolutePath), + url: fileUrl.toString(), + }; + payload.token = jwt.sign(payload, onlyoffice.secret, { algorithm: 'HS256' }); + + res.json(payload); + }) +); + /** * Who can be mentioned in a comment. * * ONLYOFFICE asks for the whole list and filters it in the editor as the - * comment is typed, so this answers with names and addresses rather than to a - * query. Only signed-in people get it: a visitor editing through a share link - * has no business being handed the user directory. + * comment is typed, so this returns names and addresses rather than answering a + * query. Only signed-in users get it: a guest editing through a share link has + * no business being handed the user directory. */ router.get( '/onlyoffice/users', @@ -1037,22 +1086,19 @@ router.get( if (!req.user?.id) { throw new ForbiddenError('Mentions require a signed-in user.'); } - // Required here rather than at the top: the search service reaches into the - // database, which the route file does not otherwise touch. - // eslint-disable-next-line global-require const { listUsersForMentions } = require('../services/userSearchService'); res.json({ users: await listUsersForMentions() }); }) ); /** - * A comment mentioning somebody was posted. + * A comment mentioning someone was posted. * * ONLYOFFICE has already written the comment into the document; this is the * separate "tell them about it" step, which it leaves entirely to the - * integration. There is no notification channel to deliver it on, so the - * mention is recorded and nothing is sent — said plainly, rather than leaving - * the editor waiting on a handler that silently does nothing. + * integration. NextExplorer has no notification channel to deliver it on, so + * the mention is recorded and nothing is sent — deliberately, rather than + * leaving the editor waiting on a handler that silently does nothing. */ router.post( '/onlyoffice/notify', @@ -1074,7 +1120,11 @@ router.post( ? req.body.emails.filter((email) => typeof email === 'string').slice(0, 50) : []; logger.info( - { path: relativePath, by: String(req.user.id), recipients: emails.length }, + { + path: relativePath, + by: String(req.user.id), + recipients: emails.length, + }, 'ONLYOFFICE comment mention recorded, no notification channel configured' ); @@ -1083,15 +1133,74 @@ router.post( ); /** - * How an open folder learns that somebody joined or left a document. + * The editor is gone: flush what it holds, then let the session go. + * + * Two things in one request, and the order between them is the point. Closing + * the panel could afford two calls because it can wait between them — it asks + * for a force-save, waits until this server has accepted it, and only then + * ends the session. A browser tab being closed can wait for nothing: whatever + * is sent at that moment is sent in one breath, and a second request that + * depended on the first would arrive in whichever order the network felt like. + * + * So the order moved here, and both ways of closing use this one route. What + * the server ends up holding is the same whether the panel was closed or the + * tab was — which is the property the tests pin, because two ways of closing + * that leave two different states is how a document ends up reported as open + * by nobody. * - * Held open for up to twenty-five seconds on purpose, rather than asked for - * every second: presence changes rarely, and a poll that costs nothing while - * nothing happens is what makes it affordable to show at all. + * The advisory activity is deliberately *not* released: closing the frame does + * not mean Document Server has let the document go. It stays until its + * status-2/4 callback arrives, or until the short session TTL is reached. */ +router.post( + '/onlyoffice/session-end', + asyncHandler(async (req, res) => { + const relativePath = normalizeRelativePath(req.body?.path || ''); + const sessionId = req.body?.sessionId || ''; + if (!relativePath || typeof sessionId !== 'string' || !sessionId) { + throw new ValidationError('A valid ONLYOFFICE editing session is required.'); + } + const context = { user: req.user, guestSession: req.guestSession }; + const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); + if (!accessInfo?.canAccess || !accessInfo.canRead) throw new ForbiddenError('Access denied.'); + const session = await getEditorSession(req, sessionId, relativePath); + + // Somebody who was reading has nothing to flush, and an integration with + // no Document Server has nowhere to ask. Neither is a reason to refuse the + // close — the session still has to end, or the document goes on being + // reported as open by somebody who has left. + // + // The secret is deliberately not checked: there is always one, derived + // when none was configured (config/index.js), so a condition on it would + // read as a guard and never be false. + let requestId = null; + if (onlyoffice.serverUrl && accessInfo.canWrite) { + requestId = + pendingForceSavesBySession.get(sessionId) || + enqueueForceSave({ + sessionId, + key: session.key, + relativePath, + reason: 'close', + }); + } + + // The presence goes with the session. Without this the document stayed + // marked as open by somebody who had closed it, in every listing, until + // the entry aged out. + onlyofficeActivity.close({ absolutePath: resolved.absolutePath, sessionId }); + await editorSessions.remove(sessionId); + res.json({ ended: true, flushed: Boolean(requestId), requestId }); + }) +); + router.get( '/onlyoffice/activity-version', asyncHandler(async (req, res) => { + // Held open for up to twenty-five seconds on purpose: this is how an open + // editor learns that someone else joined without polling every second. + markLongPoll(req); + const parsedSince = Number(req.query?.since); const since = Number.isInteger(parsedSince) ? parsedSince : null; const controller = new AbortController(); @@ -1109,21 +1218,17 @@ router.get( }) ); -/** - * Write what the editor is holding, now. - * - * Answers as soon as the command is queued: the Document Server writes the - * document through the ordinary callback, asynchronously. Two requests for the - * same session are coalesced — a close arriving while an automatic save is - * still assembling queues one final command behind it rather than a second one - * beside it. - */ +// Queue a save before the embedded editor is closed. The request returns right +// away: Document Server sends the actual status-6 callback asynchronously. router.post( '/onlyoffice/force-save', asyncHandler(async (req, res) => { if (!onlyoffice.serverUrl) { throw new ValidationError('ONLYOFFICE_URL is not configured on the server.'); } + if (!onlyoffice.secret) { + throw new ValidationError('ONLYOFFICE_SECRET is required to force-save documents.'); + } const relativeRaw = req.body?.path || ''; const sessionId = req.body?.sessionId || ''; @@ -1154,6 +1259,10 @@ router.post( if (pending) { const followUp = reason === 'close' && pending.reason === 'auto'; if (followUp) pending.followUpReason = 'close'; + logger.debug( + { path: relativePath, requestId: existingRequestId, reason, followUp }, + 'ONLYOFFICE force-save coalesced with pending request' + ); return res.status(202).json({ queued: true, requestId: existingRequestId, @@ -1189,33 +1298,20 @@ router.get( } try { jwt.verify(token, onlyoffice.secret, { algorithms: ['HS256'] }); - } catch (e) { + } catch (_) { throw new UnauthorizedError('Invalid token.'); } } // Optionally, resolve from backend token (supports personal paths) - let backendCtx = null; - const backendToken = typeof req.query?.backend === 'string' ? req.query.backend : null; - if (backendToken && onlyoffice.secret) { - try { - const payload = jwt.verify(backendToken, onlyoffice.secret, { - algorithms: ['HS256'], - }); - if (payload && typeof payload === 'object' && payload.absolutePath) { - backendCtx = payload; - } - } catch (e) { - logger.warn({ err: e }, 'ONLYOFFICE backend token verification failed'); - } - } + const backendCtx = readBackendToken(req); // Determine absolute path: // - Prefer signed backend context when available (works for personal/share paths) // - Fallback to resolving logical path without user for volume-only paths - let abs = null; - if (backendCtx && typeof backendCtx.absolutePath === 'string' && backendCtx.absolutePath) { - abs = backendCtx.absolutePath; + let abs; + if (backendCtx) { + abs = await resolveSaveTarget(backendCtx); } else { const context = { user: req.user, guestSession: req.guestSession }; const { accessInfo, resolved } = await resolvePathWithAccess(context, relativePath); @@ -1231,8 +1327,8 @@ router.get( if (stat.isDirectory()) { throw new ValidationError('Cannot fetch a directory.'); } - const ext = toExt(abs); - const mime = resolveMime(ext); + const ext = toExtension(abs); + const mime = resolveMimeType(ext); res.writeHead(200, { 'Content-Type': mime, 'Content-Length': stat.size, @@ -1266,7 +1362,7 @@ router.post( } try { jwt.verify(token, onlyoffice.secret, { algorithms: ['HS256'] }); - } catch (e) { + } catch (_) { throw new UnauthorizedError('Invalid token.'); } } @@ -1280,8 +1376,8 @@ router.post( const activityPath = backendCtx?.absolutePath; // Status 1 reports the users currently connected to the document. It is - // presence only: this never becomes a filesystem lock, and it expires if - // the Document Server stops sending callbacks. + // presence only: this never becomes a filesystem lock and expires if + // Document Server stops sending callbacks. if (status === 1 && activityPath) { onlyofficeActivity.updateDocumentServerUsers({ absolutePath: activityPath, @@ -1296,16 +1392,23 @@ router.post( // The session knows where the document is now; the token only knows // where it was when the editor opened, which a rename since then would // have made wrong. - const closing = backendCtx?.sessionId + const session = backendCtx?.sessionId ? await editorSessions.get(backendCtx.sessionId) : null; await documentKeys.releaseDocumentKey( - closing?.relativePath || backendCtx?.logicalPath || relativePath + session?.relativePath || backendCtx?.logicalPath || relativePath ); } + + if (status === 7) { + finishForceSave(forceSaveRequestId, { saved: false, failed: true }); + logger.warn({ path: relativePath, forceSaveRequestId }, 'ONLYOFFICE force-save failed'); + return res.json({ error: 0 }); + } // See ONLYOFFICE callback statuses: 2 - Save, 6 - Force Save if ((status === 2 || status === 6) && body.url) { - // Only the Document Server we handed the document to may be fetched from. + // The Document Server hands us a URL to pull the saved document from. + // Only the configured server may be contacted. const downloadUrl = ensureAllowedDownloadUrl(body.url); let abs = null; if (backendCtx) { @@ -1315,8 +1418,6 @@ router.post( throw new ForbiddenError('This editing session is read-only.'); } await assertShareStillValid(backendCtx); - // Where the document is now, not where it was called when the editor - // opened it. abs = await resolveSaveTarget(backendCtx); } else { const context = { user: req.user, guestSession: req.guestSession }; @@ -1329,53 +1430,64 @@ router.post( abs = resolved.absolutePath; } await ensureDir(path.dirname(abs)); - - // Keep the permissions the document already had; one the editor is - // creating starts private. - let mode = 0o600; + let previousSize = 0; + let previousMode = 0o600; + let existed = false; try { const previous = await fsp.stat(abs); - if (previous.isFile()) mode = previous.mode & 0o777; + existed = previous.isFile(); + previousSize = existed ? previous.size : 0; + previousMode = previous.mode & 0o777; } catch { - // A document that is not there yet has nothing to keep. + // A newly-created document is valid. } - - // A save on purpose — the editor's own Save, or the last one made once - // everybody has left the document — is a state worth keeping. The - // automatic saves in between are not, beyond the checkpoint the - // versions take of a session that runs long. + // A save on purpose — the editor's own Save, closing the document, or + // the last save once everyone has left — is a state worth keeping; the + // automatic ones in between are not, unless the session runs long. const explicit = status === 2 || Number(body.forcesavetype) === 1 || pendingForceSaves.get(forceSaveRequestId)?.reason === 'close'; - + // Keep the permissions the document already had; a new one starts + // private. await versions.saveFile( abs, - (temporaryPath) => fetchDocumentInto(downloadUrl, temporaryPath, mode), + (temporaryPath) => + fetchDocumentInto(downloadUrl, temporaryPath, existed ? previousMode : 0o600), { purpose: 'onlyoffice', author: authorFromCallback(body, backendCtx), source: 'onlyoffice', session: { - // Everybody editing together shares the document key: it is the - // session, and its saves belong to it rather than each standing - // as a state of its own. + // Everyone editing together shares the document key: it is the session. key: typeof body.key === 'string' && body.key ? body.key : null, startedAt: Number.isFinite(backendCtx?.iat) ? backendCtx.iat * 1000 : null, }, explicit, } ); + const updated = await fsp.stat(abs); + if (existed) { + await folderSizeHooks.onFileReplaced(abs, previousSize, updated.size); + } else { + await folderSizeHooks.onFileWritten(abs, updated.size); + } finishForceSave(forceSaveRequestId, { saved: status === 6 }); - logger.debug({ path: relativePath, status }, 'ONLYOFFICE file updated'); + logger.debug( + { + path: relativePath, + status, + forceSaveType: body.forcesavetype, + forceSaveRequestId, + size: updated.size, + }, + 'ONLYOFFICE file updated' + ); // MUST return {error:0} according to ONLYOFFICE spec return res.json({ error: 0 }); } - // Status 7 is the Document Server saying the force-save failed; status 6 - // without a URL is the same shape. Either way whoever is waiting on that - // request must be told, or the close hangs until it times out. - if (status === 6 || status === 7) { + if (status === 6) { finishForceSave(forceSaveRequestId, { saved: false, failed: true }); } diff --git a/backend/src/routes/permissions.js b/backend/src/routes/permissions.js index 0d005f38a..7d358795a 100644 --- a/backend/src/routes/permissions.js +++ b/backend/src/routes/permissions.js @@ -1,12 +1,12 @@ const express = require('express'); const fs = require('fs/promises'); -const { exec } = require('child_process'); +const { execFile } = require('child_process'); const { promisify } = require('util'); const { normalizeRelativePath } = require('../utils/pathUtils'); const { ACTIONS, authorizeAndResolve } = require('../services/authorizationService'); -const logger = require('../utils/logger'); const { ensureAdmin } = require('../middleware/ensureAdmin'); +const logger = require('../utils/logger'); const asyncHandler = require('../utils/asyncHandler'); const { ValidationError, @@ -16,7 +16,21 @@ const { } = require('../errors/AppError'); const router = express.Router(); -const execAsync = promisify(exec); +// execFile never spawns a shell: user-supplied owner/group names and file paths +// stay plain arguments instead of being interpolated into a command string. +const execFileAsync = promisify(execFile); + +// POSIX-portable account name, or a numeric id. Rejecting anything else keeps +// `chown` from receiving a value it would read as an option. +const ACCOUNT_NAME_PATTERN = /^[a-zA-Z0-9_][a-zA-Z0-9._-]*$/; + +const ensureValidAccountName = (value, label) => { + if (value === undefined || value === null || value === '') return ''; + if (typeof value !== 'string' || !ACCOUNT_NAME_PATTERN.test(value)) { + throw new ValidationError(`Invalid ${label} name.`); + } + return value; +}; /** * Get file permissions, owner, and group information @@ -53,7 +67,7 @@ router.get( if (process.platform !== 'win32') { try { // Get owner name from uid - const { stdout: ownerOut } = await execAsync(`id -nu ${stats.uid}`); + const { stdout: ownerOut } = await execFileAsync('id', ['-nu', String(stats.uid)]); owner = ownerOut.trim(); } catch (e) { logger.debug({ err: e }, 'Failed to get owner name'); @@ -61,7 +75,7 @@ router.get( try { // Get group name from gid - const { stdout: groupOut } = await execAsync(`id -gn ${stats.gid}`); + const { stdout: groupOut } = await execFileAsync('id', ['-gn', String(stats.gid)]); group = groupOut.trim(); } catch (e) { logger.debug({ err: e }, 'Failed to get group name'); @@ -91,10 +105,8 @@ router.get( */ router.post( '/permissions/chmod', - // Changing modes and ownership on a shared volume is an administration - // task: a plain write permission on a path is not consent to re-permission - // its tree. `ensureAdmin` says as much in its own comment, and these are the - // two routes it was written for. + // Changing modes on a shared volume is an administration task: a plain + // write permission on a path is not consent to re-permission its tree. ensureAdmin, asyncHandler(async (req, res) => { const { path: rawPath, mode, recursive } = req.body; @@ -107,13 +119,16 @@ router.post( throw new ValidationError('Mode must be a 3-digit octal string (e.g., "755").'); } + // Guests never reach this point: they have no req.user. Carrying a guest + // session on top of a real account does not make the account a guest. if (!req.user || !req.user.id) { throw new UnauthorizedError('Authentication required'); } - // Guests never reach this point: they have no req.user. Carrying a guest - // session on top of a real account does not make the account a guest. const relativePath = normalizeRelativePath(rawPath); + if (relativePath.startsWith('share/')) { + throw new ForbiddenError('Permissions cannot be changed through a share.'); + } const context = { user: req.user, guestSession: req.guestSession }; const { allowed, accessInfo, resolved } = await authorizeAndResolve( context, @@ -125,11 +140,14 @@ router.post( } try { - // Check if path exists - await fs.stat(resolved.absolutePath); - - // Use chmod via Node.js built-in - const modeInt = parseInt(mode, 8); + const before = await fs.stat(resolved.absolutePath); + + // The three digits set read, write and execute. The setuid, setgid and + // sticky bits are not among them, and `chmod` writes the whole mode it is + // given: unticking one box on a shared setgid folder, or on /tmp-like + // sticky one, would silently take those bits away. They are kept as the + // item already had them. + const modeInt = parseInt(mode, 8) | (before.mode & 0o7000); await fs.chmod(resolved.absolutePath, modeInt); // If recursive and directory, apply to all children @@ -139,10 +157,12 @@ router.post( // Use chmod -R for recursive on Unix systems if (process.platform !== 'win32') { try { - await execAsync(`chmod -R ${mode} "${resolved.absolutePath}"`); + // No `--` separator here: BSD chmod (macOS) does not accept it. + // The path is always absolute, so it can never look like a flag. + await execFileAsync('chmod', ['-R', mode, resolved.absolutePath]); } catch (e) { logger.error({ err: e }, 'Failed to apply recursive chmod'); - throw new Error('Failed to apply permissions recursively.'); + throw new Error('Failed to apply permissions recursively.', { cause: e }); } } else { // On Windows, we'd need to recursively walk the directory @@ -176,10 +196,6 @@ router.post( */ router.post( '/permissions/chown', - // Changing modes and ownership on a shared volume is an administration - // task: a plain write permission on a path is not consent to re-permission - // its tree. `ensureAdmin` says as much in its own comment, and these are the - // two routes it was written for. ensureAdmin, asyncHandler(async (req, res) => { const { path: rawPath, owner, group } = req.body; @@ -192,12 +208,18 @@ router.post( throw new ValidationError('Either owner or group must be specified.'); } + const safeOwner = ensureValidAccountName(owner, 'owner'); + const safeGroup = ensureValidAccountName(group, 'group'); + + // Same as above: only a real account gets here. if (!req.user || !req.user.id) { throw new UnauthorizedError('Authentication required'); } - // As above: a guest session beside an account is not a guest. const relativePath = normalizeRelativePath(rawPath); + if (relativePath.startsWith('share/')) { + throw new ForbiddenError('Ownership cannot be changed through a share.'); + } const context = { user: req.user, guestSession: req.guestSession }; const { allowed, accessInfo, resolved } = await authorizeAndResolve( context, @@ -212,21 +234,27 @@ router.post( // Check if path exists await fs.stat(resolved.absolutePath); - // chown requires shell execution as Node.js doesn't have built-in owner/group change - // This requires elevated privileges on most systems + // Node has no built-in owner/group change by name, so the system tools do + // it. Arguments are passed as an array, never through a shell, and the + // account names were validated above so they cannot look like flags + // (the path is absolute, so it cannot either). if (process.platform !== 'win32') { - let chownCmd = ''; - - if (owner && group) { - chownCmd = `chown "${owner}:${group}" "${resolved.absolutePath}"`; - } else if (owner) { - chownCmd = `chown "${owner}" "${resolved.absolutePath}"`; - } else if (group) { - chownCmd = `chgrp "${group}" "${resolved.absolutePath}"`; + let command = ''; + let args = []; + + if (safeOwner && safeGroup) { + command = 'chown'; + args = [`${safeOwner}:${safeGroup}`, resolved.absolutePath]; + } else if (safeOwner) { + command = 'chown'; + args = [safeOwner, resolved.absolutePath]; + } else { + command = 'chgrp'; + args = [safeGroup, resolved.absolutePath]; } try { - await execAsync(chownCmd); + await execFileAsync(command, args); logger.info({ path: relativePath, owner, group }, 'Ownership changed'); } catch (e) { logger.error({ err: e }, 'Failed to change ownership'); @@ -236,7 +264,7 @@ router.post( 'Permission denied. Changing ownership typically requires root/admin privileges.' ); } - throw new Error('Failed to change ownership: ' + e.message); + throw new Error('Failed to change ownership: ' + e.message, { cause: e }); } } else { throw new ValidationError('Changing ownership is not supported on Windows.'); diff --git a/backend/src/routes/settings.js b/backend/src/routes/settings.js index 7063be9c0..c3b32010f 100644 --- a/backend/src/routes/settings.js +++ b/backend/src/routes/settings.js @@ -3,54 +3,27 @@ const { getPublicSettings, getSettingsForUser, setUserSetting, - USER_SETTING_KEYS, - setSystemSetting, - getSettings, + setUserFolderSort, + setUserFolderView, + checkSystemSection, + mergeSystemSection, + replaceBranding, + WRITABLE_USER_SETTINGS, } = require('../services/settingsService'); +const { forgetReplacedLogo, replaceLogo } = require('../services/brandingLogo'); const activityLog = require('../services/activityLog'); +const asyncHandler = require('../utils/asyncHandler'); const { ensureAdmin } = require('../middleware/ensureAdmin'); -const { checkRulePath } = require('../services/accessControlService'); +const multer = require('multer'); const { ValidationError } = require('../errors/AppError'); +const { describeBytes, explainMultipartRefusals } = require('../middleware/multipartRefusals'); const folderSizeManager = require('../services/folderSizeManager'); const searchIndexManager = require('../services/searchIndexManager'); -const asyncHandler = require('../utils/asyncHandler'); -const multer = require('multer'); const featureSwitches = require('../services/featureSwitches'); -const { explainMultipartRefusals, describeBytes } = require('../middleware/multipartRefusals'); -const { replaceLogo, forgetReplacedLogo } = require('../services/brandingLogo'); - -/** - * A number somebody chose. - * - * Every numeric setting here has a floor above zero, and every one of them is - * a field on a form: emptied, it arrives as 0. Stored, the sanitizer lifts it - * to the floor — so clearing the trash retention used to leave a trash that - * keeps one day and sweeps everything older within the hour, and clearing the - * share of a volume left one percent. Nothing arriving means nothing chosen, - * and what is stored stays. - * - * One reading for all of them rather than a condition per field, so a section - * added later cannot be the one that forgot. - */ -const chosenNumber = (value) => Number.isFinite(value) && value > 0; +const { checkRulePath } = require('../services/accessControlService'); const router = express.Router(); -// Middleware to check if user is admin -const keepValid = (section, fields) => { - const update = {}; - for (const [name, isAcceptable] of Object.entries(fields)) { - if (isAcceptable(section[name])) update[name] = section[name]; - } - return update; -}; - -const isBoolean = (value) => typeof value === 'boolean'; - -// An application name of spaces is no name: the header and the sign-in page showed -// nothing where it belonged. -const isName = (value) => typeof value === 'string' && value.trim() !== ''; - const LOGO_MAX_BYTES = 2 * 1024 * 1024; // Configure multer for logo uploads @@ -72,6 +45,7 @@ const upload = multer({ const acceptLogo = explainMultipartRefusals(upload.single('logo'), { LIMIT_FILE_SIZE: `A logo can be at most ${describeBytes(LOGO_MAX_BYTES)}.`, }); + /** * GET /api/branding * Returns public branding settings (no auth required) @@ -169,277 +143,314 @@ router.post( * - Users can update their own user settings (user.*) * - Admins can update system settings (thumbnails, access, branding) */ -router.patch( - '/settings', - asyncHandler(async (req, res) => { - const payload = req.body || {}; - const user = req.user; - const isAdmin = user && Array.isArray(user.roles) && user.roles.includes('admin'); - const updated = {}; - - // User settings (all authenticated users can update) - if (payload.user && typeof payload.user === 'object' && user && user.id) { - const userUpdates = {}; - for (const [key, value] of Object.entries(payload.user)) { - // Which keys are preferences is the settings service's to say: this - // route used to keep a second list of its own, and a preference added - // to one and not the other was silently dropped here. - if (USER_SETTING_KEYS.has(key)) { - userUpdates[key] = await setUserSetting(user.id, key, value); - } - } - if (Object.keys(userUpdates).length > 0) { - updated.user = userUpdates; - } +/** + * Keep the fields of a section that arrived in a shape worth storing. + * + * A field nobody sent is not a field set to nothing, and a size that is not a + * number is a size nobody chose: both are left out, so the stored value stays + * what it was rather than becoming something the caller never asked for. + */ +const keepValid = (section, fields) => { + const update = {}; + for (const [name, isAcceptable] of Object.entries(fields)) { + if (isAcceptable(section[name])) update[name] = section[name]; + } + return update; +}; + +const isBoolean = (value) => typeof value === 'boolean'; +const isText = (value) => typeof value === 'string'; + +// A size or a count of nothing, or of less than nothing, is what an emptied or +// mistyped field sends, not a value anyone chose. The service would bring it up +// to its lowest bound — a chunk size of 0 became 1 MiB — which replaced what +// was stored with something nobody asked for. A positive value outside the +// bounds is still brought within them there. +const isPositiveNumber = (value) => Number.isFinite(value) && value > 0; + +// An application name of spaces is no name: the header and the sign-in page +// showed nothing where it belonged. +const isName = (value) => typeof value === 'string' && value.trim() !== ''; + +/** + * Merge an update over what is stored, and give back the whole section. + * + * The merge is the service's, which reads the stored section and writes it + * back without yielding in between. Merging over the settings read at the + * start of the request, as this did, left two awaits between the read and the + * write: two saves of one section at once both started from the same stored + * value, and the second wrote over the first's field while telling the person + * who set it that it was saved. Branding already had its own reason for a + * read and a write in one step; every section has this one. + * + * @returns {Promise} null when there was nothing to change, so a + * caller can tell "no valid field" from "field set to its current value". + */ +const mergeSection = async (category, key, update) => { + if (Object.keys(update).length === 0) return null; + return mergeSystemSection(category, key, update); +}; + +/** A person's own preferences, which they may change whatever their role. */ +const applyUserPreferences = async (user, section) => { + const updates = {}; + + for (const [key, value] of Object.entries(section)) { + if (key === 'folderSort') { + const folderSorts = await setUserFolderSort(user.id, value?.path, value?.sort); + if (folderSorts) updates.folderSorts = folderSorts; + } else if (key === 'folderView') { + const folderViews = await setUserFolderView(user.id, value?.path, value?.view); + if (folderViews) updates.folderViews = folderViews; + } else if (WRITABLE_USER_SETTINGS.has(key)) { + updates[key] = await setUserSetting(user.id, key, value); } + } - // System settings (admin only) - if (isAdmin) { - const systemUpdates = {}; - - // Thumbnails settings - if (payload.thumbnails && typeof payload.thumbnails === 'object') { - const thumbnailsUpdate = {}; - if (payload.thumbnails.enabled != null) { - thumbnailsUpdate.enabled = Boolean(payload.thumbnails.enabled); - } - if (chosenNumber(payload.thumbnails.size)) { - thumbnailsUpdate.size = payload.thumbnails.size; - } - if (chosenNumber(payload.thumbnails.quality)) { - thumbnailsUpdate.quality = payload.thumbnails.quality; - } - if (chosenNumber(payload.thumbnails.concurrency)) { - thumbnailsUpdate.concurrency = payload.thumbnails.concurrency; - } - if (Object.keys(thumbnailsUpdate).length > 0) { - const current = await getSettings(); - await setSystemSetting('system', 'thumbnails', { - ...current.thumbnails, - ...thumbnailsUpdate, - }); - systemUpdates.thumbnails = { ...current.thumbnails, ...thumbnailsUpdate }; - } - } + return Object.keys(updates).length > 0 ? updates : null; +}; - // Access control rules, and whether they hold administrators. The two are - // saved apart on the settings page, so each is merged over what is stored - // rather than replacing the section: saving the rules used to drop the - // setting above them, and saving the setting used to drop the rules. - if (payload.access && typeof payload.access === 'object') { - const accessUpdate = {}; - if (Array.isArray(payload.access.rules)) accessUpdate.rules = payload.access.rules; - if (typeof payload.access.applyToAdmins === 'boolean') { - accessUpdate.applyToAdmins = payload.access.applyToAdmins; - } - if (Object.keys(accessUpdate).length > 0) { - const current = await getSettings(); - const merged = await setSystemSetting('system', 'access', { - ...current.access, - ...accessUpdate, - }); - systemUpdates.access = merged; - } - } +/** + * A section checked in one step and written in another. + * + * Both halves exist because one save carries several sections: a refusal in + * the third must not leave the first two stored. `check` answers what is to be + * written, or null when the section sends nothing this route stores, and it is + * where a refusal comes from. `write` stores it, and cannot refuse. + */ +const merging = (key, fields) => ({ + check: (section) => keepValid(section, fields), + write: (update) => mergeSection('system', key, update), +}); - // Trash settings: only the fields that arrived in a usable shape are - // merged over what is stored; setSystemSetting sanitizes and clamps them. - if (payload.trash && typeof payload.trash === 'object') { - const trashUpdate = {}; - if (typeof payload.trash.enabled === 'boolean') { - trashUpdate.enabled = payload.trash.enabled; - } - if (chosenNumber(payload.trash.retentionDays)) { - trashUpdate.retentionDays = payload.trash.retentionDays; - } - if (chosenNumber(payload.trash.maxPercent)) { - trashUpdate.maxPercent = payload.trash.maxPercent; - } - if (payload.trash.maxBytes === null || chosenNumber(payload.trash.maxBytes)) { - trashUpdate.maxBytes = payload.trash.maxBytes; - } - if (Object.keys(trashUpdate).length > 0) { - const current = await getSettings(); - const merged = await setSystemSetting('system', 'trash', { - ...current.trash, - ...trashUpdate, - }); - systemUpdates.trash = merged; - } - } +const thumbnailsSection = merging('thumbnails', { + // Anything but a boolean used to be read as "on": "false" switched + // thumbnails on for everybody. + enabled: isBoolean, + size: isPositiveNumber, + quality: isPositiveNumber, + concurrency: isPositiveNumber, +}); - // Upload settings: whether uploads go out in chunks, and how big one is. - if (payload.uploads && typeof payload.uploads === 'object') { - const uploadsUpdate = {}; - if (typeof payload.uploads.chunkedEnabled === 'boolean') { - uploadsUpdate.chunkedEnabled = payload.uploads.chunkedEnabled; - } - if (chosenNumber(payload.uploads.chunkSizeBytes)) { - uploadsUpdate.chunkSizeBytes = payload.uploads.chunkSizeBytes; - } - if (Object.keys(uploadsUpdate).length > 0) { - const current = await getSettings(); - const merged = await setSystemSetting('system', 'uploads', { - ...current.uploads, - ...uploadsUpdate, - }); - systemUpdates.uploads = merged; - } - } +const uploadsSection = merging('uploads', { + chunkedEnabled: isBoolean, + chunkedAutoFallback: isBoolean, + chunkSizeBytes: isPositiveNumber, +}); - // File-version settings: only the fields that arrived usable are merged; - // setSystemSetting sanitizes and keeps them consistent. - if (payload.versions && typeof payload.versions === 'object') { - const versionsUpdate = {}; - if (typeof payload.versions.enabled === 'boolean') { - versionsUpdate.enabled = payload.versions.enabled; - } - for (const key of [ - 'keepAllHours', - 'hourlyDays', - 'dailyDays', - 'maxPerFile', - 'sessionCheckpointMinutes', - ]) { - if (chosenNumber(payload.versions[key])) versionsUpdate[key] = payload.versions[key]; - } - if (Object.keys(versionsUpdate).length > 0) { - const current = await getSettings(); - const merged = await setSystemSetting('system', 'versions', { - ...current.versions, - ...versionsUpdate, - }); - systemUpdates.versions = merged; - } - } +// The trash's size cap is the one field where nothing is a value: null removes +// the cap. Zero is not that — it is what an emptied field sends, and the +// service read it as "no cap given" and put the default back. +const isPositiveNumberOrNull = (value) => value === null || isPositiveNumber(value); + +// A retention of no days, or of fewer than none, is what an emptied or +// mistyped field sends. The service brought each up to its lowest bound — a +// retention of 0 became one day, of -5 became one day — in place of the ninety +// the administrator had. The settings page refuses them with the same bounds; +// this is what an API client used to see instead. +const trashSection = merging('trash', { + enabled: isBoolean, + retentionDays: isPositiveNumber, + maxPercent: isPositiveNumber, + maxBytes: isPositiveNumberOrNull, +}); - // Activity log settings: the switch, and how long a line is kept. - if (payload.activity && typeof payload.activity === 'object') { - const activityUpdate = {}; - if (typeof payload.activity.enabled === 'boolean') { - activityUpdate.enabled = payload.activity.enabled; - } - if (chosenNumber(payload.activity.retentionDays)) { - activityUpdate.retentionDays = payload.activity.retentionDays; - } - if (Object.keys(activityUpdate).length > 0) { - const current = await getSettings(); - const merged = await setSystemSetting('system', 'activity', { - ...current.activity, - ...activityUpdate, - }); - systemUpdates.activity = merged; - } - } +const versionsSection = merging('versions', { + enabled: isBoolean, + keepAllHours: isPositiveNumber, + hourlyDays: isPositiveNumber, + dailyDays: isPositiveNumber, + maxPerFile: isPositiveNumber, + sessionCheckpointMinutes: isPositiveNumber, +}); - // The folders each background worker leaves alone. The list is stored - // and handed to the worker, which answers with the list it is really - // applying — the stored one plus whatever the environment set, which an - // administrator cannot take away from here. - for (const [key, manager] of [ - ['folderSize', folderSizeManager], - ['searchIndex', searchIndexManager], - ]) { - const section = payload[key]; - if (!section || typeof section !== 'object') continue; - - // Whether the worker runs at all, which was decided by SEARCH_INDEX and - // FOLDER_SIZE_MODE alone: turning either on meant editing a file on the host - // and restarting, while every other setting beside them was a click (#9). - // - // The environment stays the floor. A variable somebody set decides, and a - // switch sent for it is refused in words naming the variable, rather than - // accepted and quietly ignored — "false" is a decision too, so an - // installation that turned the index off in its file has not left it to - // whoever next opens the page. - const field = key === 'searchIndex' ? 'enabled' : 'mode'; - if (Object.prototype.hasOwnProperty.call(section, field)) { - const variable = key === 'searchIndex' ? 'SEARCH_INDEX' : 'FOLDER_SIZE_MODE'; - if (featureSwitches.snapshot()[key].lockedBy) { - throw new ValidationError( - `${variable} is set in the environment, so this is decided there and not here.` - ); - } - const requested = - key === 'searchIndex' - ? typeof section.enabled === 'boolean' - ? section.enabled - : undefined - : featureSwitches.FOLDER_SIZE_MODES.includes(section.mode) - ? section.mode - : undefined; - if (requested === undefined) { - throw new ValidationError(`${field} is not a value ${key} takes.`); - } - const current = await getSettings(); - systemUpdates[key] = await setSystemSetting('system', key, { - ...current[key], - [field]: requested, - }); - if (key === 'searchIndex') await featureSwitches.setSearchIndex(requested); - else await featureSwitches.setFolderSizeMode(requested); - } - - if (!Array.isArray(section.excludedPaths)) continue; - const current = await getSettings(); - const merged = await setSystemSetting('system', key, { - ...current[key], - excludedPaths: section.excludedPaths, - }); - await manager.setAdminExclusions(merged.excludedPaths); - systemUpdates[key] = merged; - } +const activitySection = merging('activity', { + enabled: isBoolean, + retentionDays: isPositiveNumber, +}); - // Branding settings - let previousLogoUrl; - if (payload.branding && typeof payload.branding === 'object') { - const brandingUpdate = {}; - if (typeof payload.branding.appName === 'string') { - brandingUpdate.appName = payload.branding.appName; - } - if (typeof payload.branding.appLogoUrl === 'string') { - brandingUpdate.appLogoUrl = payload.branding.appLogoUrl; - } - if (typeof payload.branding.showPoweredBy === 'boolean') { - brandingUpdate.showPoweredBy = payload.branding.showPoweredBy; - } - if (Object.keys(brandingUpdate).length > 0) { - const current = await getSettings(); - previousLogoUrl = current.branding?.appLogoUrl ?? null; - await setSystemSetting('branding', 'branding', { - ...current.branding, - ...brandingUpdate, - }); - systemUpdates.branding = { ...current.branding, ...brandingUpdate }; - } - } +/** + * Branding is read and written in one step rather than merged over the + * settings read at the start of the request, because a logo it replaces is + * then removed: reset to the default, or pointed elsewhere, the old file would + * otherwise stay behind with nothing to serve or remove it. + */ +const brandingSection = { + check: (section) => { + const update = keepValid(section, { + appName: isName, + appLogoUrl: isText, + showPoweredBy: isBoolean, + }); + return Object.keys(update).length > 0 ? update : null; + }, + write: async (update) => { + const { previous, current } = await replaceBranding(update); + await forgetReplacedLogo(previous.appLogoUrl, current.appLogoUrl); + }, +}; - if (Object.keys(systemUpdates).length > 0) { - Object.assign(updated, systemUpdates); - // Which settings, not what they were set to: values belong in the - // settings, and some of them are somebody's business alone. - await activityLog.record({ - action: 'admin.settings', - user, - detail: { sections: Object.keys(systemUpdates) }, - req, - }); - } +/** + * Access rules replace the list rather than merging into it, and they are the + * one section that refuses what it was sent: a rule with no folder, or a + * permission that is not one of the three, is answered rather than dropped. + * Which is why it is checked here, before any other section is written — a + * list sent as something that is not a list is still dropped, as it always + * was, because then there is nothing to store. + * + * The switch that holds administrators to every rule is saved from a control of + * its own, so each half is taken only when it was sent and merged over what is + * stored: saving the rules must not switch it off, and switching it must not + * empty the rules. + */ +const accessSection = { + check: (section) => { + const update = {}; + if (Array.isArray(section.rules)) { + update.rules = checkSystemSection('access', { rules: section.rules }).rules; + } + if (section.applyToAdmins !== undefined) { + update.applyToAdmins = checkSystemSection('access', { + applyToAdmins: section.applyToAdmins, + }).applyToAdmins; + } + return Object.keys(update).length > 0 ? update : null; + }, + write: (update) => mergeSystemSection('system', 'access', update), +}; - // The logo that was replaced is forgotten, and only once nothing points at it - // any more. Removing the files under a fixed name meant a logo could not be - // changed back, and a branding change that failed halfway took the logo in use - // with it. - if (previousLogoUrl !== undefined) { - const settingsNow = await getSettings(); - await forgetReplacedLogo(previousLogoUrl, settingsNow.branding?.appLogoUrl); +/** + * A background worker: the folders it leaves alone, and whether it runs. + * + * The list is stored and handed to the worker, which answers with the list it + * is actually applying — the stored one plus whatever the environment set, + * which an administrator cannot remove from here. + * + * The switch follows the same rule. When the environment set it, it is refused + * rather than quietly stored: an administrator who flips a switch and sees + * nothing happen deserves to be told which variable is in the way, and a page + * that shows the switch locked will not send it in the first place. + * + * @param {string} key the settings section + * @param {object} manager the worker, for its exclusions + * @param {string} field `enabled` or `mode` + * @param {(value: *) => *} valid the value to store, or undefined to refuse it + * @param {(value: *) => Promise} apply switch the worker to it + * @param {string} variable the environment variable that would lock it + */ +const background = ({ key, manager, field, valid, apply, variable }) => ({ + check: (section) => { + const update = {}; + if (Array.isArray(section.excludedPaths)) update.excludedPaths = section.excludedPaths; + + if (Object.prototype.hasOwnProperty.call(section, field)) { + if (featureSwitches.snapshot()[key].lockedBy) { + throw new ValidationError( + `${variable} is set in the environment, so this is decided there and not here.` + ); } - } else if (payload.thumbnails || payload.access || payload.branding) { - // Non-admin trying to update system settings + const value = valid(section[field]); + if (value === undefined) throw new ValidationError(`${field} is not a value ${key} takes.`); + update[field] = value; + } + + return Object.keys(update).length ? update : null; + }, + write: async (update) => { + const saved = await mergeSection('system', key, update); + if (!saved) return false; + if (update.excludedPaths) await manager.setAdminExclusions(saved.excludedPaths); + if (Object.prototype.hasOwnProperty.call(update, field)) await apply(saved[field]); + return true; + }, +}); + +const searchIndexSection = background({ + key: 'searchIndex', + manager: searchIndexManager, + field: 'enabled', + valid: (value) => (typeof value === 'boolean' ? value : undefined), + apply: (value) => featureSwitches.setSearchIndex(value), + variable: 'SEARCH_INDEX', +}); + +const folderSizeSection = background({ + key: 'folderSize', + manager: folderSizeManager, + field: 'mode', + valid: (value) => (featureSwitches.FOLDER_SIZE_MODES.includes(value) ? value : undefined), + apply: (value) => featureSwitches.setFolderSizeMode(value), + variable: 'FOLDER_SIZE_MODE', +}); + +/** Every section only an administrator may write, and what writes it. */ +const SYSTEM_SECTIONS = { + thumbnails: thumbnailsSection, + access: accessSection, + uploads: uploadsSection, + trash: trashSection, + versions: versionsSection, + activity: activitySection, + branding: brandingSection, + folderSize: folderSizeSection, + searchIndex: searchIndexSection, +}; + +router.patch( + '/settings', + asyncHandler(async (req, res) => { + const payload = req.body || {}; + const user = req.user; + const isAdmin = user && Array.isArray(user.roles) && user.roles.includes('admin'); + + // Asked before anything is written, not after. The user section used to be + // applied first and the refusal raised afterwards, so a payload carrying + // both a preference and a system setting answered 403 with the preference + // already saved — a request reported as refused that had changed something. + const wantsSystemSettings = Object.keys(SYSTEM_SECTIONS).some((name) => payload[name]); + if (!isAdmin && wantsSystemSettings) { return res.status(403).json({ error: 'Admin access required for system settings.' }); } - // Return updated settings + // Every section is checked before any of them is written. A save carrying + // a valid section and a refused one used to store the first and then answer + // 400: a request reported as refused that had changed something, and left + // the page showing settings the server had only half taken. + const toWrite = []; + if (isAdmin) { + for (const [name, section] of Object.entries(SYSTEM_SECTIONS)) { + const sent = payload[name]; + if (!sent || typeof sent !== 'object') continue; + const update = section.check(sent); + if (update !== null) toWrite.push([name, section, update]); + } + } + + if (payload.user && typeof payload.user === 'object' && user?.id) { + await applyUserPreferences(user, payload.user); + } + + // What was stored, not what was sent: a section whose every field was + // refused writes nothing, and a log line saying otherwise would send + // somebody looking for a change that never happened. + const stored = []; + for (const [name, section, update] of toWrite) { + if (await section.write(update)) stored.push(name); + } + if (stored.length) { + // Which settings, not what they were set to: values belong in the + // settings, and some of them are somebody's business alone. + await activityLog.record({ + action: 'admin.settings', + user, + detail: { sections: stored }, + req, + }); + } + + // Read back rather than assembled from what was written: the stored value + // is sanitised on its way out, so what the caller applies to its own state + // is what a later request would read. const finalSettings = await getSettingsForUser(user); res.json(finalSettings); }) diff --git a/backend/src/routes/shares.js b/backend/src/routes/shares.js index 69a44b5b7..350ebeaf1 100644 --- a/backend/src/routes/shares.js +++ b/backend/src/routes/shares.js @@ -1,11 +1,11 @@ const express = require('express'); -const { parseByteRange } = require('../utils/httpRange'); const fs = require('fs/promises'); const fss = require('fs'); const path = require('path'); -const archiver = require('archiver'); +const { ZipArchive } = require('archiver'); const rateLimit = require('express-rate-limit'); const asyncHandler = require('../utils/asyncHandler'); +const { sendTextFile } = require('../utils/textFileResponse'); const { ValidationError, UnauthorizedError, @@ -32,23 +32,25 @@ const { const { createGuestSession } = require('../services/guestSessionService'); const { normalizeRelativePath, parsePathSpace } = require('../utils/pathUtils'); const { pathExists } = require('../utils/fsUtils'); +const { parseByteRange } = require('../utils/httpRange'); const { resolvePathWithAccess, sharePasswordApplies } = require('../services/accessManager'); const { extensions, mimeTypes } = require('../config/index'); +const env = require('../config/env'); const { getSettings, getUserSettings } = require('../services/settingsService'); const { listDirectoryItems } = require('../services/directoryListingService'); const { encodeContentDisposition } = require('./files/utils'); const { collectArchiveEntries, appendEntries } = require('../services/archiveTree'); const logger = require('../utils/logger'); - -const activityLog = require('../services/activityLog'); -const versions = require('../services/versions/operations'); -const { sendTextFile } = require('../utils/textFileResponse'); -const { clientAddress } = require('../utils/clientAddress'); const { readTextFileHead, encodeText, MAX_EDITOR_FILE_SIZE, } = require('../services/textEditorService'); +const versions = require('../services/versions/operations'); +const activityLog = require('../services/activityLog'); +const versionsService = require('../services/versions'); +const { rightsFrom: versionRights } = versionsService; +const { clientAddress } = require('../utils/clientAddress'); const router = express.Router(); @@ -82,9 +84,27 @@ const guestSessionCookieOptions = (req) => ({ maxAge: 24 * 60 * 60 * 1000, // 24 hours sameSite: 'lax', secure: req.secure === true, - path: '/api', // Ensure cookie is sent for all /api/* requests + // Root path, not /api: thumbnails are served from /static, and an + // cannot carry the X-Guest-Session header the API client uses. Scoping the + // cookie to /api left share visitors with broken thumbnails. + path: '/', }); +/** + * Set the guest session cookie, clearing the /api-scoped one first. + * + * An earlier build scoped this cookie to /api. Browsers keep both when the + * path differs, and RFC 6265 sends the longer path first, so on every /api + * request cookie-parser would read the stale value and shadow the session we + * just created — a dead end the visitor could not fix by retyping the + * password. Deleting it here reaches exactly the people affected, since every + * share visitor goes through one of these three endpoints. + */ +const setGuestSessionCookie = (req, res, sessionId) => { + res.clearCookie('guestSession', { path: '/api' }); + res.cookie('guestSession', sessionId, guestSessionCookieOptions(req)); +}; + const buildPublicBaseUrl = (req) => { const { public: publicConfig } = require('../config/index'); return publicConfig.origin || `${req.protocol}://${req.get('host')}`; @@ -212,7 +232,7 @@ const buildDirectFilePath = (shareToken, innerPath = '', mode = 'auto') => { const query = normalizedMode === 'auto' ? '' : `?mode=${encodeURIComponent(normalizedMode)}`; const pathPart = encodedInnerPath ? `/api/share/${encodedToken}/file/${encodedInnerPath}` - : `/api/share/${encodedToken}/file`; + : `/api/share/${encodedToken}`; return `${pathPart}${query}`; }; @@ -299,7 +319,7 @@ const streamResolvedDirectoryZip = async ({ res.setHeader('X-Content-Type-Options', 'nosniff'); res.setHeader('X-Robots-Tag', 'noindex'); - const archive = archiver('zip', { zlib: { level: 1 } }); + const archive = new ZipArchive({ zlib: { level: 1 } }); archive.on('error', (archiveError) => { logger.error({ err: archiveError }, 'Direct share archive creation failed'); if (!res.headersSent) { @@ -310,8 +330,8 @@ const streamResolvedDirectoryZip = async ({ }); archive.pipe(res); - // What the share lets its visitor see, not everything below its folder: a - // personal root and the paths an access rule hides stay out. + // What the share lets its visitor see, not everything below its folder: the + // trash zone, a personal root and the paths an access rule hides stay out. const stats = await fs.stat(absolutePath); const { entries } = await collectArchiveEntries(context, [ { @@ -338,6 +358,13 @@ router.post( const { sourcePath, accessMode = 'readonly', + allowDelete = true, + allowCreateFolder = true, + allowCreateFile = true, + allowUpload = true, + allowDownload = true, + versionsVisible, + versionsDownload, sharingType = 'anyone', password, userIds, @@ -409,6 +436,13 @@ router.post( sourcePath: sourcePathForDb, isDirectory, accessMode, + allowDelete, + allowCreateFolder, + allowCreateFile, + allowUpload, + allowDownload, + versionsVisible, + versionsDownload, sharingType, password, userIds: sharingType === 'users' ? userIds : [], @@ -416,6 +450,11 @@ router.post( label, }); + // Generate share URL using PUBLIC_URL if configured, otherwise use request host + const baseUrl = buildPublicBaseUrl(req); + const shareUrl = `${baseUrl}/share/${share.shareToken}`; + const directFileUrl = `${baseUrl}${buildDirectFilePath(share.shareToken)}`; + await activityLog.record({ action: 'share.create', user: req.user, @@ -424,11 +463,6 @@ router.post( req, }); - // Generate share URL using PUBLIC_URL if configured, otherwise use request host - const baseUrl = buildPublicBaseUrl(req); - const shareUrl = `${baseUrl}/share/${share.shareToken}`; - const directFileUrl = `${baseUrl}${buildDirectFilePath(share.shareToken)}`; - res.status(201).json({ ...share, shareUrl, @@ -475,6 +509,16 @@ router.get( router.get( '/:id', asyncHandler(async (req, res) => { + // This router is mounted under both /api/shares (management) and + // /api/share (public links). The exact public token URL must be handled + // here before the management endpoint can interpret the token as a share + // ID. Named public routes have an additional path segment and do not match + // this route. + if (req.baseUrl === '/api/share') { + req.params.token = req.params.id; + return handleDirectFileRequest(req, res); + } + if (!req.user || !req.user.id) { throw new UnauthorizedError('Authentication required'); } @@ -527,6 +571,18 @@ router.put( updates.accessMode = req.body.accessMode; } + for (const key of [ + 'allowDelete', + 'allowCreateFolder', + 'allowCreateFile', + 'allowUpload', + 'allowDownload', + 'versionsVisible', + 'versionsDownload', + ]) { + if (key in req.body) updates[key] = req.body[key]; + } + if ('sharingType' in req.body) { updates.sharingType = req.body.sharingType; } @@ -607,16 +663,14 @@ router.get( throw new NotFoundError('Share not found'); } - // Return limited public info + // Return limited public info. requiresPassword mirrors the backend rule + // rather than hasPassword alone, so the router does not send the owner to + // a password prompt the API would have let them skip. res.json({ shareToken: share.shareToken, label: share.label, isDirectory: share.isDirectory, hasPassword: share.hasPassword, - // Whether this caller has to type it: its owner does not, and neither - // does anybody when authentication is off. The interface asks for the - // password on this, rather than on hasPassword, so the owner is not sent - // to a prompt the server would have let them skip. requiresPassword: sharePasswordApplies(share, req.user), sharingType: share.sharingType, expiresAt: share.expiresAt, @@ -650,22 +704,23 @@ router.post( if (share.sharingType === 'anyone') { const session = await createGuestSession({ shareId: share.id, - ipAddress: req.ip, + ipAddress: clientAddress(req), userAgent: req.get('user-agent'), }); + await trackShareAccess(share.id, { ipAddress: clientAddress(req) }); // Set guest session cookie - res.cookie('guestSession', session.id, guestSessionCookieOptions(req)); + setGuestSessionCookie(req, res, session.id); res.json({ success: true, guestSessionId: session.id, }); return; - } else { - // User-specific share without password still requires auth - throw new UnauthorizedError('Authentication required'); } + + // User-specific share without password still requires auth + throw new UnauthorizedError('Authentication required'); } // Verify password @@ -679,12 +734,13 @@ router.post( if (share.sharingType === 'anyone') { const session = await createGuestSession({ shareId: share.id, - ipAddress: req.ip, + ipAddress: clientAddress(req), userAgent: req.get('user-agent'), }); + await trackShareAccess(share.id, { ipAddress: clientAddress(req) }); // Set guest session cookie - res.cookie('guestSession', session.id, guestSessionCookieOptions(req)); + setGuestSessionCookie(req, res, session.id); res.json({ success: true, @@ -716,9 +772,6 @@ router.get( throw new ForbiddenError('Share has expired'); } - // Track access - await trackShareAccess(share.id, { ipAddress: req.ip }); - // Check if user has permission if (share.sharingType === 'users') { if (!req.user || !req.user.id) { @@ -751,12 +804,15 @@ router.get( // looking made the branch below ask for the password a second time, // for a share it had just been given. if (req.guestSession && req.guestSession.shareId === share.id) { + await trackShareAccess(share.id, { ipAddress: clientAddress(req) }); + return res.json({ share: { shareToken: share.shareToken, label: share.label, sourcePath: `share/${share.shareToken}`, accessMode: share.accessMode, + allowDownload: share.allowDownload !== false, isDirectory: share.isDirectory, }, guestSessionId: req.guestSession.id, @@ -767,12 +823,13 @@ router.get( if (!share.hasPassword) { const session = await createGuestSession({ shareId: share.id, - ipAddress: req.ip, + ipAddress: clientAddress(req), userAgent: req.get('user-agent'), }); + await trackShareAccess(share.id, { ipAddress: clientAddress(req) }); // Set guest session cookie (overwrites any existing session) - res.cookie('guestSession', session.id, guestSessionCookieOptions(req)); + setGuestSessionCookie(req, res, session.id); return res.json({ share: { @@ -780,14 +837,17 @@ router.get( label: share.label, sourcePath: `share/${share.shareToken}`, accessMode: share.accessMode, + allowDownload: share.allowDownload !== false, isDirectory: share.isDirectory, }, guestSessionId: session.id, }); - } else { - throw new UnauthorizedError('Password verification required'); } + + throw new UnauthorizedError('Password verification required'); } + + await trackShareAccess(share.id, { ipAddress: clientAddress(req) }); } // Return share access info @@ -797,6 +857,7 @@ router.get( label: share.label, sourcePath: `share/${share.shareToken}`, accessMode: share.accessMode, + allowDownload: share.allowDownload !== false, isDirectory: share.isDirectory, expiresAt: share.expiresAt, }, @@ -805,36 +866,6 @@ router.get( }) ); -/** - * A file that left through a link. - * - * The share's own counters answer "how many"; this answers "which file, when, - * and from where" — the question somebody actually asks the day a link turns - * out to have been handed around. The person on the other end has no account, - * so the actor is the link itself. - */ -const recordShareDownload = ({ share, resolved, req }) => - activityLog.record({ - action: 'share.download', - user: req.user, - actor: req.user?.username || share.label || `link ${share.shareToken?.slice(0, 8)}`, - target: resolved.relativePath || share.sourcePath, - detail: { share: share.label || null, token: share.shareToken?.slice(0, 8) || null }, - req, - }); - -/** - * The file a share link points at, and what this caller may do with it. - * - * Everything a request through a link has to get past before the file is - * touched: the link itself, who is following it, and what the location - * underneath still allows. Written once because three routes ask it — the - * direct file, and the two the editor uses — and each asks for something - * slightly different, which is what the options are. - * - * @returns the target, or null when the caller was redirected to the share's - * own door and there is nothing more for the route to do. - */ const resolveSharedFileTarget = async ( req, res, @@ -918,15 +949,32 @@ const resolveSharedFileTarget = async ( return { share, innerPath, accessInfo, resolved, stats, context }; }; +/** + * A file that left through a link. + * + * The share's own counters answer "how many"; this answers "which file, when, + * and from where" — the question somebody actually asks the day a link turns + * out to have been handed around. The person on the other end has no account, + * so the actor is the link itself. + */ +const recordShareDownload = ({ share, resolved, req }) => + activityLog.record({ + action: 'share.download', + user: req.user, + actor: req.user?.username || share.label || `link ${share.shareToken?.slice(0, 8)}`, + target: resolved.relativePath || share.sourcePath, + detail: { share: share.label || null, token: share.shareToken?.slice(0, 8) || null }, + req, + }); + const handleDirectFileRequest = async (req, res) => { - const mode = normalizeDirectFileMode(req.query?.mode); const target = await resolveSharedFileTarget(req, res, { requireDownload: true }); if (!target) return; const { share, resolved, stats, context } = target; if (stats.isDirectory()) { - // A folder leaving as a zip is a download like any other. - await trackShareDownload(share.id, { ipAddress: req.ip }); + // Directories are always delivered as a ZIP attachment. + await trackShareDownload(share.id, { ipAddress: clientAddress(req) }); await recordShareDownload({ share, resolved, req }); await streamResolvedDirectoryZip({ absolutePath: resolved.absolutePath, @@ -942,27 +990,37 @@ const handleDirectFileRequest = async (req, res) => { return; } - await trackShareDownload(share.id, { ipAddress: req.ip }); - await recordShareDownload({ share, resolved, req }); - await streamResolvedFile({ absolutePath: resolved.absolutePath, stats, mode, req, res }); + // Count the hit the way the client receives the file: inline previews are + // accesses, attachment deliveries (explicit download mode or formats the + // browser cannot display) are downloads — same split as POST /api/download. + const mode = normalizeDirectFileMode(req.query?.mode); + const { disposition } = getDirectFilePresentation(path.basename(resolved.absolutePath), mode); + if (disposition === 'attachment') { + await trackShareDownload(share.id, { ipAddress: clientAddress(req) }); + await recordShareDownload({ share, resolved, req }); + } else { + await trackShareAccess(share.id, { ipAddress: clientAddress(req) }); + } + await streamResolvedFile({ + absolutePath: resolved.absolutePath, + stats, + mode, + req, + res, + }); }; /** * GET /api/share/:token/file/* - Open a shared file directly. * - * This keeps the same share rules as the Web UI but streams the target file - * itself, letting the browser preview supported formats or download others. + * The exact /api/share/:token alias is handled by the mounted router's + * management route above. It intentionally calls this same handler so token, + * expiry, password, guest-session and permission checks cannot drift apart. + * Keep /file routes for existing external integrations. */ router.get('/:token/file', asyncHandler(handleDirectFileRequest)); router.get('/:token/file/{*splat}', asyncHandler(handleDirectFileRequest)); -/** - * GET /api/share/:token/editor/* — read a shared text file, to edit it. - * - * The same rules as the direct file, and the same answer the editor gets - * inside the application: the text, what it is called, and what this visitor - * may do with it. - */ const handleSharedEditorRequest = async (req, res) => { const target = await resolveSharedFileTarget(req, res, { allowSharedFileName: true }); if (!target) return; @@ -985,12 +1043,12 @@ const handleSharedEditorRequest = async (req, res) => { }); }; +/** + * GET /api/share/:token/editor/* - Read a shared text file. + */ router.get('/:token/editor', asyncHandler(handleSharedEditorRequest)); router.get('/:token/editor/{*splat}', asyncHandler(handleSharedEditorRequest)); -/** - * PUT /api/share/:token/editor/* — save it back, when the link allows writing. - */ const handleSharedEditorSaveRequest = async (req, res) => { const target = await resolveSharedFileTarget(req, res, { requireWrite: true, @@ -1003,9 +1061,10 @@ const handleSharedEditorSaveRequest = async (req, res) => { if (typeof content !== 'string') { throw new ValidationError('Text editor content must be a string.'); } - // The editor's own text validation before anything is written, so a writable - // share cannot be used to modify a directory, a binary or an oversized file. - // It also says what the file is written in, so the save keeps that. + // Reuse the editor's text validation before writing so a writable share + // cannot be used to modify directories, binaries, or oversized files. It also + // says what the file is written in, so the save keeps that. From the head of + // the file: this asked for the whole of it, decoded, to read three bytes. const { encoding } = await readTextFileHead(resolved.absolutePath); const payload = encodeText(content, encoding); if (payload.length > MAX_EDITOR_FILE_SIZE) { @@ -1072,7 +1131,8 @@ router.get( // Determine thumbnail settings const settings = await getSettings(); const userSettings = req.user?.id ? await getUserSettings(req.user.id) : {}; - const thumbsEnabled = settings?.thumbnails?.enabled !== false; + const thumbsEnabled = + env.THUMBNAILS_ENABLED !== false && settings?.thumbnails?.enabled !== false; const includeHiddenFiles = userSettings?.showHiddenFiles === true; // Directory share or navigating inside a directory share @@ -1083,24 +1143,51 @@ router.get( shareCache.set(resolved.shareInfo.shareToken, resolved.shareInfo); } const userVolumeCache = new Map(); + const marks = + userSettings?.showVersionMarks === false + ? null + : await versionsService.marksForFolder(resolved.absolutePath).catch((error) => { + logger.warn( + { err: error, path: resolved.absolutePath }, + 'File versions were not counted for a shared listing' + ); + return null; + }); + const items = await listDirectoryItems({ absoluteDir: resolved.absolutePath, parentLogicalPath: resolved.relativePath, context, thumbsEnabled, - excludeDownloadArtifacts: false, includeHiddenFiles, access: settings?.access || null, shareCache, userVolumeCache, - itemExtras: () => ({ + itemExtras: ({ name, stats, access }) => ({ access: { canRead: true, canWrite: accessInfo.canWrite, canDelete: accessInfo.canDelete, + canCreateFolder: accessInfo.canCreateFolder, + canCreateFile: accessInfo.canCreateFile, canShare: false, - canDownload: true, + // Follows the share rather than being hard true: otherwise every + // row in a share with downloads withheld still shows the button, + // and clicking it is the only way to find out. + canDownload: accessInfo.canDownload, }, + // The same mark the browser shows, under the same rule: a share + // says nothing about a file's history unless its owner turned + // histories on for it. + ...(marks && stats?.isFile() && marks.get(name) && versionRights(access).see + ? { + versions: { + count: marks.get(name).versions, + bytes: marks.get(name).bytes, + newest: marks.get(name).newest, + }, + } + : null), }), }); @@ -1111,8 +1198,11 @@ router.get( canWrite: accessInfo.canWrite, canUpload: accessInfo.canUpload, canDelete: accessInfo.canDelete, + canCreateFolder: accessInfo.canCreateFolder, + canCreateFile: accessInfo.canCreateFile, canShare: false, canDownload: accessInfo.canDownload, + canSeeVersions: versionRights(accessInfo).see, }, current: { isDirectory: true, @@ -1169,8 +1259,11 @@ router.get( canWrite: accessInfo.canWrite, canUpload: false, canDelete: accessInfo.canDelete, + canCreateFolder: false, + canCreateFile: false, canShare: false, canDownload: accessInfo.canDownload, + canSeeVersions: versionRights(accessInfo).see, }, current: { isDirectory: false, diff --git a/backend/src/routes/trash.js b/backend/src/routes/trash.js index 329857867..dc9e5fe03 100644 --- a/backend/src/routes/trash.js +++ b/backend/src/routes/trash.js @@ -21,6 +21,9 @@ router.get( }) ); +/** How many items a restore was asked for, for the line that records it. */ +const countOf = (value) => (Array.isArray(value) ? value.length : value ? 1 : 0); + // POST /api/trash/restore - put items back where they were deleted from router.post( '/trash/restore', @@ -35,7 +38,7 @@ router.post( action: 'file.restore', user: req.user, target: restored[0]?.restoredName || restored[0]?.name || null, - detail: { items: restored.length }, + detail: { items: restored.length || countOf(req.body?.ids) }, req, }); res.json(outcome); @@ -55,11 +58,17 @@ router.get( router.post( '/trash/items/:id/restore', asyncHandler(async (req, res) => { - res.json( - await trash.restoreEntries(req.params.id, req.body?.paths, contextOf(req), { - shares: req.body?.shares, - }) - ); + const outcome = await trash.restoreEntries(req.params.id, req.body?.paths, contextOf(req), { + shares: req.body?.shares, + }); + await activityLog.record({ + action: 'file.restore', + user: req.user, + target: Array.isArray(req.body?.paths) ? req.body.paths[0] : null, + detail: { items: countOf(req.body?.paths), from: req.params.id }, + req, + }); + res.json(outcome); }) ); @@ -105,6 +114,16 @@ const restoreTo = (planFrom) => onEvent: writeEvent, signal: controller.signal, }); + // Recorded where the restore finished rather than where it was asked + // for: the stream can be closed half-way, and what matters is what came + // back out of the trash. + await activityLog.record({ + action: 'file.restore', + user: req.user, + target: req.body?.destination || null, + detail: { chosenDestination: true }, + req, + }); writeEvent({ type: 'done', ...result }); } catch (error) { writeEvent({ diff --git a/backend/src/routes/usage.js b/backend/src/routes/usage.js index f7c5060e8..067320eaa 100644 --- a/backend/src/routes/usage.js +++ b/backend/src/routes/usage.js @@ -1,28 +1,41 @@ const express = require('express'); -const { promisify } = require('util'); -const { exec } = require('child_process'); +const fs = require('fs/promises'); const { normalizeRelativePath } = require('../utils/pathUtils'); const { resolvePathWithAccess } = require('../services/accessManager'); const logger = require('../utils/logger'); const asyncHandler = require('../utils/asyncHandler'); -const execp = promisify(exec); const router = express.Router(); -// Fast directory size using du command -const dirSize = async (root) => { +const toSafeNumber = (value) => { + const numeric = typeof value === 'bigint' ? Number(value) : Number(value); + return Number.isFinite(numeric) && numeric > 0 ? numeric : 0; +}; + +const getFilesystemUsage = async (root) => { try { - // -sb: summarize in bytes, don't follow symlinks - // This is orders of magnitude faster than fs.stat() recursion - const { stdout } = await execp(`du -sb "${root}"`, { - maxBuffer: 1024 * 1024 * 10, // 10MB buffer for large outputs - }); + const stats = await fs.statfs(root); + const blockSize = toSafeNumber(stats.bsize); + const total = toSafeNumber(stats.blocks) * blockSize; + const free = toSafeNumber(stats.bavail) * blockSize; + const used = Math.max(0, total - free); + const percentUsed = total > 0 ? Math.min(100, Math.max(0, (used / total) * 100)) : 0; - // Output format: "12345\t/path/to/dir" - const size = parseInt(stdout.split('\t')[0], 10); - return size || 0; + return { + size: used, + used, + free, + total, + percentUsed, + }; } catch (err) { - logger.debug(err); - return 0; + logger.debug({ err, root }, 'Failed to read filesystem usage'); + return { + size: 0, + used: 0, + free: 0, + total: 0, + percentUsed: 0, + }; } }; @@ -36,31 +49,18 @@ router.get( const { accessInfo, resolved } = await resolvePathWithAccess(context, inputRel); if (!accessInfo || !accessInfo.canAccess || !accessInfo.canRead) { - // Treat denied access the same as du failing; zero usage - return res.json({ path: inputRel, size: 0, free: 0, total: 0 }); + return res.json({ + path: inputRel, + size: 0, + used: 0, + free: 0, + total: 0, + percentUsed: 0, + }); } const { absolutePath: abs, relativePath: rel } = resolved; - - // Run both commands in parallel for maximum speed - const [size, dfResult] = await Promise.all([ - dirSize(abs), - execp(`df -Pk "${abs}"`).catch(() => ({ stdout: '' })), - ]); - - let total = 0, - free = 0; - - if (dfResult.stdout) { - const line = dfResult.stdout.trim().split('\n').pop(); - const parts = line.trim().split(/\s+/); - const totalKb = parseInt(parts[1], 10) || 0; - const availKb = parseInt(parts[3], 10) || 0; - total = totalKb * 1024; - free = availKb * 1024; - } - - res.json({ path: rel, size, free, total }); + res.json({ path: rel, ...(await getFilesystemUsage(abs)) }); }) ); diff --git a/backend/src/routes/userVolumes.js b/backend/src/routes/userVolumes.js index 6437f33fe..d8890a943 100644 --- a/backend/src/routes/userVolumes.js +++ b/backend/src/routes/userVolumes.js @@ -144,7 +144,7 @@ router.get( let targetPath; try { targetPath = path.resolve(requestedPath); - } catch (err) { + } catch (_) { throw new ValidationError('Invalid path'); } @@ -152,7 +152,7 @@ router.get( let stats; try { stats = await fs.stat(targetPath); - } catch (err) { + } catch (_) { throw new NotFoundError('Path not found'); } @@ -164,7 +164,7 @@ router.get( let entries; try { entries = await fs.readdir(targetPath, { withFileTypes: true }); - } catch (err) { + } catch (_) { throw new ForbiddenError('Cannot read directory'); } diff --git a/backend/src/routes/users.js b/backend/src/routes/users.js index 9413843e7..52381f0c3 100644 --- a/backend/src/routes/users.js +++ b/backend/src/routes/users.js @@ -8,17 +8,18 @@ const { setLocalPasswordAdmin, deleteUser, getById, - countAdmins, disableTwoFactor, + twoFactorStatus, } = require('../services/users'); -const activityLog = require('../services/activityLog'); -const { ensureAdmin } = require('../middleware/ensureAdmin'); -const { clearLock } = require('../services/users/lockout'); -const { deleteAllPasskeys } = require('../services/users/passkeys'); const asyncHandler = require('../utils/asyncHandler'); -const logger = require('../utils/logger'); const { searchLocalUsers } = require('../services/userSearchService'); const { NotFoundError, ValidationError, UnauthorizedError } = require('../errors/AppError'); +const { ensureAdmin } = require('../middleware/ensureAdmin'); +const { clearLock, listActiveLocks } = require('../services/users/lockout'); +const { deleteAllPasskeys } = require('../services/users/passkeys'); +const activityLog = require('../services/activityLog'); +const logger = require('../utils/logger'); +const { startAuthenticatedSession } = require('../utils/authenticatedSession'); const router = express.Router(); @@ -52,8 +53,19 @@ router.get( '/users', ensureAdmin, asyncHandler(async (req, res) => { - const users = await listUsers(); - res.json({ users }); + const [users, locks] = await Promise.all([listUsers(), listActiveLocks()]); + // Locks are keyed on the account (see localAuth), so the list can say which + // account is locked and until when — the first question an administrator + // has when somebody cannot sign in. Whether a second factor is on is the + // second one, and the answer to "I have lost my phone". + const withFactors = await Promise.all( + users.map(async (user) => ({ + ...user, + lockedUntil: locks.get(user.id) || null, + twoFactorEnabled: (await twoFactorStatus(user.id)).enabled, + })) + ); + res.json({ users: withFactors }); }) ); @@ -139,39 +151,16 @@ router.post( asyncHandler(async (req, res) => { const { id } = req.params || {}; const { newPassword } = req.body || {}; - await setLocalPasswordAdmin({ userId: id, newPassword }); - await activityLog.record({ - action: 'admin.user', - user: req.user, - target: id, - detail: { passwordReset: true }, - req, - }); - res.status(204).end(); - }) -); - -// DELETE /api/users/:id/lock - release an account locked by failed sign-ins (admin only) -router.delete( - '/users/:id/lock', - ensureAdmin, - asyncHandler(async (req, res) => { - const { id } = req.params || {}; - const existing = await getById(id); - if (!existing) { - throw new NotFoundError('User not found.'); - } - // The count as well as the deadline. Left on the books, the failures would - // let the next typo lock the account straight back. - await clearLock(id); - logger.info({ adminId: req.user?.id, userId: id }, 'Sign-in lock released by an administrator'); - await activityLog.record({ - action: 'admin.user', - user: req.user, - target: existing.username || existing.email || id, - detail: { lockReleased: true }, - req, + // Every session of the account ends. An administrator resetting their own + // password here is changing it, and keeps the session they did it from — on + // a new id, as a change from their own settings does. + const ownSession = Boolean(req.session) && req.session.localUserId === id; + await setLocalPasswordAdmin({ + userId: id, + newPassword, + keepSessionId: ownSession ? req.sessionID : null, }); + if (ownSession) await startAuthenticatedSession(req, id); res.status(204).end(); }) ); @@ -190,20 +179,13 @@ router.delete( asyncHandler(async (req, res) => { const { id } = req.params || {}; const user = await getById(id); - if (!user) throw new NotFoundError('User not found.'); + if (!user) throw new NotFoundError('User not found'); const removed = await disableTwoFactor(id); logger.warn( { userId: id, by: req.user?.id || null, removed }, 'An administrator turned two-factor authentication off for an account' ); - await activityLog.record({ - action: 'admin.user', - user: req.user, - target: user.username || user.email || id, - detail: { twoFactorRemoved: true }, - req, - }); res.status(204).end(); }) ); @@ -223,7 +205,7 @@ router.delete( asyncHandler(async (req, res) => { const { id } = req.params || {}; const user = await getById(id); - if (!user) throw new NotFoundError('User not found.'); + if (!user) throw new NotFoundError('User not found'); const removed = await deleteAllPasskeys(id); logger.warn( @@ -241,6 +223,24 @@ router.delete( }) ); +// DELETE /api/users/:id/lock - release an account locked by failed sign-ins (admin only) +router.delete( + '/users/:id/lock', + ensureAdmin, + asyncHandler(async (req, res) => { + const { id } = req.params || {}; + const existing = await getById(id); + if (!existing) { + throw new NotFoundError('User not found.'); + } + // The count as well as the deadline. Left on the books, the failures would + // let the next typo lock the account straight back. + await clearLock(id); + logger.info({ adminId: req.user?.id, userId: id }, 'Sign-in lock released by an administrator'); + res.status(204).end(); + }) +); + // DELETE /api/users/:id - remove a user (admin only) router.delete( '/users/:id', @@ -251,18 +251,16 @@ router.delete( if (req.user?.id === id) { throw new ValidationError('You cannot delete your own account.'); } - // Prevent removing last admin explicitly const existing = await getById(id); if (!existing) { throw new NotFoundError('User not found.'); } - if (Array.isArray(existing.roles) && existing.roles.includes('admin')) { - const admins = await countAdmins(); - if (admins <= 1) { - throw new ValidationError('Cannot remove the last admin.'); - } - } + // The last administrator is protected by the service, so every caller of + // deleteUser gets the rule and not only this route. A second copy here + // read as the enforcement and was not: removing it changed nothing. await deleteUser({ userId: id }); + // Written after the deletion, and it outlives it: the row names the + // account as text, and nothing cascades this log away. await activityLog.record({ action: 'admin.user', user: req.user, diff --git a/backend/src/routes/versions.js b/backend/src/routes/versions.js index 339e3975e..65dda1aa3 100644 --- a/backend/src/routes/versions.js +++ b/backend/src/routes/versions.js @@ -1,11 +1,11 @@ const express = require('express'); const fs = require('fs'); -const activityLog = require('../services/activityLog'); const asyncHandler = require('../utils/asyncHandler'); -const logger = require('../utils/logger'); +const activityLog = require('../services/activityLog'); const { sendCompressible } = require('../utils/compressedResponse'); -const { mimeTypes } = require('../config/index'); +const logger = require('../utils/logger'); +const { resolveMimeType, toExtension } = require('../utils/fileTypes'); const versions = require('../services/versions'); const { encodeContentDisposition } = require('./files/utils'); @@ -20,13 +20,10 @@ const router = express.Router(); const contextOf = (req) => ({ user: req.user, guestSession: req.guestSession }); -const mimeTypeOf = (name = '') => - mimeTypes[String(name).split('.').pop().toLowerCase()] || 'application/octet-stream'; - router.get( '/versions', asyncHandler(async (req, res) => { - res.set('Cache-Control', 'no-store'); + res.set('Cache-Control', 'private, no-store'); res.json(await versions.listVersions(contextOf(req), req.query?.path)); }) ); @@ -36,7 +33,7 @@ router.get( asyncHandler(async (req, res) => { const located = await versions.downloadVersion(contextOf(req), req.query?.path, req.params.id); res.writeHead(200, { - 'Content-Type': mimeTypeOf(located.name), + 'Content-Type': resolveMimeType(toExtension(located.name)) || 'application/octet-stream', 'Content-Length': located.size, 'Content-Disposition': encodeContentDisposition(located.downloadName, 'attachment'), 'Cache-Control': 'private, no-store', @@ -51,10 +48,6 @@ router.get( }) ); -/** - * The text of a version, for the editor to show read only. Never cached: what a - * version holds does not change, but what this person may read does. - */ router.get( '/versions/:id/text', asyncHandler(async (req, res) => { @@ -114,10 +107,10 @@ router.post( all: req.body?.all === true, }); - // Earlier copies going while the file stays: apart from `file.purge` - // because they are apart in the interface too, and because this is the - // half that nothing else can put back. - if (Number(outcome?.deleted) > 0) { + // Recorded like every other way versions go. Leaving this one out would + // have made the log answer "nobody" to the only question it is asked + // about a history that is no longer there. + if (outcome.deleted > 0) { await activityLog.record({ action: 'versions.purge', user: req.user, diff --git a/backend/src/routes/versionsAdmin.js b/backend/src/routes/versionsAdmin.js index aeddf8029..0f25fa1cf 100644 --- a/backend/src/routes/versionsAdmin.js +++ b/backend/src/routes/versionsAdmin.js @@ -2,8 +2,8 @@ const express = require('express'); const asyncHandler = require('../utils/asyncHandler'); const { ensureAdmin } = require('../middleware/ensureAdmin'); -const versions = require('../services/versions'); const activityLog = require('../services/activityLog'); +const versions = require('../services/versions'); /** * Every file that has a history, for an administrator. @@ -60,10 +60,6 @@ router.get( * A POST with a body rather than a DELETE with a list, as the route beside it * does, so that deleting forty versions is one request and one answer per * version — a DELETE per id would report forty times and fail in the middle. - * - * This is the one route here that destroys something, and what it destroys may - * belong to somebody else — which is the case the log exists for. It is off by - * default and never fails a request. */ router.post( '/versions/admin/files/:id/delete', @@ -77,6 +73,9 @@ router.post( all: req.body?.all === true, }); + // The one route here that destroys something, and the data it destroys + // may belong to somebody else — which is the case the log exists for. It + // is off by default and never fails a request. await activityLog.record({ action: 'versions.purge', user: req.user, diff --git a/backend/src/routes/zip.js b/backend/src/routes/zip.js index 7e09a4bfc..81ef7b9b7 100644 --- a/backend/src/routes/zip.js +++ b/backend/src/routes/zip.js @@ -29,6 +29,7 @@ const { archiveBaseName, normalizeArchivePassword, } = require('../services/archiveService'); +const folderSizeHooks = require('../services/folderSizeHooks'); const { ensureArchiveWithinLimits, buildItemMetadata, @@ -208,6 +209,7 @@ router.post( ); // The archive has produced an entire new tree. Queue its index refresh, // but never hold the archive operation open on background filesystem I/O. + folderSizeHooks.onDirectoryTreeCreated(placed.path); const item = await buildItemMetadata(placed.path, parentRelativePath, placed.name); writeEvent({ type: 'done', success: true, item, items: [item] }); @@ -410,6 +412,8 @@ router.post( destinationAbsolutePath, requestedName ); + const zipStats = await fs.stat(placed.path); + await folderSizeHooks.onFileWritten(placed.path, zipStats.size); const item = await buildItemMetadata(placed.path, normalizedDestination, placed.name); writeEvent({ type: 'done', success: true, item }); diff --git a/backend/src/scripts/downloadSamples.js b/backend/src/scripts/downloadSamples.js index b6d487086..0f9116e06 100644 --- a/backend/src/scripts/downloadSamples.js +++ b/backend/src/scripts/downloadSamples.js @@ -187,11 +187,11 @@ async function main() { }); console.log(`INFO: Extracting ${zipPath} to ${samplesDir}`); - const unzip = spawnSync('unzip', ['-q', zipPath, '-d', samplesDir], { + const extract = spawnSync('7z', ['x', '-y', '-bd', `-o${samplesDir}`, zipPath], { stdio: 'inherit', }); - if (unzip.error) throw unzip.error; - if (unzip.status !== 0) throw new Error(`unzip failed with exit code ${unzip.status}`); + if (extract.error) throw extract.error; + if (extract.status !== 0) throw new Error(`7z failed with exit code ${extract.status}`); await removeMacJunk(samplesDir); await chmodReadOnlyRecursive(samplesDir); diff --git a/backend/src/server.js b/backend/src/server.js index 4aa8dc520..d8d5cd77f 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -3,25 +3,39 @@ * This file is responsible for starting the server and should NOT be imported in tests. * Tests should import the app directly from ./app.js */ + +// Size the libuv thread pool up front, before any async filesystem work runs. +// Directory listings do one fs.stat per entry through this pool; with the Node +// default of 4 threads those stats queue behind concurrent thumbnail-generation +// fs operations (realpath/stat/rename), which makes folder navigation stall +// while a large media folder is being processed. Overridable via the env var +// (also set in the Docker image); this default only applies when unset. +if (!process.env.UV_THREADPOOL_SIZE) { + process.env.UV_THREADPOOL_SIZE = '16'; +} + const { createApp } = require('./app'); const { port, http, features, address } = require('./config/index'); const logger = require('./utils/logger'); const { printStartupBanner } = require('./utils/startupBanner'); +const { cleanupExpiredShares } = require('./services/sharesService'); +const { cleanupExpiredSessions } = require('./services/guestSessionService'); +const { purgeExpiredDocumentKeys } = require('./services/onlyofficeDocumentKeyService'); const terminalService = require('./services/terminalService'); -const searchIndexManager = require('./services/searchIndexManager'); const folderSizeManager = require('./services/folderSizeManager'); +const searchIndexManager = require('./services/searchIndexManager'); +const featureSwitches = require('./services/featureSwitches'); +const capabilities = require('./services/capabilities'); +const performanceDiagnostics = require('./services/performanceDiagnostics'); +const { reportOrphanedBindings } = require('./services/orphanedBindingsService'); +const { reportLegacyCache } = require('./services/legacyCacheCheck'); +const { sweepUnreferencedLogos } = require('./services/brandingLogo'); const { sweepInterrupted } = require('./services/inFlightFiles'); const trashMaintenance = require('./services/trash/maintenance'); -const tusUploads = require('./services/tusUploadService'); -const { cleanupExpiredShares } = require('./services/sharesService'); -const { cleanupExpiredSessions } = require('./services/guestSessionService'); -const { purgeExpiredDocumentKeys } = require('./services/onlyofficeDocumentKeyService'); -const editorSessions = require('./services/onlyofficeEditorSessionService'); const { sweepActivity } = require('./services/activityLog'); -const capabilities = require('./services/capabilities'); +const databaseMaintenance = require('./services/databaseMaintenance'); +const tusUploads = require('./services/tusUploadService'); const { installProcessFailureHandlers } = require('./utils/processFailures'); -const { sweepUnreferencedLogos } = require('./services/brandingLogo'); -const featureSwitches = require('./services/featureSwitches'); let server = null; @@ -62,31 +76,32 @@ const startServer = async () => { logger.warn('Terminal disabled at runtime'); } - // Deliberately not awaited: a server does not wait for its index to be - // ready, it answers from the live search until it is. - // Whether each worker runs: the environment's answer when somebody set the - // variable, and Settings' otherwise — read before either of them starts, so one - // switched on from the page comes back on after a restart. + // Whether the two background workers run is the environment's to say when it + // said it, and Settings' otherwise — read before either of them starts, so + // one switched on from the page comes back on after a restart. await featureSwitches.load(); + // Start the folder size indexer worker (no-op unless its mode is not off). + // It runs off the Express event loop and keeps the folder_size_index fresh. folderSizeManager.start(); searchIndexManager.start(); // Which optional tools are here and which are not, said once. Not awaited: a // server does not wait on `--version` to answer its first request. capabilities.report(); + performanceDiagnostics.start(); // Finishes what a crash interrupted before anything else touches a zone, // then keeps each zone within its retention and budget. trashMaintenance.start(); // Chunked uploads abandoned, or finished and never moved into place, leave // the upload cache once past TUS_INCOMPLETE_UPLOAD_TTL_MS. tusUploads.startCacheSweep(); + // Hands the database's free space back to the filesystem once there is + // enough of it to matter, so that /config and its backups do not keep it. + databaseMaintenance.start(); - // Rows that expire and were never swept. The ONLYOFFICE key of a document - // whose browser was closed is one: only a terminal callback released a key, - // so a crash or a restart left the row for good, one for every document ever - // opened. The same sweep takes the two that were already here and had no - // caller at all — `cleanupExpiredShares` and `cleanupExpiredSessions` — so an - // expired share no longer sits on disk indefinitely. + // Expired shares and guest sessions were never purged: the services had a + // cleanup function each, and nothing ever called them, so both tables grew + // forever and an expired share stayed on disk indefinitely. const EXPIRY_SWEEP_INTERVAL_MS = 60 * 60 * 1000; const sweepExpiredRecords = async () => { try { @@ -94,9 +109,8 @@ const startServer = async () => { cleanupExpiredShares(), cleanupExpiredSessions(), purgeExpiredDocumentKeys(), - editorSessions.purgeExpired(), - // Whether or not the log is on: switching it off should let the disk go - // back rather than freeze yesterday's rows for ever. + // Whether or not the log is on: switching it off should let the disk + // go back rather than freeze yesterday's rows for ever. sweepActivity(), ]); if (shares || sessions || documentKeys || activity) { @@ -112,23 +126,33 @@ const startServer = async () => { const expirySweep = setInterval(sweepExpiredRecords, EXPIRY_SWEEP_INTERVAL_MS); // Never keep the process alive just for the sweep. expirySweep.unref?.(); - void sweepExpiredRecords(); + sweepExpiredRecords(); + + // Say what points at a volume that is not there. Removing nothing is the + // whole point: an unmounted volume and a deleted one look identical from + // here, and only a person can tell them apart. + reportOrphanedBindings(); + // And what releases before 2.0.3 left in the cache directory: an old app.db + // nothing reads, or the links 1.1.8 left beside it. + reportLegacyCache(); // A logo left behind by a stop in the middle of a branding change, or by a - // removal that failed, is 2 MB nothing can reach. Here, where nothing is being - // placed, so a file under one of our names is a finished one. + // removal that failed, is 2 MB nothing can reach. Here, where nothing is + // being placed, so a file under one of our names is a finished one. sweepUnreferencedLogos().catch((error) => { logger.warn({ err: error }, 'Sweeping logos no longer in use failed'); }); // Cleanup on process termination - const cleanup = () => { + const cleanup = async () => { logger.info('Shutting down server...'); - terminalService.cleanup(); clearInterval(expirySweep); - tusUploads.stopCacheSweep(); - folderSizeManager.stop(); + terminalService.cleanup(); + performanceDiagnostics.stop(); trashMaintenance.stop(); + await tusUploads.stopCacheSweep(); + databaseMaintenance.stop(); + await folderSizeManager.stop(); searchIndexManager.stop(); server.close(() => { logger.info('Server closed'); diff --git a/backend/src/services/accessControlService.js b/backend/src/services/accessControlService.js index 12a51d96a..a2206f1aa 100644 --- a/backend/src/services/accessControlService.js +++ b/backend/src/services/accessControlService.js @@ -2,18 +2,22 @@ const fs = require('fs/promises'); const path = require('path'); const { directories } = require('../config/index'); -const { isInsidePersonalRoot, normalizeRelativePath } = require('../utils/pathUtils'); -const { getSettings, setSettings } = require('../services/settingsService'); +const { normalizeRelativePath, isInsidePersonalRoot } = require('../utils/pathUtils'); const { ruleAppliesToAdmins } = require('../utils/accessRules'); +const { getSettings, setSettings } = require('../services/settingsService'); /** - * Whether this rule binds the caller. + * Whether a rule is one of the rules this caller is held to. * - * An administrator used to sit outside read-only rules and inside hidden ones, + * An administrator used to be outside read-only rules and inside hidden ones, * which is neither and was written nowhere: a read-only rule left the Create - * button on a folder for them and they wrote into it, while a hidden rule took - * a folder away from the one account meant to manage it, with no way to say - * otherwise. It is one switch now, the same whatever the rule grants. + * button there for them (nxzai/NextExplorer#407), while a hidden rule took the + * folder away from the one account meant to manage it. Each rule now says it, + * with one switch whatever it grants, and the setting above it applies them all + * to administrators at once. + * + * A rule an administrator is not held to is not a rule that lets them through: + * it is skipped, so a later rule still has its say. */ const heldTo = (rule, { isAdmin, applyToAdmins }) => { if (!isAdmin) return true; @@ -22,12 +26,12 @@ const heldTo = (rule, { isAdmin, applyToAdmins }) => { }; /** - * The rules, as a question that can be asked many times without reading them - * again. + * Resolve a path against the access rules, for one caller. * - * @param {{rules?: Array, applyToAdmins?: boolean}} access the access - * section, rules and the setting above them together — not the rules alone, - * because whom a rule holds is decided by both. + * @param {{rules?: Array, applyToAdmins?: boolean}} access the rules in + * force and whether every one of them also holds administrators. An object + * rather than the bare list, so a caller cannot pass the rules and quietly + * lose the setting that decides who they bind. * @returns {(relativePath: string, who?: {isAdmin?: boolean}) => 'rw'|'ro'|'hidden'} */ const createPermissionResolver = (access = {}) => { @@ -42,19 +46,12 @@ const createPermissionResolver = (access = {}) => { for (const rule of normalizedRules) { const rulePath = normalizeRelativePath(rule.path || ''); if (!rulePath) continue; - // A rule that does not hold this caller does not stand in the way of a - // later one either: it is passed over, not matched and waived. if (!heldTo(rule, { isAdmin, applyToAdmins })) continue; - if (rule.recursive) { - if (rel === rulePath || rel.startsWith(rulePath + '/')) { - return rule.permissions || 'rw'; - } - } else { - if (rel === rulePath) { - return rule.permissions || 'rw'; - } - } + const matches = rule.recursive + ? rel === rulePath || rel.startsWith(`${rulePath}/`) + : rel === rulePath; + if (matches) return rule.permissions || 'rw'; } return 'rw'; @@ -62,7 +59,7 @@ const createPermissionResolver = (access = {}) => { }; // Determine permission for a given relative path: 'rw' | 'ro' | 'hidden' -const getPermissionForPath = async (relativePath, who) => { +const getPermissionForPath = async (relativePath, who = {}) => { const settings = await getSettings(); return createPermissionResolver(settings?.access)(relativePath, who); }; diff --git a/backend/src/services/accessManager.js b/backend/src/services/accessManager.js index ce8096df1..6389ef691 100644 --- a/backend/src/services/accessManager.js +++ b/backend/src/services/accessManager.js @@ -18,18 +18,19 @@ const { auth, features, directories } = require('../config/index'); * everyone is the same synthetic admin who already browses the whole * filesystem, so the prompt would only lock the share without protecting it. * - * Everyone else is subject to it, and that includes a signed-in account: being - * authenticated is not knowing the password. The check used to be "is there a - * user or a guest session", so any account on the instance opening a protected - * link walked straight past the prompt its owner set up. + * Everyone else is subject to it, and that includes a visitor with no account + * at all — the very people a public password is for. This used to require a + * user, so the predicate answered "no password here" for anonymous callers and + * each caller made up the difference on its own: one added `|| (hasPassword && + * !user)` to what it reported, another let the case fall through to a later + * branch that happened to refuse. The protection was real and lived in two + * places under a name that promised one. */ const sharePasswordApplies = (share, user) => Boolean(share.hasPassword) && auth.enabled !== false && !(user && String(user.id) === String(share.ownerId)); -const PERSONAL_SIDEWAYS = 'Personal folders are reached through the personal space'; - /** * Get comprehensive access information for a path * @param {Object} context - { user, guestSession, shareToken } @@ -91,7 +92,7 @@ const getVolumeAccess = async (context, relativePath, options = {}) => { // The same rule as below, from the folder this account was assigned. const innerPath = relativePath.split('/').filter(Boolean).slice(1).join('/'); if (reachesIntoPersonalRoot(userVolume.path, path.resolve(userVolume.path, innerPath))) { - return createDeniedAccess(PERSONAL_SIDEWAYS); + return createDeniedAccess('Personal folders are reached through the personal space'); } // Use the volume's access mode @@ -112,6 +113,7 @@ const getVolumeAccess = async (context, relativePath, options = {}) => { canDelete: !effectiveReadOnly, canUpload: !effectiveReadOnly, canCreateFolder: !effectiveReadOnly, + canCreateFile: !effectiveReadOnly, canShare: true, canDownload: true, isShared: false, @@ -127,23 +129,22 @@ const getVolumeAccess = async (context, relativePath, options = {}) => { // Somebody's personal folder is not part of the volume, even where it sits // inside it — `/_users` by default. Resolving such a path already // refuses; this is the same answer for a caller who asks about a path it - // never resolves. The search is one: ripgrep and the index read the whole - // volume and ask only this whether each path may be shown, so an ordinary - // account searching the volume was offered another account's private - // files, by name and by the line that matched. + // never resolves. The search is one: it takes paths from the index, which + // reads the whole volume, and asks only this whether each may be shown. An + // ordinary account searching the volume was offered another account's + // private files, by name and by what they said. if ( reachesIntoPersonalRoot( directories.volume, path.resolve(directories.volume, relativePath || '') ) ) { - return createDeniedAccess(PERSONAL_SIDEWAYS); + return createDeniedAccess('Personal folders are reached through the personal space'); } // Check access control rules. A rule that does not hold administrators was // already passed over while resolving, so what comes back is what binds this - // caller — and an administrator is no longer excused from it a second time - // here, which is how a read-only rule left them the Create button. + // caller: an administrator is no longer excused from it here. const permission = await getPerm(relativePath, { isAdmin }); if (permission === 'hidden') { return createDeniedAccess('Path is hidden'); @@ -158,6 +159,7 @@ const getVolumeAccess = async (context, relativePath, options = {}) => { canDelete: !isReadOnly, canUpload: !isReadOnly, canCreateFolder: !isReadOnly, + canCreateFile: !isReadOnly, canShare: true, canDownload: true, isShared: false, @@ -192,6 +194,7 @@ const getPersonalAccess = async (context, relativePath) => { canDelete: true, canUpload: true, canCreateFolder: true, + canCreateFile: true, canShare: true, canDownload: true, isShared: false, @@ -202,163 +205,146 @@ const getPersonalAccess = async (context, relativePath) => { }; /** - * Get access info for share paths + * Whether the share itself may be opened, before anyone is considered. + * + * @returns {object|null} a denial, or null when the share is usable */ -const getShareAccess = async (context, shareToken, innerPath, options = {}) => { - const { user, guestSession } = context; - const permissionResolver = - typeof options.permissionResolver === 'function' ? options.permissionResolver : null; - // What a share opens is bound by the rules whoever follows the link is: the - // link is the lens, not the account behind it, so no rule is waived here for - // an administrator. - const getPerm = async (p) => - permissionResolver - ? permissionResolver(p, { isAdmin: false }) - : await getPermissionForPath(p, { isAdmin: false }); - const shareCache = options && options.shareCache instanceof Map ? options.shareCache : null; - const userVolumeCache = - options && options.userVolumeCache instanceof Map ? options.userVolumeCache : null; - - if (!shareToken) { - return createDeniedAccess('Share token is required'); - } - - // Validate share exists - let share = shareCache ? shareCache.get(shareToken) : null; - if (!share) { - share = await getShareByToken(shareToken); - if (shareCache && share) shareCache.set(shareToken, share); - } - if (!share) { - return createDeniedAccess('Share not found'); - } - - // Check expiration - if (isShareExpired(share)) { - return createDeniedAccess('Share has expired'); - } +const shareIsUnusable = (share) => { + if (!share) return createDeniedAccess('Share not found'); + if (isShareExpired(share)) return createDeniedAccess('Share has expired'); + return null; +}; - // Check sharing type and permissions +/** + * Whether this caller may open it. + * + * The two sharing types ask different questions — one wants an account on the + * list, the other wants a session that came through the door — and anything + * else fails closed rather than falling through to the grant below. + * + * @returns {Promise} a denial, or null when the caller may open it + */ +const callerMayNotOpen = async (share, { user, guestSession }) => { if (share.sharingType === 'users') { - // User-specific share requires authentication - if (!user || !user.id) { - return createDeniedAccess('Authentication required'); - } - - // Check if user has permission + if (!user || !user.id) return createDeniedAccess('Authentication required'); const permitted = await hasUserPermission(share.id, user.id); - if (!permitted) { - return createDeniedAccess('Access denied'); - } - } else if (share.sharingType === 'anyone') { - // Anyone shares require either user auth OR guest session - if (!user && !guestSession) { - // Password verification happens during share access/login - // If neither user nor guest session exists, they need to go through verification - return createDeniedAccess('Share access required'); - } + return permitted ? null : createDeniedAccess('Access denied'); + } + + if (share.sharingType === 'anyone') { + // Password verification happens during share access; a caller with neither + // an account nor a guest session has been through neither. + if (!user && !guestSession) return createDeniedAccess('Share access required'); - // If guest session exists, verify it belongs to this share if (guestSession && !user && guestSession.shareId !== share.id) { return createDeniedAccess('Invalid guest session for this share'); } - // A guest session for this share is the proof the password was typed. + // Being signed in is not the same as knowing the password. Without this, + // any authenticated user opening a protected link skipped the prompt the + // owner set it up for. if (sharePasswordApplies(share, user)) { const verified = guestSession && guestSession.shareId === share.id; if (!verified) return createDeniedAccess('Password verification required'); } - } else { - // Neither of the two types this knows: fail closed rather than fall - // through to the grant below. - return createDeniedAccess('Unknown sharing type'); + return null; } - const isOwner = user && user.id === share.ownerId; - const shareReadWrite = share.accessMode === 'readwrite'; + // Fail closed: a sharing type we do not know about must not fall through to + // the permission grant. + return createDeniedAccess('Unknown sharing type'); +}; - // Cap share write permissions by the underlying source permission. - // This allows admin changes (hide/ro/user-volume readonly) to take effect immediately. +/** + * What the location underneath still allows, which caps what the share grants. + * + * An administrator hiding a folder, marking it read-only or reassigning a + * personal volume takes effect on every existing link immediately, because the + * answer is read here on every request rather than frozen when the link was + * made. + * + * @returns {Promise<{denial: object}|{readOnly: boolean}>} + */ +const readSourceLimits = async (share, innerPath, { getPerm, userVolumeCache }) => { const isDirShare = Boolean(share.isDirectory); const safeInnerPath = typeof innerPath === 'string' ? innerPath : ''; - let underlyingPermission = 'rw'; - let underlyingReadOnly = false; + const under = (base) => + isDirShare && safeInnerPath ? combineRelativePath(base, safeInnerPath) : base; + + // Somebody's personal folder is not handed out by a share of a folder that + // holds it, any more than by the volume itself. + const personalRootDenial = (root, inner) => + reachesIntoPersonalRoot(root, path.resolve(root, inner || '')) + ? { denial: createDeniedAccess('Personal folders are reached through the personal space') } + : null; if (share.sourceSpace === 'volume') { - const combined = - isDirShare && safeInnerPath - ? combineRelativePath(share.sourcePath, safeInnerPath) - : share.sourcePath; - // Somebody's personal folder is not handed out by a share of a folder that - // holds it, any more than by the volume itself. - if ( - reachesIntoPersonalRoot(directories.volume, path.resolve(directories.volume, combined || '')) - ) { - return createDeniedAccess(PERSONAL_SIDEWAYS); - } - underlyingPermission = await getPerm(combined); - if (underlyingPermission === 'hidden') { - return createDeniedAccess('Path is hidden'); - } - underlyingReadOnly = underlyingPermission === 'ro'; - } else if (share.sourceSpace === 'user_volume') { + const refused = personalRootDenial(directories.volume, under(share.sourcePath)); + if (refused) return refused; + // What a share opens is bound by the rules whoever follows the link is: + // the link is the lens, not the account behind it, so no rule is waived + // here for an administrator. + const permission = await getPerm(under(share.sourcePath), { isAdmin: false }); + if (permission === 'hidden') return { denial: createDeniedAccess('Path is hidden') }; + return { readOnly: permission === 'ro' }; + } + + if (share.sourceSpace === 'user_volume') { const [volumeId, ...rest] = String(share.sourcePath || '') .split('/') .filter(Boolean); - if (!volumeId) { - return createDeniedAccess('Share source volume is invalid'); - } + if (!volumeId) return { denial: createDeniedAccess('Share source volume is invalid') }; + let userVolume = userVolumeCache ? userVolumeCache.get(volumeId) : null; if (!userVolume) { userVolume = await getVolumeById(volumeId); if (userVolumeCache && userVolume) userVolumeCache.set(volumeId, userVolume); } - if (!userVolume) { - return createDeniedAccess('Share source volume not found'); - } + if (!userVolume) return { denial: createDeniedAccess('Share source volume not found') }; + + // A share may only hand out a volume its own owner holds: without this, an + // account that once had one assigned could go on sharing it afterwards. if (String(userVolume.userId) !== String(share.ownerId)) { - return createDeniedAccess('Share source volume mismatch'); + return { denial: createDeniedAccess('Share source volume mismatch') }; } - const baseWithinVolume = rest.join('/'); - const combinedWithinVolume = - isDirShare && safeInnerPath - ? combineRelativePath(baseWithinVolume, safeInnerPath) - : baseWithinVolume; - if ( - reachesIntoPersonalRoot( - userVolume.path, - path.resolve(userVolume.path, combinedWithinVolume || '') - ) - ) { - return createDeniedAccess(PERSONAL_SIDEWAYS); - } - const logicalForRules = `${userVolume.label}${combinedWithinVolume ? `/${combinedWithinVolume}` : ''}`; - underlyingPermission = await getPerm(logicalForRules); - if (underlyingPermission === 'hidden') { - return createDeniedAccess('Path is hidden'); - } - underlyingReadOnly = userVolume.accessMode === 'readonly' || underlyingPermission === 'ro'; + const refused = personalRootDenial(userVolume.path, under(rest.join('/'))); + if (refused) return refused; + + const logicalForRules = `${userVolume.label}${under(rest.join('/')) ? `/${under(rest.join('/'))}` : ''}`; + const permission = await getPerm(logicalForRules, { isAdmin: false }); + if (permission === 'hidden') return { denial: createDeniedAccess('Path is hidden') }; + return { readOnly: userVolume.accessMode === 'readonly' || permission === 'ro' }; } - const isReadWrite = shareReadWrite && !underlyingReadOnly; + return { readOnly: false }; +}; + +/** What the share hands out, once the location underneath has had its say. */ +const grantFor = (share, { user, readOnly }) => { + const isReadWrite = share.accessMode === 'readwrite' && !readOnly; return { canAccess: true, canRead: true, canWrite: isReadWrite, - canDelete: isReadWrite, - canUpload: isReadWrite, - canCreateFolder: isReadWrite, + canDelete: isReadWrite && share.allowDelete !== false, + canUpload: isReadWrite && share.allowUpload !== false, + canCreateFolder: isReadWrite && share.allowCreateFolder !== false, + canCreateFile: isReadWrite && share.allowCreateFile !== false, canShare: false, // Cannot create shares within shares - canDownload: true, + // Deliberately not gated on `isReadWrite` like the others above it: a + // read-only share is exactly where withholding downloads means something — + // "read this" rather than "take a copy of this". Defaults to allowed, so + // every share made before this existed behaves as it always did. + canDownload: share.allowDownload !== false, isShared: true, shareInfo: { shareId: share.id, shareToken: share.shareToken, accessMode: isReadWrite ? 'readwrite' : 'readonly', expiresAt: share.expiresAt, - isOwner, + isOwner: Boolean(user && user.id === share.ownerId), label: share.label, }, share, // Include full share object for path resolution (avoids duplicate DB query) @@ -367,6 +353,60 @@ const getShareAccess = async (context, shareToken, innerPath, options = {}) => { }; }; +/** + * What a caller may do with a path inside a share. + * + * Three questions in order, each answerable on its own: may this share be + * opened at all, may this caller open it, and what does the location underneath + * still allow. Only then is a grant composed. It was one function of fifty-three + * paths, which is fifty-three tests to know it — and the reason it is worth + * splitting is that it decides what a link hands out. + */ +/** + * The optional machinery a caller may hand in: a permission resolver, and two + * caches for a route that is asking about many paths at once. Normalised here + * so the decision below reads as the sequence of questions it is. + */ +const readOptions = (options = {}) => { + const permissionResolver = + typeof options.permissionResolver === 'function' ? options.permissionResolver : null; + + return { + getPerm: async (p, who) => + permissionResolver ? permissionResolver(p, who) : getPermissionForPath(p, who), + shareCache: options.shareCache instanceof Map ? options.shareCache : null, + userVolumeCache: options.userVolumeCache instanceof Map ? options.userVolumeCache : null, + }; +}; + +/** The share this token names, from the caller's cache when it has one. */ +const loadShare = async (shareToken, shareCache) => { + const cached = shareCache ? shareCache.get(shareToken) : null; + if (cached) return cached; + + const share = await getShareByToken(shareToken); + if (shareCache && share) shareCache.set(shareToken, share); + return share; +}; + +const getShareAccess = async (context, shareToken, innerPath, options = {}) => { + if (!shareToken) return createDeniedAccess('Share token is required'); + + const { getPerm, shareCache, userVolumeCache } = readOptions(options); + const share = await loadShare(shareToken, shareCache); + + const unusable = shareIsUnusable(share); + if (unusable) return unusable; + + const refused = await callerMayNotOpen(share, context); + if (refused) return refused; + + const limits = await readSourceLimits(share, innerPath, { getPerm, userVolumeCache }); + if (limits.denial) return limits.denial; + + return grantFor(share, { user: context.user, readOnly: limits.readOnly }); +}; + /** * Helper to create a denied access object */ @@ -378,6 +418,7 @@ const createDeniedAccess = (reason) => { canDelete: false, canUpload: false, canCreateFolder: false, + canCreateFile: false, canShare: false, canDownload: false, isShared: false, @@ -432,42 +473,12 @@ const resolvePathWithAccess = async (context, relativePath, options = {}) => { return { accessInfo, resolved }; }; -/** - * Check if user can create shares (only authenticated users, not guests) - */ -const canCreateShare = (context) => { - const { user, guestSession } = context; - - // Guests cannot create shares - if (guestSession) { - return false; - } - - // Must be authenticated - return Boolean(user && user.id); -}; - -/** - * Get context from request object - */ -const getContextFromRequest = (req) => { - return { - user: req.user || null, - guestSession: req.guestSession || null, - shareToken: req.shareToken || null, - }; -}; - module.exports = { getAccessInfo, - getVolumeAccess, getPersonalAccess, - sharePasswordApplies, getShareAccess, canAccess, canWrite, - canCreateShare, - getContextFromRequest, - createDeniedAccess, + sharePasswordApplies, resolvePathWithAccess, }; diff --git a/backend/src/services/archiveCacheService.js b/backend/src/services/archiveCacheService.js index 87fec95c3..fedb151be 100644 --- a/backend/src/services/archiveCacheService.js +++ b/backend/src/services/archiveCacheService.js @@ -6,7 +6,7 @@ const fs = require('fs/promises'); * application's cache directory, written here and read here. Nothing anybody * put anywhere ever passes through it, so it does not go through the trash. */ -/* eslint-disable no-restricted-properties */ + const fss = require('fs'); const crypto = require('crypto'); const { spawn } = require('child_process'); diff --git a/backend/src/services/archiveExtraction.js b/backend/src/services/archiveExtraction.js index d2bcbd3c9..0728003ec 100644 --- a/backend/src/services/archiveExtraction.js +++ b/backend/src/services/archiveExtraction.js @@ -6,6 +6,7 @@ const { placeWithoutOverwrite } = require('../utils/placeWithoutOverwrite'); const { takeInventory } = require('../utils/ownedTree'); const { ValidationError } = require('../errors/AppError'); const { archives } = require('../config/index'); +const folderSizeHooks = require('./folderSizeHooks'); /** * What is shared between extracting a whole archive and extracting part of one. @@ -79,6 +80,13 @@ const extractIntoCurrentFolder = async ({ ); movedPaths.push({ path: destinationPath, inventory }); + if (entry.isDirectory()) { + folderSizeHooks.onDirectoryTreeCreated(destinationPath); + } else { + const stats = await fs.stat(destinationPath); + folderSizeHooks.onFileWritten(destinationPath, stats.size); + } + items.push(await buildItemMetadata(destinationPath, relativeParentPath, destinationName)); } diff --git a/backend/src/services/archiveTree.js b/backend/src/services/archiveTree.js index d60071e30..96f690a5b 100644 --- a/backend/src/services/archiveTree.js +++ b/backend/src/services/archiveTree.js @@ -1,7 +1,7 @@ const fs = require('fs/promises'); const fss = require('fs'); const path = require('path'); -const archiver = require('archiver'); +const { ZipArchive } = require('archiver'); const { excludedFiles } = require('../config/index'); const { combineRelativePath, isInsidePersonalRoot } = require('../utils/pathUtils'); @@ -30,7 +30,7 @@ const { getSettings } = require('./settingsService'); */ const readAccess = async () => { const settings = await getSettings(); - return settings?.access || null; + return settings?.access && typeof settings.access === 'object' ? settings.access : { rules: [] }; }; /** @@ -42,8 +42,9 @@ const readAccess = async () => { */ const collectArchiveEntries = async (context, sources) => { const access = await readAccess(); + const rules = Array.isArray(access.rules) ? access.rules : []; const accessOptions = { - permissionResolver: access?.rules?.length ? createPermissionResolver(access) : undefined, + permissionResolver: rules.length ? createPermissionResolver(access) : undefined, shareCache: new Map(), userVolumeCache: new Map(), }; @@ -56,7 +57,7 @@ const collectArchiveEntries = async (context, sources) => { if (guardPersonalRoot && isInsidePersonalRoot(absolutePath)) return false; // With no rules, nothing below a readable folder is less readable than it: // every other decision was made once, for the source. - if (!access?.rules?.length) return true; + if (!rules.length) return true; const info = await getAccessInfo(context, logicalPath, accessOptions); return Boolean(info?.canAccess && info.canRead); }; @@ -68,20 +69,20 @@ const collectArchiveEntries = async (context, sources) => { const absolutePath = path.join(absoluteDir, child.name); const logicalPath = combineRelativePath(logicalDir, child.name); const name = `${entryDir}/${child.name}`; - // eslint-disable-next-line no-await-in-loop + if (!(await visible({ absolutePath, logicalPath, name: child.name, guardPersonalRoot }))) { excluded += 1; continue; } - // eslint-disable-next-line no-await-in-loop + const stats = await fs.lstat(absolutePath); if (stats.isSymbolicLink()) { // Kept as the link it is, never followed: its target is text, not content. - // eslint-disable-next-line no-await-in-loop + entries.push({ type: 'symlink', name, target: await fs.readlink(absolutePath) }); } else if (stats.isDirectory()) { entries.push({ type: 'directory', name }); - // eslint-disable-next-line no-await-in-loop + await walk({ absoluteDir: absolutePath, logicalDir: logicalPath, @@ -104,7 +105,7 @@ const collectArchiveEntries = async (context, sources) => { entries.push({ type: 'directory', name: entryName }); // A folder that is itself inside the personal root was reached through the // personal space, or a share of it, which already decided whose it is. - // eslint-disable-next-line no-await-in-loop + await walk({ absoluteDir: source.absolutePath, logicalDir: source.logicalPath, @@ -155,7 +156,7 @@ const writeZipFile = (entries, destinationPath, { totalBytes = 0, onPercent, sig } const output = fss.createWriteStream(destinationPath); - const archive = archiver('zip', { zlib: { level: 1 } }); + const archive = new ZipArchive({ zlib: { level: 1 } }); let settled = false; const finish = (error) => { if (settled) return; diff --git a/backend/src/services/authorizationService.js b/backend/src/services/authorizationService.js index a940b9e1b..dd11ac436 100644 --- a/backend/src/services/authorizationService.js +++ b/backend/src/services/authorizationService.js @@ -26,10 +26,8 @@ const actionToFlag = (action) => { return 'canUpload'; case ACTIONS.createFolder: return 'canCreateFolder'; - // Upstream expresses "may put a file here" as canUpload; a trash restore of - // a file asks for exactly that. case ACTIONS.createFile: - return 'canUpload'; + return 'canCreateFile'; case ACTIONS.rename: return 'canWrite'; case ACTIONS.download: diff --git a/backend/src/services/capabilities.js b/backend/src/services/capabilities.js index d2c7e35cb..5e07eed13 100644 --- a/backend/src/services/capabilities.js +++ b/backend/src/services/capabilities.js @@ -138,7 +138,7 @@ const probeMachine = async () => { const ffmpegRunner = require('./ffmpegRunner'); const { hasPdfToText } = require('./pdfTextExtract'); const archives = require('./archiveService'); - const { nativeCopyEnabled } = require('./fileTransferService'); + const { nativeTransferEnabled } = require('./fileTransferService'); const [ripgrep, rsync, pdftotext, exiftool, sevenZip, missingFormats] = await Promise.all([ probe('rg', ['--version']), @@ -181,7 +181,7 @@ const probeMachine = async () => { exiftool, sevenZip, missingFormats, - nativeTransfers: nativeCopyEnabled(), + nativeTransfers: nativeTransferEnabled(), versions, }; }; diff --git a/backend/src/services/collaboraDiscoveryService.js b/backend/src/services/collaboraDiscoveryService.js index 6f20f52d6..7b19e2d66 100644 --- a/backend/src/services/collaboraDiscoveryService.js +++ b/backend/src/services/collaboraDiscoveryService.js @@ -20,13 +20,13 @@ const parseDiscoveryXml = (xml) => { const content = typeof xml === 'string' ? xml : ''; const actionTagRegex = /]*?)\/?>/gi; - let match = null; + let match; while ((match = actionTagRegex.exec(content))) { const attrsRaw = match[1] || ''; const attrs = {}; const attrRegex = /([A-Za-z0-9:_-]+)="([^"]*)"/g; - let a = null; + let a; while ((a = attrRegex.exec(attrsRaw))) { attrs[a[1]] = a[2]; } diff --git a/backend/src/services/db.js b/backend/src/services/db.js index 09b8189cd..91c6b1a06 100644 --- a/backend/src/services/db.js +++ b/backend/src/services/db.js @@ -207,6 +207,9 @@ const ACTIVITY_DDL = ` CREATE INDEX IF NOT EXISTS idx_activity_user ON activity_events(user_id, at DESC); `; +const tableExists = (db, name) => + Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name)); + const getDbPath = () => { const configDir = directories.config; // Generic app database for auth, shares, and user settings. @@ -221,6 +224,176 @@ const generateId = () => : `${Date.now().toString(36)}-${crypto.randomBytes(8).toString('hex')}`; const DEFAULT_FAVORITE_ICON = favorites.defaultIcon; +/** + * Where a user has recently moved or copied things to. + * + * Kept as history rather than a preference: the point is that the folders you + * actually use rise to the top of the destination picker without anyone + * curating a list. One row per user and path — using a destination again moves + * it up rather than adding a duplicate. + */ +const RECENT_DESTINATIONS_DDL = ` + CREATE TABLE IF NOT EXISTS recent_destinations ( + user_id TEXT NOT NULL, + path TEXT NOT NULL, + used_at DATETIME NOT NULL, + PRIMARY KEY (user_id, path) + ); +`; + +/** + * What a user chose for one folder: how to sort it, how to show it. + * + * A row per folder rather than one JSON blob per user. The blob had to be + * capped — it was read and rewritten whole on every change, and shipped + * entire on every load — so the hundred-and-first folder silently forgot the + * oldest. More importantly, a blob cannot be cleaned up: deleting a folder + * could not remove what everyone else had chosen for it, and renaming one left + * the preferences behind on a path that no longer existed. + */ +const FOLDER_PREFERENCES_DDL = ` + CREATE TABLE IF NOT EXISTS folder_preferences ( + user_id TEXT NOT NULL, + path TEXT NOT NULL, + sort_by TEXT, + sort_order TEXT, + view_mode TEXT, + updated_at DATETIME NOT NULL, + PRIMARY KEY (user_id, path) + ); + CREATE INDEX IF NOT EXISTS idx_folder_preferences_path ON folder_preferences(path); +`; + +/** + * Carry per-folder preferences out of the JSON blob they used to live in. + * + * They were two maps under `user_settings` — one for sorting, one for the view + * mode — capped at a hundred entries each because the whole blob was rewritten + * on every change. As rows they need no cap, and they can finally be cleaned up + * when the folder they describe is deleted or renamed. + * + * Best-effort: a preference that fails to migrate costs a folder its remembered + * sort, which is not worth failing a startup over. + */ +const migrateFolderPreferencesFromUserSettings = (db) => { + let rows; + try { + rows = db + .prepare( + "SELECT user_id, key, value FROM user_settings WHERE key IN ('folderSorts', 'folderViews')" + ) + .all(); + } catch (error) { + logger.debug({ err: error }, '[DB Migration] No folder preferences to carry over'); + return; + } + + const merged = new Map(); + for (const row of rows) { + let parsed; + try { + parsed = JSON.parse(row.value); + } catch { + continue; + } + if (!parsed || typeof parsed !== 'object') continue; + + for (const [folderPath, entry] of Object.entries(parsed)) { + if (!folderPath || !entry || typeof entry !== 'object') continue; + + const key = `${row.user_id}\u0000${folderPath}`; + const current = merged.get(key) || { + userId: row.user_id, + path: folderPath, + sortBy: null, + sortOrder: null, + viewMode: null, + updatedAt: 0, + }; + + if (row.key === 'folderSorts' && typeof entry.by === 'string') { + current.sortBy = entry.by; + current.sortOrder = entry.order === 'desc' ? 'desc' : 'asc'; + } else if (row.key === 'folderViews' && typeof entry.mode === 'string') { + current.viewMode = entry.mode; + } + + const updatedAt = Number(entry.updatedAt); + if (Number.isFinite(updatedAt) && updatedAt > current.updatedAt) { + current.updatedAt = updatedAt; + } + merged.set(key, current); + } + } + + if (merged.size === 0) return; + + const insert = db.prepare( + `INSERT OR REPLACE INTO folder_preferences + (user_id, path, sort_by, sort_order, view_mode, updated_at) + VALUES (?, ?, ?, ?, ?, ?)` + ); + + for (const entry of merged.values()) { + insert.run( + entry.userId, + entry.path, + entry.sortBy, + entry.sortOrder, + entry.viewMode, + new Date(entry.updatedAt || Date.now()).toISOString() + ); + } + + db.prepare("DELETE FROM user_settings WHERE key IN ('folderSorts', 'folderViews')").run(); + logger.info({ count: merged.size }, '[DB Migration] Folder preferences moved to their own table'); +}; + +const ensureShareOperationPermissionColumns = (db) => { + addColumnIfMissing(db, 'shares', 'allow_delete', 'allow_delete INTEGER NOT NULL DEFAULT 1'); + addColumnIfMissing( + db, + 'shares', + 'allow_create_folder', + 'allow_create_folder INTEGER NOT NULL DEFAULT 1' + ); + addColumnIfMissing( + db, + 'shares', + 'allow_create_file', + 'allow_create_file INTEGER NOT NULL DEFAULT 1' + ); + addColumnIfMissing(db, 'shares', 'allow_upload', 'allow_upload INTEGER NOT NULL DEFAULT 1'); + // Defaults to 1 so every share that already exists keeps working exactly as + // it did: withholding downloads is something an owner opts into, never + // something a migration decides for them. + addColumnIfMissing(db, 'shares', 'allow_download', 'allow_download INTEGER NOT NULL DEFAULT 1'); + + // What a share hands out of a file's history: the list of its versions, and + // the versions themselves. A link for anyone shows none until its owner says + // so; a share with named accounts shows what those accounts would see anyway, + // which is why the ones that exist already are switched on as they gain it. + const columns = new Set( + db + .prepare('PRAGMA table_info(shares)') + .all() + .map((column) => column.name) + ); + for (const column of ['versions_visible', 'versions_download']) { + if (columns.has(column)) continue; + db.exec(`ALTER TABLE shares ADD COLUMN ${column} INTEGER NOT NULL DEFAULT 0`); + db.exec(`UPDATE shares SET ${column} = 1 WHERE sharing_type = 'users'`); + } +}; + +const PERSONAL_FOLDER_RESERVATIONS_DDL = ` + CREATE TABLE IF NOT EXISTS personal_folder_reservations ( + name TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + reserved_at TEXT NOT NULL + ); +`; + const migrate = (db) => { // Simple schema versioning db.exec(` @@ -554,7 +727,7 @@ const migrate = (db) => { } if (version < 9) { logger.info('[DB Migration] Migrating to v9: Full-text search index...'); - // eslint-disable-next-line global-require + db.exec(require('./searchIndexStore').SEARCH_INDEX_DDL); db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run( 'schema_version', @@ -579,7 +752,7 @@ const migrate = (db) => { db.exec( 'CREATE UNIQUE INDEX IF NOT EXISTS idx_users_personal_folder ON users(personal_folder_name);' ); - // eslint-disable-next-line global-require + const { claimAllPersonalFolderNames } = require('./personalFolders'); const claimed = claimAllPersonalFolderNames(db); logger.info({ claimed }, '[DB Migration] Personal folder names assigned'); @@ -665,6 +838,53 @@ const migrate = (db) => { ); version = 19; } + if (version < 20) { + logger.info('[DB Migration] Migrating to v20: per-folder preferences as rows...'); + db.exec(FOLDER_PREFERENCES_DDL); + migrateFolderPreferencesFromUserSettings(db); + db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run( + 'schema_version', + String(20) + ); + version = 20; + } + if (version < 21) { + logger.info('[DB Migration] Migrating to v21: the folders somebody files into...'); + db.exec(RECENT_DESTINATIONS_DDL); + db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run( + 'schema_version', + String(21) + ); + version = 21; + } + if (version < 22) { + logger.info('[DB Migration] Migrating to v22: what a share permits, and when it was used...'); + ensureShareOperationPermissionColumns(db); + addColumnIfMissing(db, 'shares', 'last_downloaded_at', 'last_downloaded_at DATETIME'); + addColumnIfMissing(db, 'shares', 'last_download_ip', 'last_download_ip TEXT'); + db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run( + 'schema_version', + String(22) + ); + version = 22; + } + if (version < 23) { + logger.info('[DB Migration] Migrating to v23: one personal folder per account...'); + db.exec(PERSONAL_FOLDER_RESERVATIONS_DDL); + // Rows an account's deletion used to leave behind. None of these tables + // points at users through a foreign key, so nothing ever removed them. + if (tableExists(db, 'folder_preferences')) { + db.exec('DELETE FROM folder_preferences WHERE user_id NOT IN (SELECT id FROM users)'); + } + if (tableExists(db, 'recent_destinations')) { + db.exec('DELETE FROM recent_destinations WHERE user_id NOT IN (SELECT id FROM users)'); + } + db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run( + 'schema_version', + String(23) + ); + version = 23; + } })(); }; @@ -866,9 +1086,7 @@ const ensureAnonymousUser = (db) => { } }; -const getDb = async () => { - if (dbInstance) return dbInstance; - +const openDb = async () => { const dbDir = directories.config; await ensureDir(dbDir); const dbPath = getDbPath(); @@ -896,6 +1114,12 @@ const getDb = async () => { // /config may be past v12 without the trash tables. try { db.exec(TRASH_DDL); + addColumnIfMissing(db, 'trash_items', 'restore_entry', 'restore_entry TEXT'); + // `CREATE TABLE IF NOT EXISTS` adds nothing to a table that is already + // there, so an installation that made `trash_items` before this column + // joined the definition never gets it — and a restore into a chosen folder + // writes it. Added here rather than in a numbered migration because the + // table itself is ensured on every open, for the same reason. } catch (err) { logger.warn({ err }, '[DB] Failed to ensure trash tables'); } @@ -938,11 +1162,67 @@ const getDb = async () => { logger.warn({ err }, '[DB] Failed to ensure the ONLYOFFICE editor session table'); } ensureAnonymousUser(db); - dbInstance = db; - return dbInstance; + return db; +}; + +/** + * The application database, opened on first use. + * + * Everything that starts with the server asks for it at once. Each caller passed the + * check for an open database before the first one had finished opening it, and went on + * to open app.db again and run the migrations over it in parallel: four connections at + * every start, four sets of `CREATE TABLE IF NOT EXISTS`, and whichever finished last + * became the one everybody used. One opening is shared instead. + */ +let dbOpening = null; +const getDb = async () => { + if (dbInstance) return dbInstance; + if (!dbOpening) { + dbOpening = openDb() + .then((db) => { + dbInstance = db; + return db; + }) + .finally(() => { + dbOpening = null; + }); + } + return dbOpening; +}; + +const closeDb = () => { + if (!dbInstance) return; + dbInstance.close(); + dbInstance = null; +}; + +/** + * A statement prepared once per database and kept. + * + * `db.prepare` compiles the SQL every time it is called, and the hot paths — a listing + * asking whether each of a thousand rows is a favourite — called it per row. Kept in a + * WeakMap so the cache goes when the connection does, which is what a test that opens a + * database per case needs. + */ +const statementCache = new WeakMap(); + +const prepared = (db, sql) => { + let cache = statementCache.get(db); + if (!cache) { + cache = new Map(); + statementCache.set(db, cache); + } + let statement = cache.get(sql); + if (!statement) { + statement = db.prepare(sql); + cache.set(sql, statement); + } + return statement; }; module.exports = { + closeDb, + prepared, getDb, getDbPath, // The index database keeps its own copy of this table, so it needs the same diff --git a/backend/src/services/directoryListingService.js b/backend/src/services/directoryListingService.js index 1e3c4e4c4..01e247fe9 100644 --- a/backend/src/services/directoryListingService.js +++ b/backend/src/services/directoryListingService.js @@ -2,7 +2,7 @@ const path = require('path'); const fs = require('fs/promises'); const { excludedFiles, extensions, hiddenFiles } = require('../config/index'); -const { combineRelativePath } = require('../utils/pathUtils'); +const { combineRelativePath, resolveLogicalPath } = require('../utils/pathUtils'); const { getAccessInfo } = require('./accessManager'); const { createPermissionResolver } = require('./accessControlService'); const logger = require('../utils/logger'); @@ -41,28 +41,54 @@ const mapWithConcurrency = async (items, concurrency, mapper) => { return results; }; +/** + * Whether a symbolic link leads out of the space it sits in. + * + * Asked of the same resolver every operation goes through, so the listing and + * the operations cannot disagree: a link the resolver refuses is one nothing + * can be done through. A link it cannot follow at all — to nothing — is left to + * the stat that comes next, which skips it as before. + */ +const linkLeavesTheSpace = async (context, logicalPath, access) => { + try { + await resolveLogicalPath(logicalPath, { + user: context?.user || null, + guestSession: context?.guestSession || null, + share: access?.share || null, + userVolume: access?.userVolume || null, + }); + return false; + } catch (error) { + return error?.statusCode === 403; + } +}; + /** * List a directory and filter out entries that the caller cannot access. * * - Uses accessManager for per-child visibility (covers shares + user volumes + hidden rules). * - Does not throw for child-level failures; unreadable / inaccessible children are skipped. + * - A symbolic link that leads out of the space is listed as what it is — a link, + * marked `link: 'outside'` — and never followed. It used to be described by + * what it points at: the size and type of a file outside the volume, on a row + * every action then refused with "Resolved path is outside the configured + * volume root", with nothing on screen to say why. */ const listDirectoryItems = async ({ absoluteDir, parentLogicalPath, context, thumbsEnabled, - excludeDownloadArtifacts = false, includeHiddenFiles = false, itemExtras = null, access = null, shareCache = null, userVolumeCache = null, }) => { - // The whole access section rather than the rules alone: whom a rule holds is - // decided by the rule and by the setting above it together, so a resolver - // built from half of it would answer for the wrong caller. - const permissionResolver = access?.rules?.length ? createPermissionResolver(access) : null; + // The whole section, never the bare list: a caller handing over the rules + // alone would silently drop the setting that says whom they hold. + const permissionResolver = + Array.isArray(access?.rules) && access.rules.length ? createPermissionResolver(access) : null; const accessOptions = { ...(permissionResolver ? { permissionResolver } : null), @@ -70,21 +96,40 @@ const listDirectoryItems = async ({ ...(userVolumeCache instanceof Map ? { userVolumeCache } : null), }; + /** + * Which entries carry the read-only mark: the ones a rule holds this caller + * to, and only where that starts. + * + * A rule was invisible until something was attempted in the folder it covers, + * and on an account no rule held it was never refused at all + * (nxzai/NextExplorer#407). The mark says it up front — but a recursive rule + * covers everything below it, and a lock on every row inside a folder that is + * already read-only says nothing the row above did not. So it is drawn where + * the restriction begins: on the entry whose folder is not itself read-only. + * + * Asked of the rules alone, not of the whole access decision: this mark is + * about a rule, and a volume read-only for another reason carries its own. + */ + const isAdmin = Boolean(context?.user?.roles?.includes?.('admin')); + const ruleSays = (logicalPath) => + permissionResolver ? permissionResolver(logicalPath || '', { isAdmin }) : 'rw'; + const insideReadOnly = ruleSays(parentLogicalPath) === 'ro'; + const entries = await fs.readdir(absoluteDir); const filtered = entries .filter((name) => !excludedFiles.includes(name)) - .filter((name) => includeHiddenFiles || !hiddenFiles.isHiddenName(name)) - .filter((name) => - excludeDownloadArtifacts ? path.extname(name).toLowerCase() !== '.download' : true - ); + .filter((name) => includeHiddenFiles || !hiddenFiles.isHiddenName(name)); const items = await mapWithConcurrency(filtered, LIST_DIRECTORY_CONCURRENCY, async (name) => { const filePath = path.join(absoluteDir, name); + const logicalChildPath = combineRelativePath(parentLogicalPath || '', name); let stats; + let entry; try { - stats = await fs.stat(filePath); + entry = await fs.lstat(filePath); + stats = entry.isSymbolicLink() ? null : entry; } catch (err) { if (['EPERM', 'EACCES', 'ENOENT', 'ELOOP'].includes(err?.code)) { logger.warn({ filePath, err }, 'Skipping unreadable entry'); @@ -93,12 +138,33 @@ const listDirectoryItems = async ({ throw err; } - const logicalChildPath = combineRelativePath(parentLogicalPath || '', name); const childAccess = await getAccessInfo(context, logicalChildPath, accessOptions); if (!childAccess?.canAccess) { return null; } + if (!stats) { + if (await linkLeavesTheSpace(context, logicalChildPath, childAccess)) { + return { + name, + path: parentLogicalPath, + dateModified: entry.mtime, + size: null, + kind: toKind(entry, name), + link: 'outside', + }; + } + try { + stats = await fs.stat(filePath); + } catch (err) { + if (['EPERM', 'EACCES', 'ENOENT', 'ELOOP'].includes(err?.code)) { + logger.warn({ filePath, err }, 'Skipping unreadable entry'); + return null; + } + throw err; + } + } + const kind = toKind(stats, name); const item = { name, @@ -108,8 +174,6 @@ const listDirectoryItems = async ({ kind, }; - // Advisory only, and never a lock: who has this document open in an - // editor, so the row can say so and a move can ask first. if (stats.isFile()) { const activity = onlyofficeActivity.get(filePath); if (activity?.active) item.onlyofficeActivity = activity; @@ -119,6 +183,9 @@ const listDirectoryItems = async ({ item.supportsThumbnail = true; } + // `access`, as a volume held to reading says it: the same lock, the same reason. + if (!insideReadOnly && ruleSays(logicalChildPath) === 'ro') item.readOnly = 'access'; + if (typeof itemExtras === 'function') { Object.assign(item, itemExtras({ name, stats, kind, access: childAccess }) || {}); } diff --git a/backend/src/services/documentText.js b/backend/src/services/documentText.js index 9ce1d08f2..c88457793 100644 --- a/backend/src/services/documentText.js +++ b/backend/src/services/documentText.js @@ -19,7 +19,11 @@ const { extractPdfTextLines } = require('./pdfTextExtract'); const SEARCHABLE_EXTENSIONS = [...OFFICE_EXTENSIONS, 'pdf']; -const extensionOf = (filePath) => path.extname(filePath || '').slice(1).toLowerCase(); +const extensionOf = (filePath) => + path + .extname(filePath || '') + .slice(1) + .toLowerCase(); /** Whether this is a document whose text has to be extracted to be searched. */ const isSearchableDocument = (filePath) => SEARCHABLE_EXTENSIONS.includes(extensionOf(filePath)); diff --git a/backend/src/services/favoritesService.js b/backend/src/services/favoritesService.js index 26a702b4a..cfef15666 100644 --- a/backend/src/services/favoritesService.js +++ b/backend/src/services/favoritesService.js @@ -1,17 +1,13 @@ const fs = require('fs/promises'); -const crypto = require('crypto'); -const { getDb } = require('./db'); +const { getDb, prepared } = require('./db'); +const { listKnownVolumeNames, volumeOf } = require('./orphanedBindingsService'); const { normalizeRelativePath } = require('../utils/pathUtils'); const { resolvePathWithAccess } = require('./accessManager'); const config = require('../config'); +const { generateId } = require('../utils/ids'); const DEFAULT_FAVORITE_ICON = config.favorites.defaultIcon; -const generateId = () => - typeof crypto.randomUUID === 'function' - ? crypto.randomUUID() - : `${Date.now().toString(36)}-${crypto.randomBytes(8).toString('hex')}`; - /** * Validate and sanitize a favorite */ @@ -96,7 +92,19 @@ const validatePath = async (relativePath, user) => { throw err; } - const stats = await fs.stat(resolved.absolutePath); + // Resolving a path does not require it to exist. Without this, bookmarking a + // folder that has since been deleted answered 500 instead of saying so. + let stats; + try { + stats = await fs.stat(resolved.absolutePath); + } catch (error) { + if (error?.code === 'ENOENT') { + const err = new Error('Path not found'); + err.status = 404; + throw err; + } + throw error; + } if (!stats.isDirectory()) { const err = new Error('Path must be a directory'); @@ -123,7 +131,19 @@ const getFavorites = async (userId) => { ) .all(userId); - return favorites.map(mapDbFavorite); + const mapped = favorites.map(mapDbFavorite); + + // A favourite whose volume is no longer mounted is still a favourite: it is + // marked, not hidden and not removed, because the volume may well come back. + // Where the volume list cannot be established, nothing is marked — saying + // "unavailable" about everything would be worse than saying nothing. + const known = await listKnownVolumeNames(); + if (!known) return mapped; + + return mapped.map((favorite) => { + const volume = volumeOf(favorite.path); + return volume && !known.has(volume) ? { ...favorite, available: false } : favorite; + }); }; /** @@ -147,7 +167,8 @@ const addFavorite = async (userOrId, { path, label, icon, color }) => { const id = generateId(); const position = getNextFavoritePosition(db, userId); - db.prepare( + prepared( + db, ` INSERT INTO favorites (id, user_id, path, label, icon, color, created_at, updated_at, position) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -191,7 +212,8 @@ const removeFavorite = async (userId, path) => { const normalizedPath = normalizeRelativePath(path); const db = await getDb(); - db.prepare( + prepared( + db, ` DELETE FROM favorites WHERE user_id = ? AND path = ? ` @@ -201,46 +223,6 @@ const removeFavorite = async (userId, path) => { return getFavorites(userId); }; -const escapeLikePattern = (value = '') => String(value).replace(/[\\%_]/g, '\\$&'); - -/** - * Remove favorites that point to a deleted path. - * For directories, nested favorites are also removed. - */ -const removeFavoritesForDeletedPath = async (userId, path, { includeChildren = false } = {}) => { - ensureUserId(userId); - - const normalizedPath = normalizeRelativePath(path); - if (!normalizedPath) { - return 0; - } - - const db = await getDb(); - if (!includeChildren) { - const result = db - .prepare( - ` - DELETE FROM favorites - WHERE user_id = ? AND path = ? - ` - ) - .run(userId, normalizedPath); - return result.changes; - } - - const result = db - .prepare( - ` - DELETE FROM favorites - WHERE user_id = ? - AND (path = ? OR path LIKE ? ESCAPE '\\') - ` - ) - .run(userId, normalizedPath, `${escapeLikePattern(normalizedPath)}/%`); - - return result.changes; -}; - /** * Update a favorite's label or icon */ @@ -370,11 +352,14 @@ const reorderFavorites = async (userId, orderedIds) => { throw err; } - const updatePosition = db.prepare(` + const updatePosition = prepared( + db, + ` UPDATE favorites SET position = ? WHERE user_id = ? AND id = ? - `); + ` + ); const transact = db.transaction((ids) => { ids.forEach((id, index) => { @@ -402,7 +387,6 @@ module.exports = { getFavorites, addFavorite, removeFavorite, - removeFavoritesForDeletedPath, updateFavorite, reorderFavorites, }; diff --git a/backend/src/services/fileTransferService.js b/backend/src/services/fileTransferService.js index 26bde480a..881b6329e 100644 --- a/backend/src/services/fileTransferService.js +++ b/backend/src/services/fileTransferService.js @@ -1,9 +1,13 @@ -const crypto = require('crypto'); const path = require('path'); +const crypto = require('crypto'); const fs = require('fs/promises'); +const fsSync = require('fs'); +const os = require('os'); const { spawn } = require('child_process'); +const env = require('../config/env'); const { ensureDir, pathExists } = require('../utils/fsUtils'); +const logger = require('../utils/logger'); const { assertNotTopLevelEntry, isTopLevelEntry, @@ -12,130 +16,585 @@ const { ensureValidName, } = require('../utils/pathUtils'); const { placeWithoutOverwrite } = require('../utils/placeWithoutOverwrite'); -const { ACTIONS, authorizeAndResolve, authorizePath } = require('./authorizationService'); -const { getSharesForSourceTargets, deleteSharesByIds } = require('./sharesService'); const { track: trackInFlight } = require('./inFlightFiles'); +const { ValidationError, ForbiddenError, NotFoundError } = require('../errors/AppError'); +const { ACTIONS, authorizeAndResolve, authorizePath } = require('./authorizationService'); +const { + getSharesForSourceTargets, + getSharesBySourceTarget, + shareTargetKey, + deleteSharesByIds, +} = require('./sharesService'); +const pathBindings = require('./pathBindingsService'); +const folderSizeHooks = require('./folderSizeHooks'); +const config = require('../config/index'); +const { getIndexDb } = require('./indexDb'); +const folderSizeIndex = require('./folderSizeIndex'); +const { getVolumeScope } = require('./folderSizeIndexer'); +const { scheduleThumbnailRemoval } = require('./thumbnailService'); const trash = require('./trash'); const { getTrashSettings } = require('./trash/settings'); -const favoritesService = require('./favoritesService'); +// How often (ms) progress is reported to the caller while bytes stream, so a +// large file emits a steady trickle of updates rather than one per chunk. +const PROGRESS_THROTTLE_MS = 75; +// Node defaults file streams to 64 KiB buffers. That makes a 90 GiB transfer +// cross JavaScript over 1.4 million times. Keep the transfer cancellable, but +// use a bounded 4 MiB buffer to cut that overhead drastically without growing +// memory with the size of the copy. +const COPY_STREAM_HIGH_WATER_MARK = 4 * 1024 * 1024; /** - * Which engine copies a folder, asked at the moment it matters. + * Which engine moves and removes files, asked at the moment it matters. * - * Copying a tree in JavaScript walks it one entry at a time on the only thread - * the server has: a folder of a hundred thousand files is a hundred thousand - * trips through the event loop, and everything else the server was doing waits - * its turn behind them. `rsync` does the same work in one process, off that - * thread entirely. + * There are two implementations of every transfer — `rsync` and `rm` on one + * side, streams and `fs.rm` on the other — and the choice used to be frozen + * into a constant when the module loaded, from the platform the process + * happened to be running on. Each half was then only ever exercised where it + * was chosen: the native path is unreachable on a developer's macOS machine, + * and the JavaScript path is unreachable on the Linux that CI runs. Nobody ran + * both, and no test named the setting at all — `FILE_TRANSFER_ENGINE` appeared + * exactly once in the repository, in the line above. * - * It is a question rather than a constant, because a choice frozen at load time - * from the platform is a choice no test can reach: the native path would never - * run on a developer's machine, and the JavaScript path would never run in the - * container. `FILE_TRANSFER_ENGINE=native` or `=stream` names either one, and - * the default is unchanged — native where the image runs, JavaScript elsewhere. + * So it is a question now rather than a constant, and `native` is accepted as + * well as `stream`. The default is unchanged — native on Linux, streams + * elsewhere — and naming either one explicitly makes both reachable from a + * test, wherever the test is running. */ -const nativeCopyEnabled = () => { +const nativeTransferEnabled = () => { const configured = process.env.FILE_TRANSFER_ENGINE; if (configured === 'stream') return false; if (configured === 'native') return true; return process.platform === 'linux'; }; +const activeNativeOperations = new Map(); +const activeWriteOperations = new Map(); +let nextNativeOperationId = 1; +let nextWriteOperationId = 1; -/** - * Whether rsync is installed, asked once per PATH. - * - * Asked before anything is written, so that an image without it copies in - * JavaScript from the start rather than discovering it halfway through a tree. - */ -let rsyncLookup = null; -const rsyncAvailable = () => { - if (rsyncLookup && rsyncLookup.path === process.env.PATH) return rsyncLookup.answer; - const answer = new Promise((resolve) => { - const child = spawn('rsync', ['--version'], { stdio: 'ignore' }); - child.on('error', () => resolve(false)); - child.on('close', (code) => resolve(code === 0)); +const isPathWithin = (candidatePath, parentPath) => + candidatePath === parentPath || candidatePath.startsWith(`${parentPath}${path.sep}`); + +// Coordinate mutation requests with an in-flight write. A copy fills a hidden +// entry inside the destination folder, and lands under its name once whole; +// deleting that folder, or the landed entry, must stop and reap the writer +// first, otherwise the child process keeps writing into a path that no longer +// exists. `displayName` is what diagnostics show while the path is hidden. +const registerWriteOperation = ( + sourcePath, + destinationPath, + parentSignal, + displayName = path.basename(destinationPath) +) => { + const id = nextWriteOperationId; + nextWriteOperationId += 1; + const controller = new AbortController(); + let complete; + const completion = new Promise((resolve) => { + complete = resolve; }); - rsyncLookup = { path: process.env.PATH, answer }; - return answer; + const abortFromParent = () => controller.abort(); + + if (parentSignal?.aborted) abortFromParent(); + else parentSignal?.addEventListener('abort', abortFromParent, { once: true }); + + const operation = { + id, + sourcePath, + destinationPath, + sourceName: path.basename(sourcePath), + destinationName: displayName, + startedAt: Date.now(), + cancel: () => controller.abort(), + completion, + }; + activeWriteOperations.set(id, operation); + + let finished = false; + return { + signal: controller.signal, + /** The entry has landed under a name: a deletion of that name now waits for it. */ + retarget: (landedPath) => { + operation.destinationPath = landedPath; + operation.destinationName = path.basename(landedPath); + }, + finish: () => { + if (finished) return; + finished = true; + parentSignal?.removeEventListener('abort', abortFromParent); + activeWriteOperations.delete(id); + complete(); + }, + }; +}; + +const cancelWritesTargeting = async (absolutePath) => { + const operations = Array.from(activeWriteOperations.values()).filter((operation) => + isPathWithin(operation.destinationPath, absolutePath) + ); + if (operations.length === 0) return; + + operations.forEach((operation) => operation.cancel()); + await Promise.all(operations.map((operation) => operation.completion)); +}; + +const registerNativeOperation = (type, child, sourcePath, destinationPath = null) => { + const id = nextNativeOperationId; + nextNativeOperationId += 1; + activeNativeOperations.set(id, { + id, + type, + pid: child?.pid || null, + sourceName: path.basename(sourcePath), + ...(destinationPath ? { destinationName: path.basename(destinationPath) } : {}), + startedAt: Date.now(), + }); + return id; +}; + +const unregisterNativeOperation = (id) => { + if (id != null) activeNativeOperations.delete(id); +}; + +const getDiagnosticsSnapshot = () => { + const now = Date.now(); + return { + nativeTransferEnabled: nativeTransferEnabled(), + activeNativeOperations: Array.from(activeNativeOperations.values()) + .map((operation) => ({ ...operation, ageMs: now - operation.startedAt })) + .sort((a, b) => b.ageMs - a.ageMs) + .slice(0, 5), + activeWriteOperations: Array.from(activeWriteOperations.values()) + .map((operation) => ({ + id: operation.id, + sourceName: operation.sourceName, + destinationName: operation.destinationName, + ageMs: now - operation.startedAt, + })) + .sort((a, b) => b.ageMs - a.ageMs) + .slice(0, 5), + }; +}; + +const createCancellationError = () => { + const error = new Error('Operation cancelled.'); + error.code = 'OPERATION_CANCELLED'; + return error; +}; + +const throwIfCancelled = (signal) => { + if (signal?.aborted) throw createCancellationError(); +}; + +const getFolderSizeLookup = async () => { + if (!config.folderSize.enabled) return null; + try { + return { db: await getIndexDb(), scope: getVolumeScope() }; + } catch (_) { + // Folder-size indexing is optional. A transfer must never depend on it. + return null; + } +}; + +const indexedDirectorySize = (lookup, absolutePath) => { + if (!lookup || !folderSizeIndex.isWithinRoot(lookup.scope.root, absolutePath)) return null; + const entry = folderSizeIndex.getByAbsolutePath(lookup.db, absolutePath); + return Number.isFinite(entry?.sizeBytes) ? entry.sizeBytes : null; +}; + +const parseRsyncProgress = (line) => { + const match = line.match(/^\s*([\d,]+)\s+(\d+)%/); + if (!match) return null; + return { + copiedBytes: Number(match[1].replaceAll(',', '')) || 0, + percent: Math.min(100, Number(match[2]) || 0), + }; }; +const stopChildProcessGroup = (child, signal) => { + if (!child?.pid) return; + try { + process.kill(-child.pid, signal); + } catch (_) { + child.kill(signal); + } +}; + +// rsync keeps file transfer outside the Node event loop while retaining three +// properties the UI needs: safe argv handling, global progress, and immediate +// cancellation. It is used only in the Linux container; local development and +// the explicit FILE_TRANSFER_ENGINE=stream override keep the JS fallback. /** - * Copy a folder with rsync. - * - * `-rlt` and not `-a`: the recursion, the symbolic links and the times are what - * the JavaScript path gives, and asking for the permissions as well makes rsync - * fail outright on a filesystem that refuses to set them — an SMB or FUSE - * mount, where the copy used to succeed. Owner and group are left to the - * destination for the same reason. + * rsync stops at 23 for a "partial transfer due to error", which covers a great + * deal more than permissions — a vanished source file gets the same code. The + * message is what distinguishes the case worth retrying, so both are required. */ -const runRsyncCopy = (sourcePath, destinationPath) => +const isPermissionPreservationFailure = (exitCode, stderr) => + exitCode === 23 && /failed to set permissions/i.test(stderr || ''); + +const runRsyncCopy = ( + sourcePath, + destinationPath, + onProgress, + signal, + { preservePermissions, inPlace = false } +) => new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(createCancellationError()); + return; + } + const child = spawn( 'rsync', - ['-rlt', '--no-perms', '--no-owner', '--no-group', '--', `${sourcePath}/`, destinationPath], - { env: { ...process.env, LC_ALL: 'C' }, stdio: ['ignore', 'ignore', 'pipe'] } + [ + '-a', + '--no-owner', + '--no-group', + ...(preservePermissions ? [] : ['--no-perms']), + // Written at the path it is given rather than under a temporary name of + // rsync's own: that path is hidden and recorded, the temporary is not. + ...(inPlace ? ['--inplace'] : []), + '--info=progress2', + '--outbuf=L', + '--out-format=%n', + '--', + sourcePath, + destinationPath, + ], + { + detached: true, + env: { ...process.env, LC_ALL: 'C' }, + stdio: ['ignore', 'pipe', 'pipe'], + } ); + const operationId = registerNativeOperation('rsync', child, sourcePath, destinationPath); + let output = ''; let errorOutput = ''; + let settled = false; + let killTimer = null; + const cleanup = () => { + signal?.removeEventListener('abort', abort); + if (killTimer) clearTimeout(killTimer); + unregisterNativeOperation(operationId); + }; + const finish = (callback, value) => { + if (settled) return; + settled = true; + cleanup(); + callback(value); + }; + const emitOutput = (chunk) => { + output += chunk.toString(); + const lines = output.split(/[\r\n]/); + output = lines.pop() || ''; + for (const line of lines) { + const progress = parseRsyncProgress(line); + if (progress) onProgress?.(progress); + } + }; + const abort = () => { + stopChildProcessGroup(child, 'SIGTERM'); + killTimer = setTimeout(() => stopChildProcessGroup(child, 'SIGKILL'), 3000); + }; + + child.stdout.on('data', emitOutput); child.stderr.on('data', (chunk) => { errorOutput += chunk.toString(); }); - child.on('error', reject); - child.on('close', (code) => { - if (code === 0) resolve(); - else reject(new Error(`rsync failed (${code}): ${errorOutput.trim().slice(0, 500)}`)); + child.once('error', (error) => finish(reject, error)); + child.once('close', (code) => { + if (signal?.aborted) return finish(reject, createCancellationError()); + if (code === 0) return finish(resolve); + const error = new Error(errorOutput.trim() || `Native copy failed with exit code ${code}.`); + error.code = 'NATIVE_COPY_FAILED'; + error.exitCode = code; + error.stderr = errorOutput; + return finish(reject, error); }); + signal?.addEventListener('abort', abort, { once: true }); }); -const copyEntry = async (sourcePath, destinationPath, isDirectory) => { - if (isDirectory) { - if (nativeCopyEnabled() && (await rsyncAvailable())) { - await ensureDir(destinationPath); - await runRsyncCopy(sourcePath, destinationPath); +/** + * Copy with rsync, preserving permissions — and once more without them if that + * is the only thing that failed. + * + * `-a` implies `-p`, so rsync chmods the destination after writing it. A ZFS + * dataset with `aclmode=restricted` refuses that chmod, because new files there + * must inherit the directory's ACL untouched (nxzai/NextExplorer#367). rsync + * copies the contents correctly and only then fails, so the data is already + * where it belongs and just the metadata step was refused. + * + * The retry is safe because rsync is idempotent: everything transferred in the + * first pass is seen as up to date in the second, which therefore moves no + * bytes and only finishes what the first could not. Preserving permissions + * remains the default — the fallback happens where it cannot work, and nowhere + * else. + * + * The retry reports no progress: percentages restart at zero for each rsync + * invocation, and the caller turns them into an absolute byte count, so passing + * them on would send the bar backwards for the moment the second pass takes. + */ +const copyWithNativeRsync = async ( + sourcePath, + destinationPath, + onProgress, + signal, + { inPlace = false } = {} +) => { + // Where the answer is known in advance, skip the attempt that cannot succeed: + // on a dataset that always refuses, every copy would otherwise pay for a + // failed pass and a retry. + if (!env.COPY_PRESERVE_PERMISSIONS) { + return runRsyncCopy(sourcePath, destinationPath, onProgress, signal, { + inPlace, + preservePermissions: false, + }); + } + + try { + return await runRsyncCopy(sourcePath, destinationPath, onProgress, signal, { + inPlace, + preservePermissions: true, + }); + } catch (error) { + if (!isPermissionPreservationFailure(error?.exitCode, error?.stderr)) throw error; + if (signal?.aborted) throw error; + + logger.info( + { sourcePath, destinationPath }, + 'Destination refuses to have its permissions set; copying again without preserving them' + ); + return runRsyncCopy(sourcePath, destinationPath, undefined, signal, { + inPlace, + preservePermissions: false, + }); + } +}; + +/** + * Whether removing this entry is worth a child process. + * + * `rm -rf` earns its fork on a directory: the recursion happens in one native + * call, and killing the process cancels it. A single file has neither — the + * unlink is one syscall — so forking per file costs about 1.2 ms of process + * setup against 0.06 ms of actual work. On a selection of two thousand files + * that is over two seconds spent starting processes, and it only happens on + * Linux, which is to say only in the container. + */ +/** + * Whether the native tool could not be used at all, as opposed to having tried + * and failed partway. + * + * The distinction is the whole point. A copy that fails midway has already + * written something, and falling back would resume over a half-written tree. + * These two failures happen before anything is written: the binary is not + * there, or it is too old to understand what it was asked for. + * + * That second one is not hypothetical. `--info=progress2` arrived in rsync 3.1, + * and RHEL 7 ships 3.0.9 while macOS ships 2.6.9 — on either, every copy failed + * with a raw rsync usage error, while a working implementation in this same + * file went unused. The setting documented for exactly this case + * (`FILE_TRANSFER_ENGINE=stream`) only helped someone who already knew to reach + * for it, after their copies had failed. + */ +const nativeToolIsUnusable = (error) => { + if (error?.code === 'ENOENT') return true; + const stderr = String(error?.stderr || error?.message || ''); + return /unrecognized option|unknown option|invalid option|illegal option/i.test(stderr); +}; + +/** + * Set once a native tool has proved unusable, so the rest of the process stops + * paying for an attempt whose answer is already known. + */ +const unusableNativeTools = new Set(); + +const recordUnusableNativeTool = (tool, error) => { + if (unusableNativeTools.has(tool)) return; + unusableNativeTools.add(tool); + logger.warn( + { + tool, + reason: String(error?.stderr || error?.message || '') + .trim() + .slice(0, 200), + }, + `${tool} cannot be used here; falling back to the in-application implementation for the life of this process` + ); +}; + +const nativeToolUsable = (tool) => !unusableNativeTools.has(tool); + +const shouldRemoveNatively = (isDirectoryEntry, nativeEnabled = nativeTransferEnabled()) => + Boolean(nativeEnabled) && Boolean(isDirectoryEntry); + +const removeWithNativeRm = (absolutePath, signal) => + new Promise((resolve, reject) => { + if (signal?.aborted) return reject(createCancellationError()); + const child = spawn('rm', ['-rf', '--', absolutePath], { detached: true, stdio: 'ignore' }); + const operationId = registerNativeOperation('rm', child, absolutePath); + let settled = false; + let killTimer = null; + const cleanup = () => { + signal?.removeEventListener('abort', abort); + if (killTimer) clearTimeout(killTimer); + unregisterNativeOperation(operationId); + }; + const finish = (callback, value) => { + if (settled) return; + settled = true; + cleanup(); + callback(value); + }; + const abort = () => { + stopChildProcessGroup(child, 'SIGTERM'); + killTimer = setTimeout(() => stopChildProcessGroup(child, 'SIGKILL'), 3000); + }; + child.once('error', (error) => finish(reject, error)); + child.once('close', (code) => { + if (signal?.aborted) return finish(reject, createCancellationError()); + if (code === 0) return finish(resolve); + const error = new Error(`Native deletion failed with exit code ${code}.`); + error.code = 'NATIVE_DELETE_FAILED'; + return finish(reject, error); + }); + signal?.addEventListener('abort', abort, { once: true }); + }); + +/** + * Where a copy is written until it is whole: a hidden entry beside where it + * goes, under a name nobody else knows. + * + * Nothing is visible under the entry's name while it is written, so nobody can + * put a file into a folder being filled, have it replaced by the copy, or have + * it removed with the copy when the copy is cancelled. The name has a fixed + * length, never derived from the file's: a long file name with a suffix + * appended could pass the filesystem's limit. + */ +const stagingPathIn = (directory) => + path.join(directory, `.nextexplorer-copying-${crypto.randomUUID()}`); + +/** Stream `sourcePath` into an open `handle`, settled once the handle is closed. */ +const streamInto = (sourcePath, handle, onBytes, signal) => + new Promise((resolve, reject) => { + if (signal?.aborted) { + handle.close().then( + () => reject(createCancellationError()), + () => reject(createCancellationError()) + ); return; } - if (typeof fs.cp === 'function') { - await fs.cp(sourcePath, destinationPath, { - recursive: true, - force: false, - errorOnExist: true, - // A relative link inside the tree was resolved and written out as an - // absolute path into the *source* tree: the copy then pointed back at - // the original, and lost its way entirely once that was moved or - // deleted. Kept verbatim, a link says what it said. - verbatimSymlinks: true, - }); - } else { - await ensureDir(destinationPath); - const entries = await fs.readdir(sourcePath, { withFileTypes: true }); - for (const entry of entries) { - const src = path.join(sourcePath, entry.name); - const dest = path.join(destinationPath, entry.name); - // eslint-disable-next-line no-await-in-loop - await copyEntry(src, dest, entry.isDirectory()); - } + + const readStream = fsSync.createReadStream(sourcePath, { + highWaterMark: COPY_STREAM_HIGH_WATER_MARK, + }); + const writeStream = fsSync.createWriteStream(null, { + fd: handle, + highWaterMark: COPY_STREAM_HIGH_WATER_MARK, + }); + + let failure = null; + const cleanup = () => signal?.removeEventListener('abort', abort); + const fail = (error) => { + if (failure || writeStream.writableFinished) return; + failure = error; + readStream.destroy(); + writeStream.destroy(); + }; + const abort = () => fail(createCancellationError()); + + readStream.on('error', fail); + writeStream.on('error', fail); + if (typeof onBytes === 'function') { + readStream.on('data', (chunk) => onBytes(chunk.length)); } - } else { - await fs.copyFile(sourcePath, destinationPath); - } -}; + writeStream.once('close', () => { + cleanup(); + if (failure) reject(failure); + else if (writeStream.writableFinished) resolve(); + else reject(createCancellationError()); + }); + signal?.addEventListener('abort', abort, { once: true }); + readStream.pipe(writeStream); + }); /** - * Copy an entry recursively, reporting the bytes copied so far and stopping when - * the signal aborts. A symbolic link is copied as a link, keeping its text; a - * file is copied and its size reported; a folder is created and its entries - * copied in turn. Answers the total bytes copied. Used by the trash to restore - * across disks with real progress. + * Copy a single regular file through streams so bytes can be reported as they + * are written. The source mode is applied at creation to mirror fs.copyFile. + * + * The file is created exclusively, so a copy never truncates a file it did not + * create: its callers hand it a fresh hidden path, and a path that is somehow + * taken fails the copy rather than being written into. */ +const copyFileWithProgress = async (sourcePath, destinationPath, mode, onBytes, signal) => { + throwIfCancelled(signal); + const handle = await fs.open(destinationPath, 'wx', mode); + try { + await streamInto(sourcePath, handle, onBytes, signal); + } catch (error) { + // The file this copy created exclusively, and nothing else. + await fs.unlink(destinationPath).catch(() => {}); + throw error; + } + + // When it was last written, kept. A copy that stamps everything with the + // moment it ran turns a folder of photographs sorted by date into a folder + // all dated today, and there is no getting the dates back. `fs.cp` and rsync + // both keep them; the path that reports progress reads a file itself, so it + // has to put them back itself. Best-effort: a filesystem that will not take + // them is not a reason to fail a copy that has already landed. + try { + const { atime, mtime } = await fs.lstat(sourcePath); + await fs.utimes(destinationPath, atime, mtime); + } catch (error) { + logger.debug({ err: error, destinationPath }, 'Could not carry the times over to the copy'); + } +}; + +/** Make a symbolic link at `destinationPath`, pointing where the source's points. */ +const copySymbolicLink = async (sourcePath, destinationPath) => { + await fs.symlink(await fs.readlink(sourcePath), destinationPath); +}; + +// Recursively copy a file/dir, reporting copied bytes. It returns the actual +// copied byte count, so folder-size updates never need a second filesystem walk. const copyEntryWithProgress = async (sourcePath, destinationPath, isDirectory, onBytes, signal) => { - if (signal?.aborted) throw createCancellationError(); - const stats = await fs.lstat(sourcePath); - if (stats.isSymbolicLink()) { - await fs.symlink(await fs.readlink(sourcePath), destinationPath); - return 0; + throwIfCancelled(signal); + if (nativeTransferEnabled() && nativeToolUsable('rsync')) { + const stats = await fs.lstat(sourcePath); + try { + if (stats.isDirectory()) { + // rsync copies the directory itself when the source lacks a trailing + // slash; the contract is to copy its contents into the target directory + // instead. A file, or a link, is written by rsync at the target path + // itself, which its callers hand over fresh. + await ensureDir(destinationPath); + await copyWithNativeRsync( + `${sourcePath}${path.sep}`, + `${destinationPath}${path.sep}`, + onBytes, + signal + ); + } else { + // A single file goes to a fresh hidden path: written there directly, so a + // stop leaves nothing but what the in-flight journal names. + await copyWithNativeRsync(sourcePath, destinationPath, onBytes, signal, { inPlace: true }); + } + return stats.isDirectory() ? null : stats.size; + } catch (error) { + // Only when nothing was written. Anything else is a real failure and the + // caller is the one who should hear about it. + if (signal?.aborted || !nativeToolIsUnusable(error)) throw error; + recordUnusableNativeTool('rsync', error); + } } - if (!stats.isDirectory() && !isDirectory) { - await fs.copyFile(sourcePath, destinationPath); - onBytes?.(stats.size); + if (!isDirectory) { + const stats = await fs.lstat(sourcePath); + if (stats.isSymbolicLink()) { + await copySymbolicLink(sourcePath, destinationPath); + return 0; + } + await copyFileWithProgress(sourcePath, destinationPath, stats.mode, onBytes, signal); return stats.size; } @@ -143,71 +602,137 @@ const copyEntryWithProgress = async (sourcePath, destinationPath, isDirectory, o const entries = await fs.readdir(sourcePath, { withFileTypes: true }); let copiedBytes = 0; for (const entry of entries) { - if (signal?.aborted) throw createCancellationError(); + throwIfCancelled(signal); const src = path.join(sourcePath, entry.name); const dest = path.join(destinationPath, entry.name); - // eslint-disable-next-line no-await-in-loop copiedBytes += await copyEntryWithProgress(src, dest, entry.isDirectory(), onBytes, signal); } return copiedBytes; }; -/* - * A copy or a move looked for a free name, "note (1).txt", and wrote under it - * afterwards. Whatever arrived under that name in between — another copy, a - * file saved over SMB — was replaced by the copied file or by the rename of a - * move, or poured into by a copied folder, and a copy lasting minutes held that - * gap open for minutes. The name is now taken by the step that puts the entry - * there, through placeWithoutOverwrite, which never replaces nor merges into - * anything and moves on to "note (1).txt" when the name is held. - */ - /** - * Copy `sourcePath` into a hidden entry of its own beside the destination, and - * put it under `desiredName`, or the first free name after it, once whole. A - * copy that fails removes only that hidden entry; one cut short by a stop is - * removed at the next start, through the in-flight journal. + * Copy an entry into `stagingPath`, hidden beside where it goes, and once it is + * whole put it under `desiredName` in `directory`, or the first free name after + * it. Answers the size copied and the name and path it took. + * + * Nothing holds the entry's name while it is written: the placement takes the + * name by an operation that fails when it is held — a link for a file, a new + * folder renamed over for a folder — so whatever arrived under it meanwhile is + * kept, and the copy takes the next name. + * + * The hidden entry is recorded before it is created, so a stop half-way leaves + * a record the next start removes it by. However the copy fails or is + * cancelled, that hidden entry is removed, and only that: nobody else knows its + * name, and nothing under a visible name is ever touched here. */ -const copyIntoPlace = async (sourcePath, directory, desiredName, isDirectory) => { - const stagingPath = path.join(directory, `.nextexplorer-copy-${crypto.randomUUID()}`); - const inFlight = trackInFlight(stagingPath, 'partial-copy'); +const copyIntoPlace = async ({ + sourcePath, + stagingPath, + directory, + desiredName, + entryIsDirectory, + holdsFolderSize, + onBytes, + signal, +}) => { + throwIfCancelled(signal); + const inFlight = trackInFlight(stagingPath, 'staging-copy'); + const hold = holdsFolderSize ? folderSizeHooks.holdHiddenDirectory(stagingPath) : null; try { - await copyEntry(sourcePath, stagingPath, isDirectory); - return await placeWithoutOverwrite(stagingPath, directory, desiredName); + const size = await copyEntryWithProgress( + sourcePath, + stagingPath, + entryIsDirectory, + onBytes, + signal + ); + throwIfCancelled(signal); + const placed = await placeWithoutOverwrite(stagingPath, directory, desiredName); + return { size, placed }; } catch (error) { - await fs.rm(stagingPath, { recursive: true, force: true }); + await fs.rm(stagingPath, { recursive: true, force: true }).catch(() => {}); throw error; } finally { + hold?.release(); inFlight.release(); } }; /** - * Move `sourcePath` under `desiredName` in `directory`, or the first free name - * after it. To another disk, the entry is copied whole first, and the source - * removed only once that copy is in place. + * Move an entry under `desiredName` in `directory`, or the first free name + * after it, reporting progress, and answer the size moved and the name and path + * it took. + * + * On one filesystem the entry is placed directly, never replacing what holds a + * name: a file is linked under it and its old name removed, a folder renamed + * over a new empty folder that refuses the rename once something is put inside. + * It is instant, so the whole size is reported at once. Across devices (EXDEV) + * the move becomes a copy under a hidden name, placed once whole, and only then + * is the source removed. `onLanded` hears of the name as soon as the entry is + * whole under it: from then on a failure, or a cancellation while the source is + * removed, must never take that copy away, since it may be the only whole one. */ -const moveIntoPlace = async (sourcePath, directory, desiredName, isDirectory) => { +const moveIntoPlace = async ({ + sourcePath, + stagingPath, + directory, + desiredName, + entryIsDirectory, + holdsFolderSize, + size, + onBytes, + signal, + onLanded, +}) => { + throwIfCancelled(signal); + let placed = null; try { - return await placeWithoutOverwrite(sourcePath, directory, desiredName); + placed = await placeWithoutOverwrite(sourcePath, directory, desiredName); } catch (error) { - if (error.code !== 'EXDEV') throw error; + if (error?.code !== 'EXDEV') throw error; } - const placed = await copyIntoPlace(sourcePath, directory, desiredName, isDirectory); - await fs.rm(sourcePath, { recursive: isDirectory, force: true }); - return placed; + if (placed) { + await onLanded?.(placed); + if (typeof onBytes === 'function' && size > 0) onBytes(size); + return { size, placed }; + } + + const copied = await copyIntoPlace({ + sourcePath, + stagingPath, + directory, + desiredName, + entryIsDirectory, + holdsFolderSize, + onBytes, + signal, + }); + await onLanded?.(copied.placed); + if (shouldRemoveNatively(entryIsDirectory)) await removeWithNativeRm(sourcePath, signal); + else await fs.rm(sourcePath, { recursive: entryIsDirectory, force: true }); + return copied; }; -const transferItems = async (items, destination, operation, options = {}) => { +// Phase 1: authorize + resolve every item. Recursive directory-size walks are +// deliberately avoided here: a large copy used to read every source file once +// for progress, then read it all again to copy. Indexed directory sizes give a +// determinate bar in O(1); otherwise the UI uses its indeterminate state while +// the copy starts immediately. +const prepareTransfer = async (items, destination, operation, options = {}) => { + const { signal } = options; + throwIfCancelled(signal); if (!Array.isArray(items) || items.length === 0) { - throw new Error('At least one item is required.'); + // The shape of the request, not the state of the server: a caller that + // sends nothing to move is told so, rather than being answered 500 and + // logged as an unexpected failure that says the server broke. + throw new ValidationError('At least one item is required.'); } const destinationRelative = normalizeRelativePath(destination); // Prevent copying/moving items directly to the root path if (!destinationRelative || destinationRelative.trim() === '') { - throw new Error( + throw new ValidationError( 'Cannot copy or move items to the root path. Please select a specific volume or folder first.' ); } @@ -221,18 +746,25 @@ const transferItems = async (items, destination, operation, options = {}) => { allowed: destAllowed, accessInfo: destAccess, resolved: destResolved, - } = await authorizeAndResolve(context, destinationRelative, ACTIONS.write); + } = await authorizeAndResolve(context, destinationRelative, ACTIONS.read); if (!destAllowed || !destResolved) { - throw new Error(destAccess?.denialReason || 'Destination path is not writable.'); + throw new ForbiddenError(destAccess?.denialReason || 'Destination path is not writable.'); } const { absolutePath: destinationAbsolute } = destResolved; + const folderSizeLookup = await getFolderSizeLookup(); - await ensureDir(destinationAbsolute); + const destinationStats = await fs.stat(destinationAbsolute).catch(() => null); + if (!destinationStats?.isDirectory()) { + throw new ValidationError('Destination path must be an existing directory.'); + } - const results = []; + const plans = []; + let totalBytes = 0; + let hasUnknownSize = false; for (const item of items) { + throwIfCancelled(signal); const sourceCombined = combineRelativePath(item.path || '', item.name); const { allowed: srcAllowed, @@ -240,13 +772,15 @@ const transferItems = async (items, destination, operation, options = {}) => { resolved: srcResolved, } = await authorizeAndResolve(context, sourceCombined, ACTIONS.read); if (!srcAllowed || !srcResolved) { - throw new Error(srcAccess?.denialReason || `Source path not accessible: ${sourceCombined}`); + throw new ForbiddenError( + srcAccess?.denialReason || `Source path not accessible: ${sourceCombined}` + ); } const { relativePath: sourceRelative, absolutePath: sourceAbsolute } = srcResolved; if (!(await pathExists(sourceAbsolute))) { - throw new Error(`Source path not found: ${sourceRelative}`); + throw new NotFoundError(`Source path not found: ${sourceRelative}`); } if (operation === 'move') { @@ -256,70 +790,283 @@ const transferItems = async (items, destination, operation, options = {}) => { ACTIONS.delete ); if (!deleteAllowed) { - throw new Error(deleteAccess?.denialReason || 'Cannot move items from this path.'); + throw new ForbiddenError(deleteAccess?.denialReason || 'Cannot move items from this path.'); } } const stats = await fs.stat(sourceAbsolute); - // What is carried cannot be one of the spaces themselves: a volume is a - // mount, and moving one out of the list is the same loss as deleting it - // (nxzai/NextExplorer#409). The destination was already refused above. - assertNotTopLevelEntry( - item.path || '', - operation === 'move' ? 'moved' : 'copied', - stats.isDirectory() - ); + const isDirectory = stats.isDirectory(); + // The destination cannot be the top level, and neither can what is carried + // from it: a volume is a mount, and moving one out of the list is the same + // loss as deleting it (nxzai/NextExplorer#409). + assertNotTopLevelEntry(item.path || '', operation === 'move' ? 'moved' : 'copied', isDirectory); const sourceParent = normalizeRelativePath(path.dirname(sourceRelative)); + if ( + isDirectory && + (destinationAbsolute === sourceAbsolute || + destinationAbsolute.startsWith(`${sourceAbsolute}${path.sep}`)) + ) { + throw new ValidationError('Cannot copy or move a folder into itself.'); + } + if (operation === 'move' && destinationRelative === sourceParent) { - results.push({ from: sourceRelative, to: sourceRelative, skipped: true }); + plans.push({ sourceRelative, skipped: true }); continue; } + const destinationAction = isDirectory ? ACTIONS.createFolder : ACTIONS.createFile; + const { allowed: createAllowed, accessInfo: createAccess } = await authorizePath( + context, + destinationRelative, + destinationAction + ); + if (!createAllowed) { + throw new ForbiddenError( + createAccess?.denialReason || 'Cannot create items in the destination path.' + ); + } + + // A copied directory may contain files as well as folders. Do not let the + // directory permission become a way around the file creation restriction. + if (isDirectory) { + const { allowed: filesAllowed, accessInfo: filesAccess } = await authorizePath( + context, + destinationRelative, + ACTIONS.createFile + ); + if (!filesAllowed) { + throw new ForbiddenError( + filesAccess?.denialReason || 'Cannot create files in the destination path.' + ); + } + } + // The name the item lands under is joined onto the destination, which is // the only directory authorized above. Taken from the request as it came, // `../x` or `../../x` wrote beside or above it — out of a read-only parent, - // out of a share into the volume. A new name has to be a name; without - // one, the item keeps the name it has on disk, not the one the request - // spelled. + // out of a share into the volume — and `.nextexplorer` planted a zone name. + // A new name has to be a name; without one, the item keeps the name it has + // on disk, not the one the request spelled. const desiredName = item.newName === undefined || item.newName === null || item.newName === '' ? path.basename(sourceAbsolute) : ensureValidName(item.newName); - let placed; - if (operation === 'copy') { - placed = await copyIntoPlace( - sourceAbsolute, - destinationAbsolute, - desiredName, - stats.isDirectory() + const size = isDirectory ? indexedDirectorySize(folderSizeLookup, sourceAbsolute) : stats.size; + if (Number.isFinite(size)) totalBytes += size; + else hasUnknownSize = true; + + plans.push({ + sourceAbsolute, + sourceRelative, + isDirectory, + size, + desiredName, + }); + } + + return { + destinationRelative, + destinationAbsolute, + plans, + totalBytes: hasUnknownSize ? 0 : totalBytes, + totalItems: plans.filter((plan) => !plan.skipped).length, + }; +}; + +// Phase 2: perform the copy/move for each prepared plan, reporting progress via +// onProgress({ copiedBytes, totalBytes, currentName }). Runs after the response +// has switched to streaming mode, so an error here is surfaced in the stream. +const executeTransfer = async (prep, operation, onProgress, options = {}) => { + const { destinationRelative, destinationAbsolute, plans, totalBytes } = prep; + const { signal } = options; + + throwIfCancelled(signal); + const destinationStats = await fs.stat(destinationAbsolute).catch(() => null); + if (!destinationStats?.isDirectory()) { + throw new NotFoundError('Destination path no longer exists.'); + } + + const results = []; + let copiedBytes = 0; + let lastEmit = 0; + let currentName = ''; + let nativePercent = null; + let activeTarget = null; + let activeWriteOperation = null; + const transferredDirectories = []; + + const emit = (force = false) => { + if (typeof onProgress !== 'function') return; + const now = Date.now(); + if (!force && now - lastEmit < PROGRESS_THROTTLE_MS) return; + lastEmit = now; + onProgress({ + copiedBytes, + totalBytes, + currentName, + ...(nativePercent != null ? { percent: nativePercent } : {}), + }); + }; + + const onBytes = (delta) => { + const wasAtStart = copiedBytes === 0; + copiedBytes += delta; + // Show the first byte immediately, then throttle the steady stream of + // updates. Besides making the UI feel responsive, this lets an operation + // become cancellable as soon as data starts moving. + emit(wasAtStart); + }; + + try { + for (const plan of plans) { + throwIfCancelled(signal); + if (plan.skipped) { + results.push({ from: plan.sourceRelative, to: plan.sourceRelative, skipped: true }); + continue; + } + + // What lands at the destination is the entry itself: a rename moves a + // link as a link, and both engines copy one as a link. + const entryIsDirectory = (await fs.lstat(plan.sourceAbsolute)).isDirectory(); + // Nothing is visible under the entry's name until it is whole there. A + // copy is written under a hidden name beside it and put in place once + // whole; a move is put in place directly. Either way the name is taken + // by an operation that fails when it is held, so whatever arrived under + // it meanwhile — another copy, a file saved over SMB — is kept, and the + // entry takes "name (1)". A name held from the start by a visible + // placeholder let others write into it while the copy ran. + const stagingPath = stagingPathIn(destinationAbsolute); + // `landedAt` once the entry is whole under its name: from then on + // nothing removes it. + const target = { isDirectory: plan.isDirectory, landedAt: null }; + activeTarget = target; + const writeOperation = registerWriteOperation( + plan.sourceAbsolute, + stagingPath, + signal, + plan.desiredName ); - } else if (operation === 'move') { - placed = await moveIntoPlace( - sourceAbsolute, - destinationAbsolute, - desiredName, - stats.isDirectory() + activeWriteOperation = writeOperation; + currentName = plan.desiredName; + nativePercent = null; + emit(true); + + const copiedBeforePlan = copiedBytes; + const onCopyProgress = (progress) => { + if (typeof progress === 'number') { + onBytes(progress); + return; + } + nativePercent = Number.isFinite(progress?.percent) ? progress.percent : null; + if (Number.isFinite(plan.size) && nativePercent != null) { + copiedBytes = copiedBeforePlan + (plan.size * nativePercent) / 100; + } + emit(true); + }; + + const land = async (placed) => { + target.landedAt = placed.path; + writeOperation.retarget(placed.path); + if (placed.name !== currentName) { + currentName = placed.name; + emit(true); + } + if (plan.isDirectory) { + // The index entry is made under the name the folder landed at, and + // held until the scan after the whole operation, so an on-view + // refresh cannot publish a size for it meanwhile. + await folderSizeHooks.beginDirectoryTransfer(placed.path); + } + }; + + const placement = { + sourcePath: plan.sourceAbsolute, + stagingPath, + directory: destinationAbsolute, + desiredName: plan.desiredName, + entryIsDirectory, + holdsFolderSize: plan.isDirectory, + onBytes: onCopyProgress, + signal: writeOperation.signal, + }; + + if (operation === 'copy') { + const copied = await copyIntoPlace(placement); + await land(copied.placed); + await folderSizeHooks.onEntryCopied(target.landedAt, { + isDirectory: plan.isDirectory, + size: copied.size ?? plan.size, + sourceAbsolutePath: plan.sourceAbsolute, + directoryTransferPrepared: plan.isDirectory, + }); + } else if (operation === 'move') { + const moved = await moveIntoPlace({ ...placement, size: plan.size, onLanded: land }); + await folderSizeHooks.onEntryMoved(plan.sourceAbsolute, target.landedAt, { + isDirectory: plan.isDirectory, + size: moved.size ?? plan.size, + directoryTransferPrepared: plan.isDirectory, + }); + } else { + throw new ValidationError(`Unsupported operation: ${operation}`); + } + + const targetAbsolute = target.landedAt; + const targetRelative = combineRelativePath( + destinationRelative, + path.basename(targetAbsolute) ); - // The history follows the file — or everything in the folder — to its new - // path; moved to another volume it names the new zone, its versions left - // where they were kept. A copy starts with no history, so this is the move - // only. - // eslint-disable-next-line global-require - await require('./versions/lifecycle').onMoved(sourceAbsolute, placed.path); - } else { - throw new Error(`Unsupported operation: ${operation}`); + if (plan.isDirectory) transferredDirectories.push(targetAbsolute); + + // A move takes the folder's bindings with it — favorites, shares, recent + // destinations, per-folder preferences. A copy leaves the original where + // it is, so its bindings stay put and the copy starts with none. + if (operation === 'move') { + await pathBindings.movePath(plan.sourceRelative, targetRelative); + // And the histories, whose versions stay where they were kept: even to + // another volume, nothing is copied for them. A copy starts with none. + await require('./versions/lifecycle').onMoved(plan.sourceAbsolute, targetAbsolute); + } + + results.push({ from: plan.sourceRelative, to: targetRelative }); + activeWriteOperation.finish(); + activeWriteOperation = null; + activeTarget = null; } - // The name actually taken: "note (1).txt" when "note.txt" was held. - results.push({ - from: sourceRelative, - to: combineRelativePath(destinationRelative, placed.name), - }); - } + // Snap to 100% once every entry is done when the total was known before + // starting. Unknown directory totals intentionally stay indeterminate. + if (totalBytes > 0) copiedBytes = totalBytes; + emit(true); + + // Rebuild copied/moved directory indexes only after the complete operation + // has finished writing. This avoids expensive disk scans competing with the + // transfer and makes the eventual size authoritative. + folderSizeHooks.refreshTransferredDirectories(transferredDirectories); - return { destination: destinationRelative, items: results }; + return { destination: destinationRelative, items: results }; + } catch (error) { + try { + // An entry that had not landed left nothing under a visible name: its + // hidden copy was removed where it failed or was cancelled, and it never + // had an index entry. One that had landed stays whole where it is, even + // when what followed failed or was cancelled — a move across disks may + // already have removed part of its source — so its index entry is + // released and scanned rather than left locked until the next restart. + if (activeTarget?.landedAt && activeTarget.isDirectory) { + folderSizeHooks.refreshTransferredDirectories([activeTarget.landedAt]); + } + // Completed entries remain after a cancellation and still need their final + // directory-size scan. + folderSizeHooks.refreshTransferredDirectories(transferredDirectories); + } finally { + // A deletion waiting on this write must always be released, even if one + // of the optional folder-size hooks fails during transfer cleanup. + activeWriteOperation?.finish(); + } + throw error; + } }; const getShareSourceTarget = (resolved, includeChildren = false) => { @@ -343,14 +1090,39 @@ const getShareSourceTarget = (resolved, includeChildren = false) => { }; }; -const resolveDeleteTargets = async (items = [], context) => { +/** + * Entries handled at once. + * + * Removals are independent and each one is mostly latency, not work: waiting + * for them one at a time leaves the storage idle in between. A bind-mounted + * volume measured ~3.7 ms per unlink where a native filesystem needs 0.06 ms, + * and that gap is exactly what overlapping recovers. + * + * Sixteen is a compromise: high enough to hide that latency, low enough not + * to bury a filesystem that answers quickly. BULK_DELETE_CONCURRENCY tunes it + * for storage that behaves differently. + */ +// Keep the default aligned with the CPU capacity available to the container. +// The frontend deliberately sends delete batches one at a time, so this is the +// real upper bound rather than one of several multiplicative limits. +const DEFAULT_DELETE_CONCURRENCY = Math.max( + 1, + typeof os.availableParallelism === 'function' ? os.availableParallelism() : os.cpus().length +); +const DELETE_CONCURRENCY = + env.BULK_DELETE_CONCURRENCY > 0 ? env.BULK_DELETE_CONCURRENCY : DEFAULT_DELETE_CONCURRENCY; + +const resolveDeleteTargets = async (items = [], context, options = {}) => { if (!Array.isArray(items) || items.length === 0) { - throw new Error('At least one item is required.'); + // Same as the transfer above: nothing to delete is a malformed request. + throw new ValidationError('At least one item is required.'); } - const targets = []; + const includeStats = options.includeStats !== false; + const includeShareDescendants = Boolean(options.includeShareDescendants); + const targets = new Array(items.length); - for (const item of items) { + const resolveOne = async (item, index) => { const combined = combineRelativePath(item.path || '', item.name); const { allowed, accessInfo, resolved } = await authorizeAndResolve( context, @@ -358,12 +1130,21 @@ const resolveDeleteTargets = async (items = [], context) => { ACTIONS.delete ); if (!allowed || !resolved) { - throw new Error(accessInfo?.denialReason || 'Cannot delete items from this path.'); + throw new ForbiddenError(accessInfo?.denialReason || 'Cannot delete items from this path.'); } const { relativePath, absolutePath } = resolved; - const exists = await pathExists(absolutePath); - const stats = exists ? await fs.stat(absolutePath) : null; + // One stat, not an existence probe followed by a stat: stat already answers + // both questions, and on network storage each of those is a round trip. + let stats = null; + if (includeStats) { + try { + stats = await fs.stat(absolutePath); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + } + const exists = includeStats ? stats !== null : null; const isDirectory = stats ? stats.isDirectory() : item?.kind === 'directory'; // A volume is a mount, not a folder in it: deleting one from here would @@ -375,20 +1156,41 @@ const resolveDeleteTargets = async (items = [], context) => { assertNotTopLevelEntry(item.path || '', 'deleted', onDisk?.isDirectory() === true); } - targets.push({ + targets[index] = { item, relativePath, absolutePath, - // Where it was reached through, and the share when it was one: the trash - // records whose folder or share a deletion came from. + // Where it was reached through, and the share when it was one: the + // trash records whose folder or share a deletion came from. space: resolved.space, shareInfo: resolved.shareInfo || null, exists, stats, isDirectory, - shareSourceTarget: getShareSourceTarget(resolved, isDirectory), - }); - } + // The delete-impact endpoint must include shares nested below a folder, + // but it does not need a filesystem stat just to determine that. Looking + // below a regular file is harmless (there cannot be matching children), + // and avoids an avoidable disk round trip before every confirmation. + shareSourceTarget: getShareSourceTarget( + resolved, + includeShareDescendants ? true : isDirectory + ), + }; + }; + + // Same reasoning as the removals: each item costs an authorization check and + // a stat, and on network storage those are round trips worth overlapping. + let next = 0; + await Promise.all( + Array.from({ length: Math.min(DELETE_CONCURRENCY, items.length) }, async () => { + for (;;) { + const index = next; + next += 1; + if (index >= items.length) return; + await resolveOne(items[index], index); + } + }) + ); return targets; }; @@ -398,7 +1200,10 @@ const getDeleteImpact = async (items = [], options = {}) => { user: options.user || null, guestSession: options.guestSession || null, }; - const targets = await resolveDeleteTargets(items, context); + const targets = await resolveDeleteTargets(items, context, { + includeStats: false, + includeShareDescendants: true, + }); const shares = await getSharesForSourceTargets( targets.map((target) => target.shareSourceTarget).filter(Boolean) ); @@ -406,11 +1211,26 @@ const getDeleteImpact = async (items = [], options = {}) => { // which before anyone presses the button — and, for each, how many share // links it carries: switched off and kept in the trash, or deleted with it. const trashPlan = await trash.describeTargets(targets); + const sharesByTarget = await getSharesBySourceTarget( + targets.map((target) => target.shareSourceTarget).filter(Boolean) + ); + // And, for anything that is not coming back, the history it takes with it. + // Only for those: into the trash a file keeps its versions and gets them + // back when it is restored, so there is nothing to warn about. + const versionLifecycle = require('./versions/lifecycle'); trashPlan.items = await Promise.all( trashPlan.items.map(async (entry, index) => { - const { shareSourceTarget } = targets[index] || {}; - const linked = shareSourceTarget ? await getSharesForSourceTargets([shareSourceTarget]) : []; - return { ...entry, shareCount: linked.length }; + const target = targets[index] || {}; + const linked = target.shareSourceTarget + ? sharesByTarget.get(shareTargetKey(target.shareSourceTarget)) + : null; + const withShares = { ...entry, shareCount: linked ? linked.length : 0 }; + if (entry.disposition !== 'permanent' || !target.absolutePath) return withShares; + + const history = await versionLifecycle.countUnder(target.absolutePath); + return history.versions > 0 + ? { ...withShares, versionCount: history.versions, versionBytes: history.bytes } + : withShares; }) ); @@ -421,14 +1241,7 @@ const getDeleteImpact = async (items = [], options = {}) => { }; }; -const createCancellationError = () => { - const error = new Error('Operation cancelled.'); - error.code = 'OPERATION_CANCELLED'; - return error; -}; - const deleteItems = async (items = [], options = {}) => { - const results = []; const context = { user: options.user || null, guestSession: options.guestSession || null, @@ -443,7 +1256,16 @@ const deleteItems = async (items = [], options = {}) => { const useTrash = Boolean(trashSettings?.enabled); const budgetFor = useTrash ? trash.budgetResolver(trashSettings) : null; + // One database pass for the whole selection instead of one per file. + const sharesByTarget = await getSharesBySourceTarget( + targets.map((target) => target.shareSourceTarget).filter(Boolean) + ); + + // Indexed rather than appended: the removals finish out of order, but the + // caller is answered in the order it asked. + const results = new Array(targets.length); let completedItems = 0; + const reportProgress = (target, relativePath) => { completedItems += 1; options.onProgress?.({ @@ -454,55 +1276,84 @@ const deleteItems = async (items = [], options = {}) => { }); }; - for (const target of targets) { - if (options.signal?.aborted) throw createCancellationError(); + const removeOne = async (target, index) => { + throwIfCancelled(options.signal); const { relativePath, absolutePath, exists, stats, isDirectory, shareSourceTarget } = target; const affectedShares = shareSourceTarget - ? await getSharesForSourceTargets([shareSourceTarget]) + ? sharesByTarget.get(shareTargetKey(shareSourceTarget)) || [] : []; if (!exists) { const deletedShareCount = await deleteSharesByIds(affectedShares.map((share) => share.id)); - results.push({ path: relativePath, status: 'missing' }); - if (deletedShareCount > 0) { - results[results.length - 1].deletedShareCount = deletedShareCount; - } + results[index] = { + path: relativePath, + status: 'missing', + ...(deletedShareCount > 0 ? { deletedShareCount } : {}), + }; reportProgress(target, relativePath); - continue; + return; } + const deletedEntryStats = stats || (await fs.stat(absolutePath)); + // A copy may still be writing its hidden entry inside this folder, or be + // finishing an entry that has just landed under this name. Stop the writer + // and wait for its cleanup before removing the tree. + await cancelWritesTargeting(absolutePath); + const isDirectoryEntry = isDirectory || deletedEntryStats.isDirectory(); let trashItemId = null; - // What a permanent deletion took with the file. Carried back so the - // deletion can be written down whole: one file on screen can be ten - // earlier copies of it on disk, and those are the half nothing restores. - let versionsTaken = null; - if (useTrash) { + // A cancelled copy removes only its hidden entry, but the entry asked for + // may still have gone meanwhile. Then there is nothing left to put in the + // trash, and the deletion finishes as it always did. + if (useTrash && (await pathExists(absolutePath))) { const outcome = await trash.trashTarget(target, context, { budgetFor }); if (outcome.status === 'missing') { - results.push({ path: relativePath, status: 'missing' }); + results[index] = { path: relativePath, status: 'missing' }; reportProgress(target, relativePath); - continue; + return; } if (outcome.status !== 'trashed') { - // Never turned into a permanent deletion here: the entry stays where it - // is, and the person is asked whether to delete it for good. - results.push({ + // Never turned into a permanent deletion here: the entry stays where + // it is, and the person is asked whether to delete it for good. + results[index] = { path: relativePath, status: 'kept', reason: outcome.reason, ...(outcome.reason === 'too-large' ? { size: outcome.size, budgetBytes: outcome.budgetBytes } : {}), - }); + }; reportProgress(target, relativePath); - continue; + return; } trashItemId = outcome.item.id; + if (!isDirectoryEntry) scheduleThumbnailRemoval(absolutePath); + } else if (shouldRemoveNatively(isDirectoryEntry) && nativeToolUsable('rm')) { + try { + await removeWithNativeRm(absolutePath, options.signal); + } catch (error) { + if (options.signal?.aborted || !nativeToolIsUnusable(error)) throw error; + recordUnusableNativeTool('rm', error); + await fs.rm(absolutePath, { recursive: true, force: true }); + } + } else if (isDirectoryEntry) { + await fs.rm(absolutePath, { recursive: true, force: true }); } else { - await fs.rm(absolutePath, { recursive: isDirectory || stats.isDirectory(), force: true }); - // Deleted for good, the history goes with it; into the trash, it went - // along with the item inside trashTarget. - // eslint-disable-next-line global-require + // The type is already known from the stat above; fs.rm would lstat again + // just to decide what it is. + await fs.unlink(absolutePath).catch((error) => { + if (error?.code !== 'ENOENT') throw error; + }); + scheduleThumbnailRemoval(absolutePath); + } + folderSizeHooks.onEntryDeleted(absolutePath, { + isDirectory: isDirectoryEntry, + size: deletedEntryStats.size, + }); + // Deleted for good, the history goes with it; into the trash, it went along. + // What it took is carried back so the deletion can be written down whole: + // one file on screen can be ten earlier copies of it on disk. + let versionsTaken = null; + if (!trashItemId) { versionsTaken = await require('./versions/lifecycle').onDeleted(absolutePath); } // In the trash, a share is switched off but kept with the item, so a restore @@ -515,21 +1366,12 @@ const deleteItems = async (items = [], options = {}) => { ); } const deletedShareCount = await deleteSharesByIds(affectedShares.map((share) => share.id)); - // Favorites the deleter had on what just went away: a favorite pointing at - // nothing is a dead end. Best-effort, and only for a signed-in account. - let removedFavoriteCount = 0; - if (context.user?.id) { - try { - removedFavoriteCount = await favoritesService.removeFavoritesForDeletedPath( - context.user.id, - relativePath, - { includeChildren: isDirectory || stats.isDirectory() } - ); - } catch { - // A favorites cleanup must never fail a deletion. - } - } - results.push({ + // Favorites, recent destinations and per-folder preferences, for every user + // who had them — not just whoever pressed delete. + const removedFavoriteCount = await pathBindings.forgetPath(relativePath, { + includeChildren: isDirectoryEntry, + }); + results[index] = { path: relativePath, status: trashItemId ? 'trashed' : 'deleted', ...(trashItemId ? { trashItemId } : {}), @@ -538,27 +1380,60 @@ const deleteItems = async (items = [], options = {}) => { ...(versionsTaken?.versions > 0 ? { versionsPurged: versionsTaken.versions, versionBytesPurged: versionsTaken.bytes } : {}), - }); + }; reportProgress(target, relativePath); - } + }; + + let next = 0; + const workers = Array.from({ length: Math.min(DELETE_CONCURRENCY, targets.length) }, async () => { + for (;;) { + const index = next; + next += 1; + if (index >= targets.length) return; + await removeOne(targets[index], index); + } + }); + + await Promise.all(workers); return results; }; +/** + * A transfer in one call, checked and then carried out. + * + * The two halves exist because a transfer reports as it goes and can be + * stopped: the route checks first, so a refusal is an ordinary HTTP error, and + * only then opens the stream. Everything that does not need to watch — the + * trash putting something back, a test, a script — wants the one call, and the + * answer it has always had. + */ +const transferItems = async (items, destination, operation, options = {}) => { + const prep = await prepareTransfer(items, destination, operation, options); + const result = await executeTransfer(prep, operation, () => {}, options); + return { destination: prep.destinationRelative, ...result }; +}; + module.exports = { transferItems, + prepareTransfer, + executeTransfer, + createCancellationError, getDeleteImpact, resolveDeleteTargets, deleteItems, - // Where a path is, as share links name it: the trash points a restored share - // at the place its content went back to. + shouldRemoveNatively, + nativeTransferEnabled, + nativeToolIsUnusable, + getDiagnosticsSnapshot, + // Exported for tests: the copy path is chosen inside a spawned process, so + // the decision to retry is what can be checked without one. + isPermissionPreservationFailure, + copyWithNativeRsync, + // Where a path is, as share links name it: the trash points a restored + // share at the place its content went back to. getShareSourceTarget, - // The trash restores across disks with a copy that reports progress, copies a - // link as a link and is cancellable. + // The trash restores across disks with the same copy a transfer uses: + // permissions kept, links copied as links, progress reported, cancellable. copyEntryWithProgress, - // The two engines it chooses between are what a test has to be able to name: - // whichever one the platform would pick, the other would never run. - copyEntry, - // Which of the two is in force, for the report of what this machine can do. - nativeCopyEnabled, }; diff --git a/backend/src/services/guestSessionService.js b/backend/src/services/guestSessionService.js index 7d2978e1d..4d40fbd36 100644 --- a/backend/src/services/guestSessionService.js +++ b/backend/src/services/guestSessionService.js @@ -1,12 +1,5 @@ -const crypto = require('crypto'); const { getDb } = require('./db'); - -const nowIso = () => new Date().toISOString(); - -const generateId = () => - typeof crypto.randomUUID === 'function' - ? crypto.randomUUID() - : `${Date.now().toString(36)}-${crypto.randomBytes(8).toString('hex')}`; +const { generateId, nowIso } = require('../utils/ids'); // Default session duration: 24 hours const DEFAULT_SESSION_HOURS = 24; @@ -67,24 +60,6 @@ const getGuestSession = async (sessionId) => { return toClientSession(row); }; -/** - * Get all guest sessions for a share - */ -const getGuestSessionsByShareId = async (shareId) => { - const db = await getDb(); - const rows = db - .prepare( - ` - SELECT * FROM guest_sessions - WHERE share_id = ? - ORDER BY created_at DESC - ` - ) - .all(shareId); - - return rows.map(toClientSession); -}; - /** * Check if a guest session is valid (exists and not expired) */ @@ -119,24 +94,6 @@ const updateGuestSessionActivity = async (sessionId) => { return result.changes > 0; }; -/** - * Delete guest session - */ -const deleteGuestSession = async (sessionId) => { - const db = await getDb(); - const result = db.prepare('DELETE FROM guest_sessions WHERE id = ?').run(sessionId); - return result.changes > 0; -}; - -/** - * Delete all guest sessions for a share - */ -const deleteGuestSessionsByShareId = async (shareId) => { - const db = await getDb(); - const result = db.prepare('DELETE FROM guest_sessions WHERE share_id = ?').run(shareId); - return result.changes; -}; - /** * Clean up expired guest sessions */ @@ -153,74 +110,10 @@ const cleanupExpiredSessions = async () => { return result.changes; }; -/** - * Get guest session count for a share - */ -const getActiveSessionCount = async (shareId) => { - const db = await getDb(); - const row = db - .prepare( - ` - SELECT COUNT(*) as count - FROM guest_sessions - WHERE share_id = ? AND expires_at > ? - ` - ) - .get(shareId, nowIso()); - - return row?.count || 0; -}; - -/** - * Extend guest session expiration - */ -const extendGuestSession = async (sessionId, additionalHours = DEFAULT_SESSION_HOURS) => { - const session = await getGuestSession(sessionId); - - if (!session) { - const e = new Error('Guest session not found'); - e.status = 404; - throw e; - } - - const currentExpiry = new Date(session.expiresAt); - const newExpiry = new Date(currentExpiry.getTime() + additionalHours * 60 * 60 * 1000); - - const db = await getDb(); - db.prepare( - ` - UPDATE guest_sessions - SET expires_at = ?, last_activity_at = ? - WHERE id = ? - ` - ).run(newExpiry.toISOString(), nowIso(), sessionId); - - return getGuestSession(sessionId); -}; - -/** - * Verify guest session belongs to a specific share - */ -const verifyGuestSessionShare = async (sessionId, shareId) => { - const session = await getGuestSession(sessionId); - - if (!session) { - return false; - } - - return session.shareId === shareId; -}; - module.exports = { createGuestSession, getGuestSession, - getGuestSessionsByShareId, isGuestSessionValid, updateGuestSessionActivity, - deleteGuestSession, - deleteGuestSessionsByShareId, cleanupExpiredSessions, - getActiveSessionCount, - extendGuestSession, - verifyGuestSessionShare, }; diff --git a/backend/src/services/inFlightFiles.js b/backend/src/services/inFlightFiles.js index 43a9cbd61..bfecd24c8 100644 --- a/backend/src/services/inFlightFiles.js +++ b/backend/src/services/inFlightFiles.js @@ -8,13 +8,14 @@ const logger = require('../utils/logger'); /** * What an operation is writing, recorded until it is done. * - * Extracting or compressing an archive, and receiving an upload, write - * something that is not finished yet — a staging folder, a hidden zip beside - * the name it will take, a hidden file filling up with an upload's bytes — and - * remove it when they fail. Nothing removed it when the process was stopped - * half-way: a container restarted mid-extraction left a - * `.nextexplorer-extract-*` folder or a half-written zip in the volume for - * good, out of sight. + * Saving a file, pulling a document from ONLYOFFICE, extracting or compressing + * an archive: each writes something that is not finished yet — a hidden + * temporary beside the file, a staging folder, a new folder filling up, a + * hidden zip beside the name it will take — and removes it when it fails. + * Nothing removed it when the process was stopped half-way: a container + * restarted mid-extraction left a `.nextexplorer-extract-*` folder, a + * half-written zip or a half-filled folder in the volume for good, and the + * hidden ones out of sight. * * So each operation records the path before creating it and releases the * record when it is done, however it ends. A record still there at the next diff --git a/backend/src/services/indexDb.js b/backend/src/services/indexDb.js index 1e078c2a4..7cf403867 100644 --- a/backend/src/services/indexDb.js +++ b/backend/src/services/indexDb.js @@ -79,13 +79,14 @@ const metaValue = (db, key) => { }; /** - * An index database of this application's own, under the cache, and the write- - * ahead files beside it. Nothing anybody put anywhere is ever in one, so it - * does not go through the trash. + * The search index and the files SQLite keeps beside it. + * + * Nothing anybody put anywhere passes through here: the index is built from + * the volume and made again whenever it is missing, which is why it does not + * go through the trash. */ const removeFile = (file) => { for (const suffix of ['', '-wal', '-shm', '-journal']) { - // eslint-disable-next-line no-restricted-properties fs.rmSync(`${file}${suffix}`, { force: true }); } }; diff --git a/backend/src/services/legacyCacheCheck.js b/backend/src/services/legacyCacheCheck.js new file mode 100644 index 000000000..0a9f67950 --- /dev/null +++ b/backend/src/services/legacyCacheCheck.js @@ -0,0 +1,83 @@ +const fs = require('fs'); +const path = require('path'); + +const { directories } = require('../config/index'); +const logger = require('../utils/logger'); + +/** + * What early releases left in the cache directory, said out loud at start. + * + * Up to 1.1.7 the database and app-config.json lived in the cache directory. + * 1.1.8 moved them to the config directory and left links behind in their + * place; 2.0.3 removed that move from the entrypoint. So an installation that + * started on 1.1.7 or earlier and skipped the releases in between comes up on a + * new, empty app.db in /config, with its accounts and shares sitting unread in + * /cache — and nothing said so. The links, where an installation passed through + * 1.1.8 to 2.0.2, are harmless but look like data. + * + * Nothing is moved: which of two databases holds what matters cannot be told + * from here, and guessing wrong would overwrite the one in use. The log says + * where the old file is and what to do with it. + */ + +const LEGACY_NAMES = ['app.db', 'app-config.json', 'extensions']; + +const readLinkOrNull = (file) => { + try { + return fs.readlinkSync(file); + } catch { + return null; + } +}; + +/** What is there, without following anything. */ +const inspectLegacyCache = (cacheDir = directories.cache) => { + const findings = []; + for (const name of LEGACY_NAMES) { + const file = path.join(cacheDir, name); + let stats; + try { + stats = fs.lstatSync(file); + } catch { + continue; + } + if (stats.isSymbolicLink()) { + findings.push({ name, path: file, kind: 'link', target: readLinkOrNull(file) }); + } else if (name === 'app.db' && stats.isFile()) { + findings.push({ name, path: file, kind: 'database', sizeBytes: stats.size }); + } + } + return findings; +}; + +const reportLegacyCache = ({ + cacheDir = directories.cache, + configDir = directories.config, + log = logger, +} = {}) => { + const findings = inspectLegacyCache(cacheDir); + + const database = findings.find((finding) => finding.kind === 'database'); + if (database) { + log.warn( + { + legacyDatabase: database.path, + sizeBytes: database.sizeBytes, + databaseInUse: path.join(configDir, 'app.db'), + }, + 'An app.db written by release 1.1.7 or earlier is in the cache directory, and nothing reads it: this server runs on the app.db in the config directory. If accounts, shares or favorites are missing, stop the container, back up both files, and copy the old one over the one in the config directory.' + ); + } + + const links = findings.filter((finding) => finding.kind === 'link'); + if (links.length > 0) { + log.info( + { links: links.map((link) => `${link.path} -> ${link.target}`) }, + 'Links left in the cache directory by releases 1.1.8 to 2.0.2 are unused and can be deleted.' + ); + } + + return findings; +}; + +module.exports = { inspectLegacyCache, reportLegacyCache }; diff --git a/backend/src/services/orphanedBindingsService.js b/backend/src/services/orphanedBindingsService.js new file mode 100644 index 000000000..89836df08 --- /dev/null +++ b/backend/src/services/orphanedBindingsService.js @@ -0,0 +1,139 @@ +const fs = require('fs/promises'); + +const { directories } = require('../config/index'); +const { parsePathSpace } = require('../utils/pathUtils'); +const { PATH_TABLES } = require('./pathBindingsService'); +const { getDb } = require('./db'); +const logger = require('../utils/logger'); + +/** + * Report what points at a volume that is not there — and remove nothing. + * + * Favourites, shares, recent destinations and folder preferences all store a + * path whose first segment names a volume. Remove a volume from the compose + * file and those rows survive, pointing nowhere: the favourite still shows in + * the sidebar and answers with an error when clicked. + * + * The obvious fix — delete them at startup — would be worse than the problem. + * A volume that is absent is not a volume that is gone: an NFS or SMB mount may + * not be ready when the container starts, an external disk may be unplugged for + * a weekend, a compose line may be mistyped and corrected a minute later. Any + * of those would silently destroy every user's favourites and shares, for good. + * Only a person can tell "not mounted yet" from "never coming back", so this + * says what it sees and leaves the decision to them. + */ + +/** Volume names that can legitimately appear at the head of a stored path. */ +const knownVolumeNames = async (db) => { + const names = new Set(); + + const entries = await fs.readdir(directories.volume, { withFileTypes: true }).catch((error) => { + logger.debug({ err: error }, 'Volume root unreadable while checking stored paths'); + return null; + }); + // Unreadable root: every path would look orphaned. Say nothing rather than + // cry wolf about all of them. + if (!entries) return null; + + for (const entry of entries) { + if (entry.isDirectory()) names.add(entry.name); + } + + // A per-user volume is addressed by its label, not by a directory under the + // volume root — without these, every one of them would look missing. + try { + for (const row of db.prepare('SELECT DISTINCT label FROM user_volumes').all()) { + if (row?.label) names.add(row.label); + } + } catch (error) { + logger.debug({ err: error }, 'Could not read user volumes while checking stored paths'); + } + + return names; +}; + +/** + * The volume a stored path belongs to, or null where it belongs to none — + * a personal folder or a share token names a space of its own, not a volume. + */ +const volumeOf = (storedPath) => { + const { space, rel } = parsePathSpace(storedPath || ''); + if (space !== 'volume' || !rel) return null; + return rel.split('/')[0] || null; +}; + +/** + * What the database points at that the filesystem does not have. + * Returns null when the question cannot be answered. + */ +const findOrphanedBindings = async () => { + const db = await getDb(); + const known = await knownVolumeNames(db); + if (!known) return null; + + const missing = new Map(); + + for (const { table, column } of PATH_TABLES) { + let rows; + try { + rows = db + .prepare( + `SELECT ${column} AS storedPath, COUNT(*) AS count FROM ${table} GROUP BY ${column}` + ) + .all(); + } catch (error) { + // A table that is not there yet (a migration mid-flight) is not a reason + // to fail the check, let alone the startup it runs from. + logger.debug({ err: error, table }, 'Skipped a table while checking stored paths'); + continue; + } + + for (const row of rows) { + const volume = volumeOf(row.storedPath); + if (!volume || known.has(volume)) continue; + + const entry = missing.get(volume) || { volume, tables: {}, total: 0 }; + entry.tables[table] = (entry.tables[table] || 0) + Number(row.count || 0); + entry.total += Number(row.count || 0); + missing.set(volume, entry); + } + } + + return [...missing.values()].sort((a, b) => b.total - a.total); +}; + +/** Say what was found, once, at startup. Never throws. */ +const reportOrphanedBindings = async () => { + try { + const orphaned = await findOrphanedBindings(); + if (!orphaned || orphaned.length === 0) return; + + logger.warn( + { + volumes: orphaned.map(({ volume, total, tables }) => ({ volume, total, tables })), + }, + `Stored paths point at ${orphaned.length} volume(s) that are not available: ` + + `${orphaned.map((entry) => `${entry.volume} (${entry.total})`).join(', ')}. ` + + 'Nothing has been removed — a volume that is not mounted yet looks exactly like one that is gone.' + ); + } catch (error) { + logger.debug({ err: error }, 'Could not check stored paths against available volumes'); + } +}; + +/** + * The volume names a stored path may legitimately start with, or null when the + * question cannot be answered — an unreadable volume root would otherwise make + * everything look missing at once. + */ +const listKnownVolumeNames = async () => { + const db = await getDb(); + return knownVolumeNames(db); +}; + +module.exports = { + findOrphanedBindings, + reportOrphanedBindings, + listKnownVolumeNames, + volumeOf, +}; diff --git a/backend/src/services/pathBindingsService.js b/backend/src/services/pathBindingsService.js new file mode 100644 index 000000000..d7db62c33 --- /dev/null +++ b/backend/src/services/pathBindingsService.js @@ -0,0 +1,123 @@ +const { getDb, prepared } = require('./db'); +const { normalizeRelativePath } = require('../utils/pathUtils'); +const logger = require('../utils/logger'); + +/** + * Everything the database ties to a path, moved or forgotten in one place. + * + * Files move and disappear; the rows that point at them did not follow. A + * favorite survived the folder it named, a share kept pointing at a path that + * no longer existed, and a folder's sort order outlived two or three folders + * that happened to be created at the same place afterwards. Each of those was + * handled — or not — wherever someone remembered to, which is why deleting a + * folder cleaned up the favorites of whoever deleted it and nobody else. + * + * These are other people's rows as much as your own, so nothing here filters by + * user: a folder that is gone is gone for everyone who had bookmarked it. + */ + +// Rows keyed by a path, and what the column is called there. +const PATH_TABLES = [ + { table: 'favorites', column: 'path' }, + { table: 'recent_destinations', column: 'path' }, + { table: 'folder_preferences', column: 'path' }, + { table: 'shares', column: 'source_path' }, +]; + +/** + * Everything under a path, as bounds on the column: every path that begins + * with `prefix/` sorts at or after it and before `prefix0`, `0` being the + * character right after `/`. + * + * `LIKE 'prefix/%'` said the same thing and ignored case besides — for ASCII + * SQLite's LIKE always does. Deleting `Docs` dropped the favorites and the + * share links of `docs/…`, and renaming it re-pointed them at `Papers/…`: on a + * Linux volume another folder, and for a share, possibly another file than the + * one that was shared. The bounds compare bytes, and use the column's index. + */ +const childRange = (prefix) => [`${prefix}/`, `${prefix}0`]; + +/** + * Forget what pointed at a path that no longer exists. + * + * @param {string} relativePath + * @param {object} [options] + * @param {boolean} [options.includeChildren] Also everything beneath it, which + * is what deleting a folder means. + * @returns {Promise} Rows removed, for logging. + */ +const forgetPath = async (relativePath, { includeChildren = false } = {}) => { + const normalized = normalizeRelativePath(relativePath); + if (!normalized) return 0; + + let removed = 0; + try { + const db = await getDb(); + db.transaction(() => { + for (const { table, column } of PATH_TABLES) { + const result = includeChildren + ? prepared( + db, + `DELETE FROM ${table} WHERE ${column} = ? OR (${column} >= ? AND ${column} < ?)` + ).run(normalized, ...childRange(normalized)) + : prepared(db, `DELETE FROM ${table} WHERE ${column} = ?`).run(normalized); + removed += result.changes; + } + })(); + } catch (error) { + // Losing a favorite is not a reason to fail the deletion that succeeded. + logger.warn({ err: error, relativePath }, 'Could not clean up bindings for deleted path'); + } + + return removed; +}; + +/** + * Follow a path that moved, so what pointed at it still does. + * + * Children come along: renaming a folder moves everything inside it, and a + * favorite two levels down is still the same folder afterwards. + * + * A row may already exist at the destination — someone had both folders + * bookmarked, and one has just taken the other's place. The move is written + * first as a replace so it wins, then the leftovers are dropped. + */ +const movePath = async (fromPath, toPath, { includeChildren = true } = {}) => { + const from = normalizeRelativePath(fromPath); + const to = normalizeRelativePath(toPath); + if (!from || !to || from === to) return 0; + + let moved = 0; + try { + const db = await getDb(); + const childOffset = from.length + 2; // SQLite substr is 1-based, past the '/' + + db.transaction(() => { + for (const { table, column } of PATH_TABLES) { + moved += prepared( + db, + `UPDATE OR REPLACE ${table} SET ${column} = ? WHERE ${column} = ?` + ).run(to, from).changes; + + if (!includeChildren) continue; + + moved += prepared( + db, + `UPDATE OR REPLACE ${table} + SET ${column} = ? || substr(${column}, ?) + WHERE ${column} >= ? AND ${column} < ?` + ).run(`${to}/`, childOffset, ...childRange(from)).changes; + } + })(); + } catch (error) { + logger.warn({ err: error, fromPath, toPath }, 'Could not follow moved path in bindings'); + } + + return moved; +}; + +module.exports = { + forgetPath, + movePath, + PATH_TABLES, +}; diff --git a/backend/src/services/pdfTextExtract.js b/backend/src/services/pdfTextExtract.js index 7ac08b461..317549d73 100644 --- a/backend/src/services/pdfTextExtract.js +++ b/backend/src/services/pdfTextExtract.js @@ -35,10 +35,7 @@ const hasPdfToText = async () => { }); if (!available) { - // At debug level this said nothing on a default install, and a PDF search - // that quietly returns no match looks like a PDF with no matching text. - // Same level as the 7-Zip probe, for the same reason. - logger.warn('pdftotext is not installed; PDF contents will not be searched'); + logger.debug('pdftotext is not installed; PDF contents will not be searched'); } return available; }; diff --git a/backend/src/services/performanceDiagnostics.js b/backend/src/services/performanceDiagnostics.js new file mode 100644 index 000000000..0b3d65faa --- /dev/null +++ b/backend/src/services/performanceDiagnostics.js @@ -0,0 +1,178 @@ +const fs = require('fs/promises'); +const { monitorEventLoopDelay, performance } = require('perf_hooks'); + +const { performanceDiagnostics: config } = require('../config'); +const logger = require('../utils/logger'); +const thumbnailService = require('./thumbnailService'); +const folderSizeManager = require('./folderSizeManager'); +const fileTransferService = require('./fileTransferService'); + +let timer = null; +let previousSample = null; +let eventLoopDelay = null; + +const toMb = (bytes) => Math.round((Number(bytes) || 0) / 1024 / 1024); + +const readText = async (filePath) => { + try { + return (await fs.readFile(filePath, 'utf8')).trim(); + } catch (_) { + return null; + } +}; + +const readNumber = async (filePath) => { + const value = await readText(filePath); + if (value == null || value === 'max') return null; + const number = Number(value); + return Number.isFinite(number) ? number : null; +}; + +const readKeyValueFile = async (filePath, allowedKeys) => { + const content = await readText(filePath); + if (!content) return null; + const result = {}; + for (const line of content.split('\n')) { + const [key, rawValue] = line.trim().split(/\s+/, 2); + if (!allowedKeys.has(key)) continue; + const value = Number(rawValue); + if (Number.isFinite(value)) result[key] = toMb(value); + } + return result; +}; + +const readCgroupMemory = async () => { + const v2Current = await readNumber('/sys/fs/cgroup/memory.current'); + const v2Limit = await readNumber('/sys/fs/cgroup/memory.max'); + const isV2 = v2Current != null; + const current = isV2 + ? v2Current + : await readNumber('/sys/fs/cgroup/memory/memory.usage_in_bytes'); + const limit = isV2 ? v2Limit : await readNumber('/sys/fs/cgroup/memory/memory.limit_in_bytes'); + const stat = await readKeyValueFile( + isV2 ? '/sys/fs/cgroup/memory.stat' : '/sys/fs/cgroup/memory/memory.stat', + new Set(['anon', 'file', 'slab', 'slab_reclaimable', 'slab_unreclaimable', 'cache', 'rss']) + ); + + if (current == null && !stat) return null; + return { + currentMb: toMb(current), + ...(limit != null ? { limitMb: toMb(limit) } : {}), + ...(stat ? { statMb: stat } : {}), + }; +}; + +const activeResourceCounts = () => { + if (typeof process.getActiveResourcesInfo !== 'function') return undefined; + return process.getActiveResourcesInfo().reduce((counts, name) => { + counts[name] = (counts[name] || 0) + 1; + return counts; + }, {}); +}; + +const sample = async () => { + const now = performance.now(); + const cpu = process.cpuUsage(); + const memory = process.memoryUsage(); + const [cgroupMemory, thumbnail, folderSize, transfers] = await Promise.all([ + readCgroupMemory(), + // Each queue reports itself when it can. One that does not is a queue this + // installation has no report for, not a reason for the whole record to fail — a + // diagnostic that throws is a diagnostic that says nothing at the moment it is + // most wanted. + Promise.resolve(thumbnailService.getDiagnosticsSnapshot?.() ?? null), + Promise.resolve(folderSizeManager.getDiagnosticsSnapshot?.() ?? null), + Promise.resolve(fileTransferService.getDiagnosticsSnapshot?.() ?? null), + ]); + + const elapsedMs = previousSample ? Math.max(1, now - previousSample.at) : null; + const cpuDeltaUs = previousSample + ? cpu.user - previousSample.cpu.user + (cpu.system - previousSample.cpu.system) + : null; + const cpuPercent = + elapsedMs != null && cpuDeltaUs != null + ? Math.round((cpuDeltaUs / 1000 / elapsedMs) * 100) + : null; + const loopDelayMs = eventLoopDelay + ? Number(eventLoopDelay.percentile(99) / 1e6).toFixed(1) + : null; + const eventLoopUtilization = previousSample?.eventLoopUtilization + ? performance.eventLoopUtilization(previousSample.eventLoopUtilization) + : null; + + previousSample = { + at: now, + cpu, + eventLoopUtilization: performance.eventLoopUtilization(), + }; + eventLoopDelay?.reset(); + + return { + cpuPercent, + ...(eventLoopUtilization + ? { eventLoopUtilizationPercent: Math.round(eventLoopUtilization.utilization * 100) } + : {}), + ...(loopDelayMs != null ? { eventLoopP99DelayMs: Number(loopDelayMs) } : {}), + memoryMb: { + rss: toMb(memory.rss), + heapUsed: toMb(memory.heapUsed), + heapTotal: toMb(memory.heapTotal), + external: toMb(memory.external), + arrayBuffers: toMb(memory.arrayBuffers), + }, + cgroupMemory, + resources: activeResourceCounts(), + thumbnail, + folderSize, + transfers, + }; +}; + +const isPressure = (snapshot) => + (snapshot.cpuPercent ?? 0) >= config.cpuThreshold || + snapshot.memoryMb.rss >= config.rssThresholdMb || + (snapshot.eventLoopP99DelayMs ?? 0) >= config.eventLoopDelayThresholdMs; + +const start = () => { + if (!config.enabled || timer) return; + + eventLoopDelay = monitorEventLoopDelay({ resolution: 20 }); + eventLoopDelay.enable(); + logger.info( + { + intervalMs: config.intervalMs, + cpuThreshold: config.cpuThreshold, + rssThresholdMb: config.rssThresholdMb, + eventLoopDelayThresholdMs: config.eventLoopDelayThresholdMs, + logEveryInterval: config.logEveryInterval, + }, + 'Performance diagnostics enabled' + ); + + const tick = () => { + sample() + .then((snapshot) => { + if (config.logEveryInterval || isPressure(snapshot)) { + logger.info( + { reason: isPressure(snapshot) ? 'resource-pressure' : 'interval', ...snapshot }, + 'Performance diagnostics' + ); + } + }) + .catch((err) => logger.debug({ err }, 'Performance diagnostics sample failed')); + }; + + tick(); + timer = setInterval(tick, config.intervalMs); + if (typeof timer.unref === 'function') timer.unref(); +}; + +const stop = () => { + if (timer) clearInterval(timer); + timer = null; + eventLoopDelay?.disable(); + eventLoopDelay = null; + previousSample = null; +}; + +module.exports = { start, stop, sample }; diff --git a/backend/src/services/personalFolders.js b/backend/src/services/personalFolders.js index 702b4a45f..a6e96de61 100644 --- a/backend/src/services/personalFolders.js +++ b/backend/src/services/personalFolders.js @@ -1,3 +1,7 @@ +const fs = require('fs'); +const path = require('path'); + +const { directories } = require('../config/index'); const { getUserFolderNameCandidates } = require('../utils/pathUtils'); const logger = require('../utils/logger'); @@ -39,6 +43,32 @@ const takenNames = (db, userId) => { return new Set(rows.map((row) => row.name)); }; +/** + * Whether a name is held for an account that was deleted. + * + * Deleting an account leaves its folder on disk, with what it kept there, its + * trash and its versions. Handing the name to the next account that derives it + * handed over that folder too. The name stays reserved for as long as the + * folder is there; an administrator frees it by removing or renaming the folder + * on the server, and the reservation goes the first time the name is asked for + * after that. + */ +const isReserved = (db, name) => { + // Names are claimed by the v15 migration, long before v20 creates this table; + // until it exists, nothing can have been reserved. + const hasTable = db + .prepare( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'personal_folder_reservations'" + ) + .get(); + if (!hasTable) return false; + const row = db.prepare('SELECT name FROM personal_folder_reservations WHERE name = ?').get(name); + if (!row) return false; + if (fs.existsSync(path.join(directories.userRoot, name))) return true; + db.prepare('DELETE FROM personal_folder_reservations WHERE name = ?').run(name); + return false; +}; + /** * Give this account a folder name of its own, and answer it. Idempotent: an * account that already holds one keeps it. @@ -53,7 +83,7 @@ const claimPersonalFolderName = (db, user) => { const candidates = getUserFolderNameCandidates(user); for (const candidate of candidates) { - if (taken.has(candidate)) continue; + if (taken.has(candidate) || isReserved(db, candidate)) continue; try { db.prepare('UPDATE users SET personal_folder_name = ? WHERE id = ?').run(candidate, user.id); @@ -61,7 +91,7 @@ const claimPersonalFolderName = (db, user) => { if (candidate !== candidates[0]) { logger.warn( { userId: user.id, preferred: candidates[0], assigned: candidate }, - 'Personal folder name was already taken by another account; assigned the next one' + 'Personal folder name was taken, or kept for a deleted account; assigned the next one' ); } return candidate; diff --git a/backend/src/services/recentDestinationsService.js b/backend/src/services/recentDestinationsService.js new file mode 100644 index 000000000..a363195b5 --- /dev/null +++ b/backend/src/services/recentDestinationsService.js @@ -0,0 +1,87 @@ +const { getDb } = require('./db'); +const logger = require('../utils/logger'); + +/** + * The folders a user actually moves things into. + * + * The destination picker opens on a list rather than at the root, because the + * folder someone wants is nearly always one they have used before. Nobody + * curates that list: it is written by the transfers themselves, so it stays + * true to how the person really files things. + * + * Kept per user. A shared favourite is a deliberate bookmark; this is a trace + * of one person's habits, and showing someone else's would be both wrong and + * a small leak of where they work. + */ + +const MAX_ENTRIES = 10; + +/** Note that a transfer landed here. Never throws: this is a convenience. */ +const record = async (userId, relativePath) => { + if (!userId || typeof relativePath !== 'string' || !relativePath.trim()) return; + + try { + const db = await getDb(); + const now = new Date().toISOString(); + + db.prepare( + `INSERT INTO recent_destinations (user_id, path, used_at) + VALUES (?, ?, ?) + ON CONFLICT(user_id, path) DO UPDATE SET used_at = excluded.used_at` + ).run(userId, relativePath, now); + + // Trim to the most recent entries. Done on write so the table cannot grow + // for a user who never opens the picker. + db.prepare( + `DELETE FROM recent_destinations + WHERE user_id = ? + AND path NOT IN ( + SELECT path FROM recent_destinations + WHERE user_id = ? + ORDER BY used_at DESC + LIMIT ? + )` + ).run(userId, userId, MAX_ENTRIES); + } catch (error) { + // A destination that fails to be remembered must never fail the transfer + // that reached it. + logger.debug({ err: error, relativePath }, 'Could not record recent destination'); + } +}; + +/** Most recently used first. */ +const list = async (userId) => { + if (!userId) return []; + + const db = await getDb(); + return db + .prepare( + `SELECT path FROM recent_destinations + WHERE user_id = ? + ORDER BY used_at DESC + LIMIT ?` + ) + .all(userId, MAX_ENTRIES) + .map((row) => row.path); +}; + +/** Drop a destination that no longer exists or is no longer reachable. */ +const forget = async (userId, relativePath) => { + if (!userId || !relativePath) return; + + try { + const db = await getDb(); + db.prepare('DELETE FROM recent_destinations WHERE user_id = ? AND path = ?').run( + userId, + relativePath + ); + } catch (error) { + logger.debug({ err: error, relativePath }, 'Could not forget recent destination'); + } +}; + +module.exports = { + record, + list, + forget, +}; diff --git a/backend/src/services/renameService.js b/backend/src/services/renameService.js index f17eaa41a..723d35457 100644 --- a/backend/src/services/renameService.js +++ b/backend/src/services/renameService.js @@ -15,6 +15,8 @@ const { NotFoundError, ConflictError, } = require('../errors/AppError'); +const folderSizeHooks = require('./folderSizeHooks'); +const pathBindings = require('./pathBindingsService'); const versionLifecycle = require('./versions/lifecycle'); /** @@ -107,8 +109,14 @@ const renameEntry = async ({ context, parentRelative, currentName, newName }) => } await fs.rename(currentAbsolute, targetAbsolute); - // The history follows the file, or the histories of everything inside the - // folder, to the new name. + // Same-parent rename: no size delta, but re-key an indexed directory subtree. + folderSizeHooks.onEntryRenamed(currentAbsolute, targetAbsolute); + // Favorites, shares, recent destinations and per-folder preferences follow the + // folder to its new name, including everything inside it. Left behind, they + // would point at a path that no longer exists — a share silently broken, a + // favorite leading nowhere. + await pathBindings.movePath(currentRelative, targetRelative); + // And the file's history, or the histories of everything inside the folder. await versionLifecycle.onMoved(currentAbsolute, targetAbsolute); return { diff --git a/backend/src/services/searchCollector.js b/backend/src/services/searchCollector.js index 64382d314..6318227c1 100644 --- a/backend/src/services/searchCollector.js +++ b/backend/src/services/searchCollector.js @@ -40,7 +40,9 @@ const collectResults = async ({ const contentReserve = Math.max(1, limit - Math.floor(limit * NAME_SHARE)); for await (const item of results) { - (item.matchLine ? contents : names).push(item); + // A content match the index vouched for without its line being read in + // time is still a content match, and is counted as one. + (item.matchLine || item.inContents ? contents : names).push(item); if (names.length < nameCap) continue; if (contents.length >= contentReserve) break; diff --git a/backend/src/services/searchIndexer.js b/backend/src/services/searchIndexer.js index 1c863f43d..d946d0380 100644 --- a/backend/src/services/searchIndexer.js +++ b/backend/src/services/searchIndexer.js @@ -31,7 +31,19 @@ const { containerMemoryLimitBytes } = require('../utils/containerMemory'); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); -const IGNORED_DIRECTORIES = new Set(['.git', 'node_modules', 'dist', 'build', '.cache']); +/** + * Folders the pass does not walk into. + * + * Dot-folders only, and for one reason each: the application keeps its trash + * and its file versions in one of them, and the rest are somebody's hidden + * things, which the search refuses to show unless they asked. `.git`, + * `node_modules`, `dist` and `build` used to be here too — an editor's habits + * in a file server, where those are ordinary folder names somebody may have a + * year of work in, and nothing that decides what is not worth finding belongs + * in the source (#11). The administrator's exclusion list is where that is + * said. + */ +const isSkippedDirectoryName = (name) => name.startsWith('.'); /** * How much of one document is worth indexing, and how much may be held at once. @@ -61,10 +73,39 @@ const NON_TEXT_EXTENSIONS = new Set([ ...extensions.rawImages, ...extensions.videos, ...extensions.audios, - 'zip', 'rar', '7z', 'gz', 'bz2', 'xz', 'zst', 'tar', 'tgz', 'iso', 'dmg', 'jar', - 'exe', 'dll', 'so', 'dylib', 'bin', 'o', 'a', 'class', 'pyc', 'wasm', - 'ttf', 'otf', 'woff', 'woff2', 'eot', - 'db', 'sqlite', 'sqlite3', 'mdb', 'pack', 'idx', + 'zip', + 'rar', + '7z', + 'gz', + 'bz2', + 'xz', + 'zst', + 'tar', + 'tgz', + 'iso', + 'dmg', + 'jar', + 'exe', + 'dll', + 'so', + 'dylib', + 'bin', + 'o', + 'a', + 'class', + 'pyc', + 'wasm', + 'ttf', + 'otf', + 'woff', + 'woff2', + 'eot', + 'db', + 'sqlite', + 'sqlite3', + 'mdb', + 'pack', + 'idx', ]); const extensionOf = (absolutePath) => path.extname(absolutePath).slice(1).toLowerCase(); @@ -182,6 +223,7 @@ const indexTree = async ({ let pendingBytes = 0; let indexed = 0; let skipped = 0; + let folders = 0; let batches = 0; let interrupted = false; @@ -351,7 +393,14 @@ const indexTree = async ({ const batch = pending.splice(0, pending.length); pendingBytes = 0; writeBatch(batch); - indexed += batch.length; + // Counted apart: `indexed` has always meant files this pass had to read, + // and a folder row is written without opening anything. Folding the two + // together would make a volume of empty folders look like a volume that + // changes constantly. + for (const document of batch) { + if (document.isDirectory) folders += 1; + else indexed += 1; + } batches += 1; if (typeof onProgress === 'function' && Date.now() - lastReport >= progressMs) { @@ -369,7 +418,9 @@ const indexTree = async ({ skipped, batches, reindexed: reindexedKnown, - ...(worstFolder ? { rereadTopFolder: worstFolder.value, rereadTopCount: worstFolder.count } : {}), + ...(worstFolder + ? { rereadTopFolder: worstFolder.value, rereadTopCount: worstFolder.count } + : {}), ...cost(), }); } @@ -392,7 +443,7 @@ const indexTree = async ({ for (const entry of entries) { throwIfAborted(); - if (entry.name.startsWith('.') || IGNORED_DIRECTORIES.has(entry.name)) continue; + if (isSkippedDirectoryName(entry.name)) continue; const absolutePath = path.join(dirAbs, entry.name); const relativePath = dirRel ? `${dirRel}/${entry.name}` : entry.name; @@ -400,16 +451,28 @@ const indexTree = async ({ if (isExcluded(relativePath)) continue; if (entry.isDirectory()) { - // eslint-disable-next-line no-await-in-loop + // A row of its own, so a folder nobody has put anything in yet can be + // found by its name. Its own timestamps say nothing useful — a folder's + // mtime moves when its children do — so the row is written once and + // left alone. + seenHere.add(relativePath); + if (!store.getIndexedDocument(db, relativePath)) { + pending.push({ + path: relativePath, + mtimeMs: 0, + size: 0, + text: null, + isDirectory: true, + }); + if (pending.length >= batchSize) flush(); + } await walk(absolutePath, relativePath); continue; } if (!entry.isFile()) continue; - // eslint-disable-next-line no-await-in-loop const stats = await fs.stat(absolutePath).catch(() => null); if (!stats) continue; - if (maxFileSizeBytes && stats.size > maxFileSizeBytes) continue; seenHere.add(relativePath); @@ -418,7 +481,6 @@ const indexTree = async ({ const known = store.getIndexedDocument(db, relativePath); if (store.isUpToDate(known, stats)) { skipped += 1; - // eslint-disable-next-line no-await-in-loop await payForTimeUsed(); continue; } @@ -453,24 +515,27 @@ const indexTree = async ({ } } - // eslint-disable-next-line no-await-in-loop - const text = await readIndexableText(absolutePath, stats.size, scratch); - if (text === null || !text.trim()) continue; + // The size bound is on reading a file, not on knowing it is there. A + // two-gigabyte recording has no words worth keeping and a name somebody + // will look for, and the row costs what the stat above already paid. + const tooLargeToRead = maxFileSizeBytes && stats.size > maxFileSizeBytes; + const text = tooLargeToRead + ? null + : await readIndexableText(absolutePath, stats.size, scratch); + const indexable = text === null || !text.trim() ? null : capText(text); - const indexable = capText(text); pending.push({ path: relativePath, mtimeMs: stats.mtimeMs, size: stats.size, text: indexable, }); - pendingBytes += indexable.length; + pendingBytes += indexable ? indexable.length : 0; // Whichever ceiling is reached first. The byte one is what keeps a // handful of large documents from being held together. if (pending.length >= batchSize || pendingBytes >= MAX_TEXT_PER_BATCH) flush(); - // eslint-disable-next-line no-await-in-loop await payForTimeUsed(); } @@ -516,6 +581,7 @@ const indexTree = async ({ return { indexed, skipped, + folders, removed, batches, pauses, @@ -538,29 +604,22 @@ const indexFile = async (db, relativePath, absolutePath) => { return { removed: true }; } - const maxBytes = searchConfig?.maxFileSizeBytes ?? 0; - if (maxBytes && stats.size > maxBytes) { - store.removeDocument(db, relativePath); - return { skipped: true }; - } - if (store.isUpToDate(store.getIndexedDocument(db, relativePath), stats)) { return { unchanged: true }; } - const text = await readIndexableText(absolutePath, stats.size); - if (text === null || !text.trim()) { - store.removeDocument(db, relativePath); - return { skipped: true }; - } + const maxBytes = searchConfig?.maxFileSizeBytes ?? 0; + const text = + maxBytes && stats.size > maxBytes ? null : await readIndexableText(absolutePath, stats.size); + const indexable = text === null || !text.trim() ? null : capText(text); store.upsertDocument(db, { path: relativePath, mtimeMs: stats.mtimeMs, size: stats.size, - text: capText(text), + text: indexable, }); - return { indexed: true }; + return indexable ? { indexed: true } : { catalogued: true }; }; module.exports = { indexTree, indexFile, readIndexableText }; diff --git a/backend/src/services/settingsService.js b/backend/src/services/settingsService.js index b274ccd04..346b5e167 100644 --- a/backend/src/services/settingsService.js +++ b/backend/src/services/settingsService.js @@ -1,61 +1,238 @@ -const { getDb } = require('./db'); -const env = require('../config/env'); -const { parseByteSize } = require('../utils/env'); +const { getDb, prepared } = require('./db'); +const { cachedForRequest } = require('../utils/requestContext'); const { normalizeRelativePath } = require('../utils/pathUtils'); -const { ruleAppliesToAdmins } = require('../utils/accessRules'); +const { parseByteSize } = require('../utils/env'); +const env = require('../config/env'); const folderSizeExclusions = require('./folderSizeExclusions'); const searchIndexExclusions = require('./searchIndexExclusions'); -const storage = require('./storage/jsonStorage'); // Keep for backward compatibility fallback +const { generateId } = require('../utils/ids'); +const { ValidationError } = require('../errors/AppError'); +const { ruleAppliesToAdmins } = require('../utils/accessRules'); + +const MIN_UPLOAD_CHUNK_SIZE_BYTES = 1024 * 1024; +const HARD_MAX_UPLOAD_CHUNK_SIZE_MIB = 512; +const DEFAULT_UPLOAD_CHUNK_SIZE_BYTES = 8 * 1024 * 1024; + +// Per-folder preferences are kept per user, and bounded: one entry per folder +// ever visited would otherwise grow without limit. +const MAX_FOLDER_PREFERENCES = 100; +const MAX_FOLDER_PATH_LENGTH = 1024; +const MAX_SORT_FIELD_LENGTH = 128; + +// Admin-configurable upper bound (env MAX_CHUNK_SIZE_MIB), capped at the hard +// ceiling. Used to clamp both the default and any saved chunk size. +const resolveMaxChunkSizeBytes = () => { + const raw = Number(env.MAX_CHUNK_SIZE_MIB); + const mib = + Number.isFinite(raw) && raw >= 1 + ? Math.min(Math.floor(raw), HARD_MAX_UPLOAD_CHUNK_SIZE_MIB) + : HARD_MAX_UPLOAD_CHUNK_SIZE_MIB; + return Math.max(MIN_UPLOAD_CHUNK_SIZE_BYTES, mib * 1024 * 1024); +}; +const MAX_UPLOAD_CHUNK_SIZE_BYTES = resolveMaxChunkSizeBytes(); + +const clampNumber = (value, min, max) => Math.max(min, Math.min(max, value)); + +const defaultUploadSettings = () => { + const configuredChunkSize = parseByteSize(env.UPLOAD_CHUNK_SIZE); + const chunkSizeBytes = + Number.isFinite(configuredChunkSize) && configuredChunkSize > 0 + ? configuredChunkSize + : DEFAULT_UPLOAD_CHUNK_SIZE_BYTES; -const generateId = () => { - const crypto = require('crypto'); - return typeof crypto.randomUUID === 'function' - ? crypto.randomUUID() - : `${Date.now().toString(36)}-${crypto.randomBytes(8).toString('hex')}`; + const chunkedAutoFallback = env.UPLOAD_CHUNKED_AUTO_FALLBACK ?? false; + return { + // Auto-fallback and forced chunked uploads are mutually exclusive — auto is a + // direct-with-fallback mode, so it turns forced chunking off. + chunkedEnabled: chunkedAutoFallback ? false : (env.UPLOAD_CHUNKED_ENABLED ?? false), + chunkedAutoFallback, + chunkSizeBytes: clampNumber( + Math.floor(chunkSizeBytes), + MIN_UPLOAD_CHUNK_SIZE_BYTES, + MAX_UPLOAD_CHUNK_SIZE_BYTES + ), + }; +}; + +const isValidFolderPath = (folderPath) => + typeof folderPath === 'string' && + folderPath.length > 0 && + folderPath.length <= MAX_FOLDER_PATH_LENGTH; + +const sanitizeFolderSort = (sort) => { + if ( + !sort || + typeof sort !== 'object' || + typeof sort.by !== 'string' || + sort.by.trim().length === 0 || + sort.by.length > MAX_SORT_FIELD_LENGTH || + (sort.order !== 'asc' && sort.order !== 'desc') + ) { + return null; + } + + return { + by: sort.by.trim(), + order: sort.order, + updatedAt: Number.isFinite(sort.updatedAt) ? Math.floor(sort.updatedAt) : 0, + }; +}; + +const VIEW_MODES = ['grid', 'list', 'tab', 'photos']; + +/** A remembered view mode for one folder, or null when it is not one we have. */ +const sanitizeFolderView = (view) => { + const mode = typeof view === 'string' ? view : view?.mode; + if (!VIEW_MODES.includes(mode)) return null; + + return { + mode, + updatedAt: Number.isFinite(view?.updatedAt) ? Math.floor(view.updatedAt) : 0, + }; +}; + +/** + * A map of folder path to preference, keeping only what is valid and only the + * most recently used — one entry per folder ever visited would grow forever. + */ +const sanitizeFolderPreferences = (preferences, sanitizeEntry) => { + if (!preferences || typeof preferences !== 'object' || Array.isArray(preferences)) { + return {}; + } + + return Object.fromEntries( + Object.entries(preferences) + .map(([folderPath, entry]) => { + const sanitized = sanitizeEntry(entry); + return isValidFolderPath(folderPath) && sanitized ? [folderPath, sanitized] : null; + }) + .filter(Boolean) + .sort(([, a], [, b]) => b.updatedAt - a.updatedAt) + .slice(0, MAX_FOLDER_PREFERENCES) + ); +}; + +const sanitizeFolderSorts = (folderSorts) => + sanitizeFolderPreferences(folderSorts, sanitizeFolderSort); + +const sanitizeFolderViews = (folderViews) => + sanitizeFolderPreferences(folderViews, sanitizeFolderView); + +/** + * The bounds thumbnail settings are held to, and their defaults. The settings + * page refuses a value outside them before sending it, with the same numbers + * (`SettingsFilesThumbnails.vue`). + */ +const THUMBNAIL_BOUNDS = { + size: { min: 64, max: 1024, fallback: 200 }, + quality: { min: 1, max: 100, fallback: 70 }, + concurrency: { min: 1, max: 50, fallback: 10 }, }; /** * Sanitize thumbnail settings */ const sanitizeThumbnails = (thumbnails = {}) => { + const integer = (key) => { + const { min, max, fallback } = THUMBNAIL_BOUNDS[key]; + return Number.isFinite(thumbnails[key]) + ? clampNumber(Math.floor(thumbnails[key]), min, max) + : fallback; + }; return { enabled: typeof thumbnails.enabled === 'boolean' ? thumbnails.enabled : true, - size: Number.isFinite(thumbnails.size) - ? Math.max(64, Math.min(1024, Math.floor(thumbnails.size))) - : 200, - quality: Number.isFinite(thumbnails.quality) - ? Math.max(1, Math.min(100, Math.floor(thumbnails.quality))) - : 70, - concurrency: Number.isFinite(thumbnails.concurrency) - ? Math.max(1, Math.min(50, Math.floor(thumbnails.concurrency))) - : 10, + size: integer('size'), + quality: integer('quality'), + concurrency: integer('concurrency'), }; }; +const FOLDER_SIZE_MODES = ['off', 'shallow', 'full']; + /** - * Sanitize access control rules + * What an administrator chose for the two background workers. Only a choice: + * when the environment set the same thing, the environment is what runs, and + * this is kept for the day the variable is taken away. */ -const sanitizeAccessRules = (rules = []) => { - if (!Array.isArray(rules)) return []; +const sanitizeFolderSize = (folderSize = {}) => ({ + excludedPaths: folderSizeExclusions.sanitizePaths(folderSize.excludedPaths || []), + mode: FOLDER_SIZE_MODES.includes(folderSize.mode) ? folderSize.mode : 'off', +}); + +const sanitizeSearchIndex = (searchIndex = {}) => ({ + excludedPaths: searchIndexExclusions.sanitizePaths(searchIndex.excludedPaths || []), + enabled: searchIndex.enabled === true, +}); + +const ACCESS_PERMISSIONS = ['rw', 'ro', 'hidden']; + +/** + * Sanitize access control rules. + * + * Read back (`strict: false`), a rule that cannot stand is dropped. Anything + * else would make one bad row — left by an older version, or edited into + * app.db by hand — unreadable settings, and unreadable settings are every + * hidden folder visible to everybody. + * + * Saved (`strict: true`), the same rule is refused with its reason and nothing + * is written. Dropping it silently answered 200 with a list the page then + * adopted: the row for `../Secret` disappeared the moment it was saved, and an + * administrator was left believing a folder was hidden that never was. The + * permissions were worse — anything not one of the three became `rw`, so a + * mistyped `readonly` opened a folder for writing instead of refusing the word. + */ +const sanitizeAccessRules = (rules = [], { strict = false } = {}) => { + if (!Array.isArray(rules)) { + if (strict) throw new ValidationError('The access rules have to be sent as a list.'); + return []; + } return rules - .map((rule) => { - if (!rule || typeof rule !== 'object') return null; + .map((rule, index) => { + // Numbered as the page numbers them, so the reason names the row. + const refuse = (reason) => { + if (!strict) return null; + throw new ValidationError(`Access rule ${index + 1}: ${reason}`); + }; + + if (!rule || typeof rule !== 'object' || Array.isArray(rule)) { + return refuse('this is not a rule.'); + } + + // A path of nothing but spaces normalises to itself: the rule was stored + // as it came and matched no folder — written by an administrator, listed + // on the page, and doing nothing. Refused now, and only when it is blank + // all through: a folder may legitimately be called "My Documents", or + // even " x ", so nothing here trims what somebody wrote. + if (!String(rule.path ?? '').trim()) return refuse('a rule needs the path of a folder.'); // Validate path let normalizedPath; try { normalizedPath = normalizeRelativePath(rule.path || ''); - } catch { - return null; // Invalid path + } catch (error) { + return refuse(`"${rule.path}" is not a folder path. ${error.message}`); } - if (!normalizedPath) return null; + if (!normalizedPath) return refuse('a rule needs the path of a folder.'); // Validate permissions - const permissions = ['rw', 'ro', 'hidden'].includes(rule.permissions) - ? rule.permissions - : 'rw'; + if (rule.permissions !== undefined && !ACCESS_PERMISSIONS.includes(rule.permissions)) { + return refuse( + `"${rule.permissions}" is not one of the permissions a rule gives: rw, ro or hidden.` + ); + } + const permissions = ACCESS_PERMISSIONS.includes(rule.permissions) ? rule.permissions : 'rw'; + + if (rule.recursive !== undefined && typeof rule.recursive !== 'boolean') { + return refuse(`"${rule.recursive}" does not say whether the rule covers what is inside.`); + } + + if (rule.appliesToAdmins !== undefined && typeof rule.appliesToAdmins !== 'boolean') { + return refuse( + `"${rule.appliesToAdmins}" does not say whether the rule holds administrators too.` + ); + } return { id: rule.id || `${Date.now()}-${Math.random().toString(36).slice(2)}`, @@ -63,8 +240,8 @@ const sanitizeAccessRules = (rules = []) => { recursive: Boolean(rule.recursive), permissions, // Stored as a plain yes or no, so the page shows a definite box and - // nothing downstream has to guess again. What a rule written before - // this switch existed means is decided in one place, utils/accessRules. + // nothing has to guess again. What a rule written before this switch + // existed means is decided in one place, utils/accessRules. appliesToAdmins: ruleAppliesToAdmins({ ...rule, permissions }), }; }) @@ -72,15 +249,19 @@ const sanitizeAccessRules = (rules = []) => { }; /** - * The access section: the rules, and whether they hold administrators. - * - * Kept together because the two are read together — a rule says whether it - * holds administrators, and this setting holds them to all of them at once. + * The access section: the rules, and whether every one of them also holds + * administrators. The setting is the blunt one — on, no rule lets an + * administrator through; off, each rule says for itself. */ -const sanitizeAccess = (access = {}) => { +const sanitizeAccess = (access = {}, { strict = false } = {}) => { const source = access && typeof access === 'object' && !Array.isArray(access) ? access : {}; + if (source.applyToAdmins !== undefined && typeof source.applyToAdmins !== 'boolean' && strict) { + throw new ValidationError( + 'Whether the rules hold administrators too has to be sent as true or false.' + ); + } return { - rules: sanitizeAccessRules(source.rules || []), + rules: sanitizeAccessRules(source.rules || [], { strict }), applyToAdmins: source.applyToAdmins === true, }; }; @@ -89,9 +270,12 @@ const sanitizeAccess = (access = {}) => { * Sanitize branding settings */ const sanitizeBranding = (branding = {}) => { + // A name of nothing but spaces was stored as it came, and the header and the + // sign-in page showed no name at all. One stored that way reads as the + // default, so an installation that saved one needs nothing done. + const appName = typeof branding.appName === 'string' ? branding.appName.trim().slice(0, 100) : ''; return { - appName: - typeof branding.appName === 'string' ? branding.appName.trim().slice(0, 100) : 'Explorer', + appName: appName || 'Explorer', appLogoUrl: typeof branding.appLogoUrl === 'string' ? branding.appLogoUrl.trim().slice(0, 500) @@ -100,6 +284,39 @@ const sanitizeBranding = (branding = {}) => { }; }; +/** + * Sanitize upload settings + */ +const sanitizeUploads = (uploads = {}) => { + const defaults = defaultUploadSettings(); + const rawChunkSize = + typeof uploads.chunkSizeBytes === 'string' + ? parseByteSize(uploads.chunkSizeBytes) + : uploads.chunkSizeBytes; + + const chunkedAutoFallback = + typeof uploads.chunkedAutoFallback === 'boolean' + ? uploads.chunkedAutoFallback + : defaults.chunkedAutoFallback; + const chunkedEnabled = chunkedAutoFallback + ? false // mutually exclusive with auto-fallback (auto wins) + : typeof uploads.chunkedEnabled === 'boolean' + ? uploads.chunkedEnabled + : defaults.chunkedEnabled; + + return { + chunkedEnabled, + chunkedAutoFallback, + chunkSizeBytes: Number.isFinite(rawChunkSize) + ? clampNumber( + Math.floor(rawChunkSize), + MIN_UPLOAD_CHUNK_SIZE_BYTES, + MAX_UPLOAD_CHUNK_SIZE_BYTES + ) + : defaults.chunkSizeBytes, + }; +}; + /** * The trash settings in force: on or off, how many days an item is kept, and * how much of a volume the trash may hold — a share of it, capped by a size @@ -107,14 +324,10 @@ const sanitizeBranding = (branding = {}) => { * out keeps the default the environment gave. */ const sanitizeTrash = (trash = {}) => { - // eslint-disable-next-line global-require const { trash: defaults } = require('../config/index'); - // eslint-disable-next-line global-require - const { parseByteSize } = require('../utils/env'); const source = trash && typeof trash === 'object' ? trash : {}; - const clamp = (value, min, max) => Math.min(max, Math.max(min, value)); const integerIn = (value, min, max, fallback) => - Number.isFinite(value) ? clamp(Math.round(value), min, max) : fallback; + Number.isFinite(value) ? clampNumber(Math.round(value), min, max) : fallback; const rawMaxBytes = typeof source.maxBytes === 'string' ? parseByteSize(source.maxBytes) : source.maxBytes; @@ -131,20 +344,40 @@ const sanitizeTrash = (trash = {}) => { }; /** - * The file-version settings in force: whether a save keeps what it replaces, - * and the retention thinning (everything for a while, then hourly, then daily), - * a per-file cap and a session-checkpoint gap. Out-of-range values are clamped, - * and the windows are kept consistent (hourly covers keep-all, daily covers - * hourly), so the policy never contradicts itself. + * The activity log settings in force: on or off, and how long a line is kept. + * + * Off is the default and stays the default: a log nobody asked for is a record + * of somebody's day that nobody reads. + */ +const sanitizeActivity = (activity = {}) => { + const { activity: defaults } = require('../config/index'); + const source = activity && typeof activity === 'object' ? activity : {}; + const retentionDays = Number(source.retentionDays); + return { + enabled: typeof source.enabled === 'boolean' ? source.enabled : defaults.enabled, + retentionDays: Number.isFinite(retentionDays) + ? clampNumber(Math.round(retentionDays), 1, 3650) + : defaults.retentionDays, + }; +}; + +/** + * The file version settings in force: on or off, how long everything is kept + * before thinning starts, how long one an hour and one a day are kept, how many + * versions a file keeps at most, and how often an editing session leaves a + * checkpoint. The space they may take is the trash's: one budget per volume. + * + * The tiers are kept in order — a week of hourly versions cannot end before the + * day of keeping everything does. */ const sanitizeVersions = (versions = {}) => { - // eslint-disable-next-line global-require const { versions: defaults, VERSION_BOUNDS } = require('../config/index'); const source = versions && typeof versions === 'object' ? versions : {}; - const clamp = (value, min, max) => Math.min(max, Math.max(min, value)); const integer = (key) => { const [min, max] = VERSION_BOUNDS[key]; - return Number.isFinite(source[key]) ? clamp(Math.round(source[key]), min, max) : defaults[key]; + return Number.isFinite(source[key]) + ? clampNumber(Math.round(source[key]), min, max) + : defaults[key]; }; const keepAllHours = integer('keepAllHours'); const hourlyDays = Math.max(integer('hourlyDays'), Math.ceil(keepAllHours / 24)); @@ -159,248 +392,146 @@ const sanitizeVersions = (versions = {}) => { }; }; -const FOLDER_SIZE_MODES = ['off', 'shallow', 'full']; - -/** - * What an administrator chose for the two background workers. Only a choice: - * when the environment set the same thing, the environment is what runs, and - * this is kept for the day the variable is taken away. - */ -const sanitizeFolderSize = (folderSize = {}) => ({ - excludedPaths: folderSizeExclusions.sanitizePaths(folderSize.excludedPaths || []), - mode: FOLDER_SIZE_MODES.includes(folderSize.mode) ? folderSize.mode : 'off', -}); - -const sanitizeSearchIndex = (searchIndex = {}) => ({ - excludedPaths: searchIndexExclusions.sanitizePaths(searchIndex.excludedPaths || []), - enabled: searchIndex.enabled === true, -}); - -/** - * The activity log settings in force: on or off, and how long a line is kept. - * - * Off is the default and stays the default: a log nobody asked for is a record - * of somebody's day that nobody reads. - */ -const sanitizeActivity = (activity = {}) => { - // eslint-disable-next-line global-require - const { activity: defaults } = require('../config/index'); - const source = activity && typeof activity === 'object' ? activity : {}; - const retentionDays = Number(source.retentionDays); - return { - enabled: typeof source.enabled === 'boolean' ? source.enabled : defaults.enabled, - retentionDays: Number.isFinite(retentionDays) - ? Math.max(1, Math.min(3650, Math.round(retentionDays))) - : defaults.retentionDays, - }; -}; - /** * Get public settings (branding only, no auth required) */ const getPublicSettings = async () => { - try { - const db = await getDb(); - const brandingRow = db - .prepare('SELECT value FROM system_settings WHERE category = ? AND key = ?') - .get('branding', 'branding'); + const db = await getDb(); + const brandingRow = db + .prepare('SELECT value FROM system_settings WHERE category = ? AND key = ?') + .get('branding', 'branding'); - if (brandingRow) { - const branding = JSON.parse(brandingRow.value); - return { - branding: sanitizeBranding(branding), - }; + let branding = {}; + if (brandingRow) { + try { + branding = JSON.parse(brandingRow.value); + } catch { + // An unreadable value is the default branding, not a failure to sign in. } - } catch (err) { - // Fallback to JSON if DB read fails - } - - // Fallback to JSON storage - try { - const data = await storage.get(); - const branding = data.settings?.branding || {}; - return { - branding: sanitizeBranding(branding), - }; - } catch (err) { - // Return defaults if all else fails - return { - branding: sanitizeBranding({}), - }; } + return { branding: sanitizeBranding(branding) }; }; /** * Get user-specific settings */ -const MIN_UPLOAD_CHUNK_SIZE_BYTES = 1024 * 1024; -const HARD_MAX_UPLOAD_CHUNK_SIZE_MIB = 512; -const DEFAULT_UPLOAD_CHUNK_SIZE_BYTES = 8 * 1024 * 1024; - -// The administrator's ceiling (MAX_CHUNK_SIZE_MIB), itself capped: a chunk is -// held whole in memory at each end, so an unbounded one is a way to run a -// server out of it. -const resolveMaxChunkSizeBytes = () => { - const raw = Number(env.MAX_CHUNK_SIZE_MIB); - const mib = - Number.isFinite(raw) && raw > 0 - ? Math.min(Math.floor(raw), HARD_MAX_UPLOAD_CHUNK_SIZE_MIB) - : HARD_MAX_UPLOAD_CHUNK_SIZE_MIB; - return Math.max(MIN_UPLOAD_CHUNK_SIZE_BYTES, mib * 1024 * 1024); -}; -const MAX_UPLOAD_CHUNK_SIZE_BYTES = resolveMaxChunkSizeBytes(); - -const clampNumber = (value, min, max) => Math.max(min, Math.min(max, value)); - -const defaultUploadSettings = () => { - const configuredChunkSize = parseByteSize(env.UPLOAD_CHUNK_SIZE); - const chunkSizeBytes = - Number.isFinite(configuredChunkSize) && configuredChunkSize > 0 - ? configuredChunkSize - : DEFAULT_UPLOAD_CHUNK_SIZE_BYTES; - - return { - chunkedEnabled: env.UPLOAD_CHUNKED_ENABLED ?? false, - chunkSizeBytes: clampNumber( - Math.floor(chunkSizeBytes), - MIN_UPLOAD_CHUNK_SIZE_BYTES, - MAX_UPLOAD_CHUNK_SIZE_BYTES - ), - }; -}; - -const sanitizeUploads = (uploads = {}) => { - const defaults = defaultUploadSettings(); - const rawChunkSize = - typeof uploads.chunkSizeBytes === 'string' - ? parseByteSize(uploads.chunkSizeBytes) - : uploads.chunkSizeBytes; - - return { - chunkedEnabled: - typeof uploads.chunkedEnabled === 'boolean' - ? uploads.chunkedEnabled - : defaults.chunkedEnabled, - chunkSizeBytes: Number.isFinite(rawChunkSize) - ? clampNumber( - Math.floor(rawChunkSize), - MIN_UPLOAD_CHUNK_SIZE_BYTES, - MAX_UPLOAD_CHUNK_SIZE_BYTES - ) - : defaults.chunkSizeBytes, - }; -}; - const getUserSettings = async (userId) => { if (!userId) return {}; try { const db = await getDb(); - const rows = db.prepare('SELECT key, value FROM user_settings WHERE user_id = ?').all(userId); + const rows = prepared(db, 'SELECT key, value FROM user_settings WHERE user_id = ?').all(userId); const settings = {}; for (const row of rows) { try { settings[row.key] = JSON.parse(row.value); - } catch (err) { + } catch (_) { // Skip invalid JSON } } + // Per-folder preferences are rows of their own now, but the client still + // receives them among the user's settings. + Object.assign(settings, await getUserFolderPreferences(userId)); + return settings; - } catch (err) { + } catch (_) { return {}; } }; +// Through `prepared` rather than db.prepare: these run on every preference +// change, and recompiling the same three statements each time is waste the +// rest of this file already avoids. +const upsertUserSetting = (db, userId, key, value) => { + const now = new Date().toISOString(); + const valueJson = JSON.stringify(value); + const existing = prepared(db, 'SELECT id FROM user_settings WHERE user_id = ? AND key = ?').get( + userId, + key + ); + + if (existing) { + prepared( + db, + 'UPDATE user_settings SET value = ?, updated_at = ? WHERE user_id = ? AND key = ?' + ).run(valueJson, now, userId, key); + } else { + prepared( + db, + 'INSERT INTO user_settings (id, user_id, key, value, updated_at) VALUES (?, ?, ?, ?, ?)' + ).run(generateId(), userId, key, valueJson, now); + } +}; + /** * Get system settings (admin only) */ +/** + * System settings, read from app.db and nowhere else. + * + * They used to fall back to app-config.json whenever the read failed. That file + * stopped following the settings long ago — the screens save to app.db alone — + * so a read that failed ran with whatever the file last held, often no access + * rules at all: a folder hidden by a rule opened for everyone for as long as the + * database could not be read. A read that fails now fails the request. + */ const getSystemSettings = async () => { - try { - const db = await getDb(); - const rows = db - .prepare('SELECT key, value FROM system_settings WHERE category = ?') - .all('system'); - - const thumbnails = { enabled: true, size: 200, quality: 70, concurrency: 10 }; - const access = { rules: [] }; - let trash = {}; - let versions = {}; - let uploads = {}; - let activity = {}; - let folderSize = {}; - let searchIndex = {}; - - for (const row of rows) { - try { - if (row.key === 'thumbnails') { - Object.assign(thumbnails, JSON.parse(row.value)); - } else if (row.key === 'access') { - Object.assign(access, JSON.parse(row.value)); - } else if (row.key === 'trash') { - trash = JSON.parse(row.value); - } else if (row.key === 'versions') { - versions = JSON.parse(row.value); - } else if (row.key === 'uploads') { - uploads = JSON.parse(row.value); - } else if (row.key === 'activity') { - activity = JSON.parse(row.value); - } else if (row.key === 'folderSize') { - folderSize = JSON.parse(row.value); - } else if (row.key === 'searchIndex') { - searchIndex = JSON.parse(row.value); - } - } catch (err) { - // Skip invalid JSON - } - } - - return { - thumbnails: sanitizeThumbnails(thumbnails), - access: sanitizeAccess(access), - trash: sanitizeTrash(trash), - versions: sanitizeVersions(versions), - uploads: sanitizeUploads(uploads), - activity: sanitizeActivity(activity), - folderSize: { - ...sanitizeFolderSize(folderSize), - environmentExcludedPaths: folderSizeExclusions.snapshot().environmentExcludedPaths, - }, - searchIndex: { - ...sanitizeSearchIndex(searchIndex), - environmentExcludedPaths: searchIndexExclusions.snapshot().environmentExcludedPaths, - }, - }; - } catch (err) { - // Fallback to JSON storage + const db = await getDb(); + const rows = db + .prepare('SELECT key, value FROM system_settings WHERE category = ?') + .all('system'); + + const thumbnails = { enabled: true, size: 200, quality: 70, concurrency: 10 }; + const access = { rules: [] }; + let uploads = defaultUploadSettings(); + const folderSize = { excludedPaths: [] }; + const searchIndex = { excludedPaths: [] }; + const trash = {}; + const versions = {}; + const activity = {}; + + for (const row of rows) { try { - const data = await storage.get(); - const settings = data.settings || {}; - return { - thumbnails: sanitizeThumbnails(settings.thumbnails), - access: sanitizeAccess(settings.access), - trash: sanitizeTrash(settings.trash), - versions: sanitizeVersions(settings.versions), - uploads: sanitizeUploads(settings.uploads), - activity: sanitizeActivity(settings.activity), - folderSize: sanitizeFolderSize(settings.folderSize), - searchIndex: sanitizeSearchIndex(settings.searchIndex), - }; - } catch (err2) { - // Return defaults - return { - thumbnails: sanitizeThumbnails({}), - access: sanitizeAccess({}), - trash: sanitizeTrash({}), - versions: sanitizeVersions({}), - uploads: sanitizeUploads({}), - activity: sanitizeActivity({}), - folderSize: sanitizeFolderSize({}), - searchIndex: sanitizeSearchIndex({}), - }; + if (row.key === 'thumbnails') { + Object.assign(thumbnails, JSON.parse(row.value)); + } else if (row.key === 'access') { + Object.assign(access, JSON.parse(row.value)); + } else if (row.key === 'uploads') { + uploads = { ...uploads, ...JSON.parse(row.value) }; + } else if (row.key === 'folderSize') { + Object.assign(folderSize, JSON.parse(row.value)); + } else if (row.key === 'searchIndex') { + Object.assign(searchIndex, JSON.parse(row.value)); + } else if (row.key === 'trash') { + Object.assign(trash, JSON.parse(row.value)); + } else if (row.key === 'versions') { + Object.assign(versions, JSON.parse(row.value)); + } else if (row.key === 'activity') { + Object.assign(activity, JSON.parse(row.value)); + } + } catch (_) { + // Skip invalid JSON } } + + return { + thumbnails: sanitizeThumbnails(thumbnails), + access: sanitizeAccess(access), + uploads: sanitizeUploads(uploads), + trash: sanitizeTrash(trash), + versions: sanitizeVersions(versions), + activity: sanitizeActivity(activity), + folderSize: { + ...sanitizeFolderSize(folderSize), + environmentExcludedPaths: folderSizeExclusions.snapshot().environmentExcludedPaths, + }, + searchIndex: { + ...sanitizeSearchIndex(searchIndex), + environmentExcludedPaths: searchIndexExclusions.snapshot().environmentExcludedPaths, + }, + }; }; /** @@ -418,18 +549,18 @@ const getSettingsForUser = async (user) => { if (user && user.id) { const userSettings = await getUserSettings(user.id); result.user = userSettings; + const systemSettings = await getSystemSettings(); + result.uploads = systemSettings.uploads; const isAdmin = Array.isArray(user.roles) && user.roles.includes('admin'); if (isAdmin) { - const systemSettings = await getSystemSettings(); result.thumbnails = systemSettings.thumbnails; result.access = systemSettings.access; + result.folderSize = systemSettings.folderSize; + result.searchIndex = systemSettings.searchIndex; result.trash = systemSettings.trash; result.versions = systemSettings.versions; - result.uploads = systemSettings.uploads; result.activity = systemSettings.activity; - result.folderSize = systemSettings.folderSize; - result.searchIndex = systemSettings.searchIndex; } } @@ -437,30 +568,64 @@ const getSettingsForUser = async (user) => { }; /** - * The preferences an account may set, in one place. + * Anything that is not a boolean is not an answer, and answers undefined, so + * the stored value stays. + * + * It used to be `Boolean(value)`, which has an opinion about everything: + * `'false'` — what a form field, a query string or a shell client sends — was + * true, and `0` was false. Either way the switch was set to something nobody + * had chosen, and the answer said it had been saved. + */ +const asBoolean = (value) => (typeof value === 'boolean' ? value : undefined); + +// null means "no answer of my own": for skipHome, defer to the environment. +const asNullableBoolean = (value) => { + if (value === null || value === undefined) return null; + return typeof value === 'boolean' ? value : undefined; +}; + +/** + * A default share expiry: null for none, or a whole number of at least one + * with its unit. + * + * Anything else is not an expiry, and answers undefined, so the stored one + * stays. It used to answer null, which is a value here: a default of minus + * three weeks, or of three years, silently removed the default the person had. + */ +const asShareExpiration = (value) => { + if (value === null || value === undefined) return null; + if (typeof value !== 'object') return undefined; + const validUnits = ['days', 'weeks', 'months']; + const amount = Number.isFinite(value.value) ? Math.floor(value.value) : 0; + if (amount < 1 || !validUnits.includes(value.unit)) return undefined; + return { value: amount, unit: value.unit }; +}; + +/** + * The view a folder gets when it has none of its own (#360). + * + * null is a value here, and means "use the built-in default". A mode we do not + * have is not: it used to become null too, so one unknown word put every + * folder back to the built-in view instead of being refused. + */ +const asViewMode = (value) => { + if (value === null || value === undefined) return null; + return VIEW_MODES.includes(value) ? value : undefined; +}; + +/** + * A language tag, or null to follow the browser. * - * There used to be two lists: this one, which decides how a value is - * sanitised, and another inside the settings route, which decides whether the - * key is written at all. Adding a preference to one and not the other produced - * a toggle that moved on screen, answered success, and stored nothing — so the - * two are the same list now, and the route asks here. - */ -const USER_BOOLEAN_SETTINGS = new Set([ - 'showHiddenFiles', - 'showThumbnails', - 'showVersionMarks', - 'documentsOpenInNewTab', - 'showSidebarFavorites', - 'showSidebarShares', - 'showSidebarTools', -]); - -/** - * A language tag, or null for "follow the browser". + * Checked for its shape and not against a list of the languages that exist: + * the translations are the interface's, and a second list here would be a + * second truth to keep — one locale added there and forgotten here would be + * refused for no reason anybody could see. A tag naming a translation nobody + * ships is stored and then falls back to the browser, which is what a reader + * whose language is gone should get anyway. * - * Checked for shape rather than against the list of translations: the list - * changes with a release, and a stored tag we no longer ship should fall back - * on screen, not be refused on the way in. + * Anything that is not a tag at all is refused rather than turned into null, + * as a view mode is: a typo would otherwise read as "follow the browser" and + * the choice would put itself back where it was. */ const LANGUAGE_TAG = /^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/; const asLocale = (value) => { @@ -470,12 +635,46 @@ const asLocale = (value) => { return LANGUAGE_TAG.test(tag) ? tag : undefined; }; -const USER_SETTING_KEYS = new Set([ - ...USER_BOOLEAN_SETTINGS, - 'defaultShareExpiration', - 'skipHome', - 'locale', -]); +/** + * Every preference a user may set, each with the coercion that belongs to it. + * + * One line per preference, in one place, because this used to be spread over + * three: a list of allowed keys in the settings route, a chain of if/else + * sanitising here, and the defaults in the client store. A key present in one + * and missing from another was accepted by the API, silently dropped, and + * answered with its previous value — which the client then applied, so the + * switch flicked itself back off. `markdownOpensInEditor` did exactly that. + * + * Adding a preference is now adding a line here. Its name and its validation + * cannot come apart, because they are the same line. + */ +const USER_SETTINGS = { + showHiddenFiles: asBoolean, + showThumbnails: asBoolean, + showSidebarFavorites: asBoolean, + showSidebarShares: asBoolean, + showSidebarTools: asBoolean, + markdownOpensInEditor: asBoolean, + documentsOpenInNewTab: asBoolean, + showVersionMarks: asBoolean, + defaultShareExpiration: asShareExpiration, + skipHome: asNullableBoolean, + defaultView: asViewMode, + locale: asLocale, +}; + +/** + * Written by the application, never straight from a request: a folder + * preference is saved one folder at a time, so that two tabs on different + * folders do not overwrite each other with whole maps. + */ +const INTERNAL_USER_SETTINGS = { + folderSorts: sanitizeFolderSorts, + folderViews: sanitizeFolderViews, +}; + +/** What PATCH /api/settings accepts under `user`. */ +const WRITABLE_USER_SETTINGS = new Set(Object.keys(USER_SETTINGS)); /** * Set a user setting @@ -484,146 +683,213 @@ const setUserSetting = async (userId, key, value) => { if (!userId) { throw new Error('User ID is required'); } - const db = await getDb(); - const now = new Date().toISOString(); - // Validate and sanitize value based on key - let sanitizedValue = value; - if (USER_BOOLEAN_SETTINGS.has(key)) { - sanitizedValue = Boolean(value); - } else if (key === 'locale') { - const tag = asLocale(value); - // `undefined` means "not a language tag": the stored value is left alone - // rather than replaced by something the interface cannot read. - if (tag === undefined) return (await getUserSettings(userId))[key]; - sanitizedValue = tag; - } else if (key === 'defaultShareExpiration') { - // Validate expiration object: { value: number, unit: 'days'|'weeks'|'months' } or null - if (value === null || value === undefined) { - sanitizedValue = null; - } else if (typeof value === 'object' && value !== null) { - const validUnits = ['days', 'weeks', 'months']; - const unit = validUnits.includes(value.unit) ? value.unit : 'weeks'; - const numValue = - Number.isFinite(value.value) && value.value > 0 ? Math.floor(value.value) : null; - sanitizedValue = numValue ? { value: numValue, unit } : null; - } else { - sanitizedValue = null; - } - } else if (key === 'skipHome') { - // Can be null (use env), true, or false - if (value === null || value === undefined) { - sanitizedValue = null; - } else { - sanitizedValue = Boolean(value); - } - } + // An unknown key is stored as it came: callers are the application itself, + // and the route only ever passes what WRITABLE_USER_SETTINGS allows. + const sanitize = USER_SETTINGS[key] || INTERNAL_USER_SETTINGS[key]; + const sanitizedValue = sanitize ? sanitize(value) : value; - const valueJson = JSON.stringify(sanitizedValue); - - // Check if setting exists - const existing = db - .prepare('SELECT id FROM user_settings WHERE user_id = ? AND key = ?') - .get(userId, key); + // What a preference cannot take is left out rather than stored as its + // default, as a section field of the wrong shape is: the stored value stays. + if (sanitizedValue === undefined) return undefined; - if (existing) { - db.prepare( - 'UPDATE user_settings SET value = ?, updated_at = ? WHERE user_id = ? AND key = ?' - ).run(valueJson, now, userId, key); - } else { - db.prepare( - 'INSERT INTO user_settings (id, user_id, key, value, updated_at) VALUES (?, ?, ?, ?, ?)' - ).run(generateId(), userId, key, valueJson, now); - } + upsertUserSetting(db, userId, key, sanitizedValue); return sanitizedValue; }; /** - * Set a system setting (admin only) - */ -/** - * Change the branding, and answer what it was and what it is now. + * Remember one folder's preference, and return the whole map back. * - * Read and written without yielding in between — the database answers - * synchronously — so two saves at once cannot both start from the same branding: - * the logo a save replaced is the one it was the last to see, and removing it - * cannot take away the logo another save has just put in place. - * - * @returns {Promise<{previous: object, current: object}>} + * Written one folder at a time rather than by sending the map: two tabs open + * on different folders would otherwise overwrite each other with whichever + * copy was saved last. The stored map is re-read here so the entry joins what + * is already there. */ -const replaceBranding = async (update) => { - const db = await getDb(); - const row = db - .prepare('SELECT value FROM system_settings WHERE category = ? AND key = ?') - .get('branding', 'branding'); +/** Every folder preference this user has, as the client expects them. */ +const getUserFolderPreferences = async (userId) => { + if (!userId) return { folderSorts: {}, folderViews: {} }; - let stored = {}; - if (row) { - try { - stored = JSON.parse(row.value); - } catch { - // An unreadable value is the default branding. + const db = await getDb(); + const rows = prepared( + db, + 'SELECT path, sort_by, sort_order, view_mode, updated_at FROM folder_preferences WHERE user_id = ?' + ).all(userId); + + const folderSorts = {}; + const folderViews = {}; + for (const row of rows) { + const updatedAt = Date.parse(row.updated_at) || 0; + if (row.sort_by) { + folderSorts[row.path] = { + by: row.sort_by, + order: row.sort_order === 'desc' ? 'desc' : 'asc', + updatedAt, + }; + } + if (row.view_mode) { + folderViews[row.path] = { mode: row.view_mode, updatedAt }; } } - const previous = sanitizeBranding(stored); - const current = sanitizeBranding({ ...previous, ...update }); + return { folderSorts, folderViews }; +}; - const now = new Date().toISOString(); - const valueJson = JSON.stringify(current); - const existing = db - .prepare('SELECT id FROM system_settings WHERE category = ? AND key = ?') - .get('branding', 'branding'); - if (existing) { - db.prepare( - 'UPDATE system_settings SET value = ?, updated_at = ? WHERE category = ? AND key = ?' - ).run(valueJson, now, 'branding', 'branding'); - } else { - db.prepare( - 'INSERT INTO system_settings (id, category, key, value, updated_at) VALUES (?, ?, ?, ?, ?)' - ).run(generateId(), 'branding', 'branding', valueJson, now); +/** + * Remember one folder's sort or view. + * + * One row per folder, so a change touches only that folder: two tabs on + * different folders no longer overwrite each other, and there is no ceiling on + * how many folders can be remembered. The row carries both preferences, so + * setting one must not erase the other. + */ +const setUserFolderPreference = async (userId, folderPath, { sort, view }) => { + if (!userId) { + throw new Error('User ID is required'); } - return { previous, current }; + const normalizedPath = normalizeRelativePath(folderPath); + const sanitizedSort = sort === undefined ? undefined : sanitizeFolderSort(sort); + const sanitizedView = view === undefined ? undefined : sanitizeFolderView(view); + + if (!isValidFolderPath(normalizedPath) || (!sanitizedSort && !sanitizedView)) { + return null; + } + + const db = await getDb(); + const now = new Date().toISOString(); + + prepared( + db, + `INSERT INTO folder_preferences (user_id, path, sort_by, sort_order, view_mode, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(user_id, path) DO UPDATE SET + sort_by = COALESCE(excluded.sort_by, folder_preferences.sort_by), + sort_order = COALESCE(excluded.sort_order, folder_preferences.sort_order), + view_mode = COALESCE(excluded.view_mode, folder_preferences.view_mode), + updated_at = excluded.updated_at` + ).run( + userId, + normalizedPath, + sanitizedSort?.by ?? null, + sanitizedSort?.order ?? null, + sanitizedView?.mode ?? null, + now + ); + + return getUserFolderPreferences(userId); }; -const setSystemSetting = async (category, key, value) => { +const setUserFolderSort = async (userId, folderPath, sort) => { + const preferences = await setUserFolderPreference(userId, folderPath, { sort }); + return preferences?.folderSorts ?? null; +}; + +const setUserFolderView = async (userId, folderPath, view) => { + const preferences = await setUserFolderPreference(userId, folderPath, { view }); + return preferences?.folderViews ?? null; +}; + +const assertSystemCategory = (category) => { if (category !== 'branding' && category !== 'system') { throw new Error('Invalid category. Must be "branding" or "system"'); } +}; + +/** + * What a section is held to before it is stored, by key. + * + * The same shaping a read applies, so a section merged over the row itself + * comes out as it would have come out of the settings: a field nobody sent + * takes the sanitiser's default, which is the one a read would have given it. + */ +const sanitizeSystemSetting = (key, value) => { + if (key === 'thumbnails') return sanitizeThumbnails(value); + // Strict: what is being stored was just written by somebody, and a rule that + // cannot be stored as they wrote it is answered rather than dropped. + if (key === 'access') return sanitizeAccess(value, { strict: true }); + if (key === 'uploads') return sanitizeUploads(value); + if (key === 'branding') return sanitizeBranding(value); + if (key === 'folderSize') return sanitizeFolderSize(value); + // The search index had no case here, so what was stored for it was the + // merge as it came: paths with spaces around them, empty entries, the same + // folder twice. The worker was handed a sanitised copy and behaved, so only + // the stored value was wrong — and it is the one the next merge starts from. + if (key === 'searchIndex') return sanitizeSearchIndex(value); + if (key === 'trash') return sanitizeTrash(value); + if (key === 'activity') return sanitizeActivity(value); + if (key === 'versions') return sanitizeVersions(value); + return value; +}; + +/** + * What a section would be stored as, without storing it. + * + * The route checks every section of a save before writing any of them, so a + * section that refuses what it was sent refuses before another has been + * stored. + */ +const checkSystemSection = (key, value) => sanitizeSystemSetting(key, value); + +/** + * Set a system setting (admin only) + */ +const setSystemSetting = async (category, key, value) => { + assertSystemCategory(category); const db = await getDb(); - const now = new Date().toISOString(); + const sanitizedValue = sanitizeSystemSetting(key, value); + + writeSystemSetting(db, category, key, sanitizedValue); + + return sanitizedValue; +}; + +/** One section as it is stored, before any default is put around it. */ +const readStoredSection = (db, category, key) => { + const row = db + .prepare('SELECT value FROM system_settings WHERE category = ? AND key = ?') + .get(category, key); + if (!row) return {}; - // Sanitize based on key - let sanitizedValue = value; - if (key === 'thumbnails') { - sanitizedValue = sanitizeThumbnails(value); - } else if (key === 'access') { - sanitizedValue = sanitizeAccess(value); - } else if (key === 'branding') { - sanitizedValue = sanitizeBranding(value); - } else if (key === 'trash') { - sanitizedValue = sanitizeTrash(value); - } else if (key === 'versions') { - sanitizedValue = sanitizeVersions(value); - } else if (key === 'uploads') { - sanitizedValue = sanitizeUploads(value); - } else if (key === 'activity') { - sanitizedValue = sanitizeActivity(value); - } else if (key === 'folderSize') { - sanitizedValue = sanitizeFolderSize(value); - } else if (key === 'searchIndex') { - // The search index had no case here, so what was stored for it was the - // merge as it came: paths with spaces around them, empty entries, the same - // folder twice. The worker was handed a sanitised copy and behaved, so only - // the stored value was wrong — and it is the one the next merge starts from. - sanitizedValue = sanitizeSearchIndex(value); + try { + const stored = JSON.parse(row.value); + return stored && typeof stored === 'object' && !Array.isArray(stored) ? stored : {}; + } catch { + // An unreadable value is the section's defaults, exactly as a read treats it. + return {}; } +}; + +/** + * Merge an update over one stored section and write it back, and answer the + * whole section as it now stands. + * + * Read and written without yielding in between — the database answers + * synchronously — so two saves of one section at once cannot both start from + * the same stored value. The route used to merge over the settings read at the + * start of the request, with two awaits between that read and the write: a + * retention of ninety days saved in one tab disappeared when the other tab + * saved a size cap a moment later, and the person who set it was told it was + * saved. Branding was taken out of this path for the same reason, where losing + * a save also left a logo file behind with nothing to serve or remove it. + */ +const mergeSystemSection = async (category, key, update) => { + assertSystemCategory(category); + + const db = await getDb(); + const merged = sanitizeSystemSetting(key, { + ...readStoredSection(db, category, key), + ...update, + }); + writeSystemSetting(db, category, key, merged); + return merged; +}; - const valueJson = JSON.stringify(sanitizedValue); +/** Store one system setting as it is, in a single synchronous step. */ +const writeSystemSetting = (db, category, key, value, now = new Date().toISOString()) => { + const valueJson = JSON.stringify(value); // Check if setting exists const existing = db @@ -631,31 +897,71 @@ const setSystemSetting = async (category, key, value) => { .get(category, key); if (existing) { - db.prepare( + prepared( + db, 'UPDATE system_settings SET value = ?, updated_at = ? WHERE category = ? AND key = ?' ).run(valueJson, now, category, key); } else { - db.prepare( + prepared( + db, 'INSERT INTO system_settings (id, category, key, value, updated_at) VALUES (?, ?, ?, ?, ?)' ).run(generateId(), category, key, valueJson, now); } +}; - return sanitizedValue; +/** + * Change the branding, and answer what it was and what it is now. + * + * Read and written without yielding in between — the database answers + * synchronously — so two saves at once cannot both start from the same + * branding: the logo a save replaced is the one it was the last to see, and + * removing it cannot take away the logo another save has just put in place. + * + * @returns {Promise<{previous: object, current: object}>} + */ +const replaceBranding = async (update) => { + const db = await getDb(); + const row = db + .prepare('SELECT value FROM system_settings WHERE category = ? AND key = ?') + .get('branding', 'branding'); + + let stored = {}; + if (row) { + try { + stored = JSON.parse(row.value); + } catch { + // An unreadable value is the default branding. + } + } + + const previous = sanitizeBranding(stored); + const current = sanitizeBranding({ ...previous, ...update }); + writeSystemSetting(db, 'branding', 'branding', current); + return { previous, current }; }; /** * Legacy method: Get all settings (for backward compatibility) * Returns system settings + branding */ -const getSettings = async () => { - const systemSettings = await getSystemSettings(); - const publicSettings = await getPublicSettings(); +/** + * Settings, read once per request. + * + * The access rules are consulted for every path, so a bulk operation asked for + * these thousands of times over — each one several queries and a JSON parse, + * to re-read values that cannot change while a single request is running. The + * promise is memoized, not the value, so concurrent callers share one read. + */ +const getSettings = async () => + cachedForRequest('settings', 'all', async () => { + const systemSettings = await getSystemSettings(); + const publicSettings = await getPublicSettings(); - return { - ...systemSettings, - branding: publicSettings.branding, - }; -}; + return { + ...systemSettings, + branding: publicSettings.branding, + }; + }); /** * Legacy method: Set settings (for backward compatibility) @@ -667,15 +973,25 @@ const setSettings = async (partial) => { // Deep merge const merged = { thumbnails: { ...current.thumbnails, ...(partial.thumbnails || {}) }, + // Each half of the section stands on its own: saving the rules alone must + // not quietly switch off whether they hold administrators, and vice versa. access: { rules: partial.access?.rules !== undefined ? partial.access.rules : current.access.rules, - // Saved apart from the rules on the settings page, so each has to survive - // the other being saved on its own. applyToAdmins: partial.access?.applyToAdmins !== undefined ? partial.access.applyToAdmins : current.access.applyToAdmins, }, + uploads: { ...current.uploads, ...(partial.uploads || {}) }, + trash: { ...current.trash, ...(partial.trash || {}) }, + versions: { ...current.versions, ...(partial.versions || {}) }, + activity: { ...current.activity, ...(partial.activity || {}) }, + folderSize: { + excludedPaths: + partial.folderSize?.excludedPaths !== undefined + ? partial.folderSize.excludedPaths + : current.folderSize.excludedPaths, + }, branding: { ...current.branding, ...(partial.branding || {}) }, }; @@ -686,54 +1002,46 @@ const setSettings = async (partial) => { if (partial.access) { merged.access = await setSystemSetting('system', 'access', merged.access); } + if (partial.folderSize) { + merged.folderSize = await setSystemSetting('system', 'folderSize', merged.folderSize); + } if (partial.branding) { merged.branding = await setSystemSetting('branding', 'branding', merged.branding); } - - // Also update JSON for backward compatibility during transition - try { - await storage.update((data) => ({ - ...data, - settings: { - thumbnails: merged.thumbnails, - access: merged.access, - branding: merged.branding, - }, - })); - } catch (err) { - // Non-fatal, continue + if (partial.uploads) { + merged.uploads = await setSystemSetting('system', 'uploads', merged.uploads); + } + if (partial.trash) { + merged.trash = await setSystemSetting('system', 'trash', merged.trash); + } + if (partial.versions) { + merged.versions = await setSystemSetting('system', 'versions', merged.versions); + } + if (partial.activity) { + merged.activity = await setSystemSetting('system', 'activity', merged.activity); } return merged; }; -/** - * Update settings with an updater function - */ -const updateSettings = async (updater) => { - const current = await getSettings(); - const next = typeof updater === 'function' ? updater(current) : current; - return setSettings(next); -}; - module.exports = { - replaceBranding, - USER_SETTING_KEYS, - MAX_UPLOAD_CHUNK_SIZE_BYTES, + checkSystemSection, getPublicSettings, - sanitizeAccess, + getUserSettings, + getSystemSettings, sanitizeTrash, sanitizeVersions, sanitizeActivity, - sanitizeFolderSize, - sanitizeSearchIndex, - getUserSettings, - getSystemSettings, getSettingsForUser, setUserSetting, + WRITABLE_USER_SETTINGS, + setUserFolderSort, + setUserFolderView, setSystemSetting, + mergeSystemSection, + replaceBranding, + MAX_UPLOAD_CHUNK_SIZE_BYTES, // Legacy methods for backward compatibility getSettings, setSettings, - updateSettings, }; diff --git a/backend/src/services/sharesService.js b/backend/src/services/sharesService.js index f9d785f39..9bcf28cf4 100644 --- a/backend/src/services/sharesService.js +++ b/backend/src/services/sharesService.js @@ -1,15 +1,9 @@ const crypto = require('crypto'); const bcrypt = require('bcryptjs'); -const { getDb } = require('./db'); +const { getDb, prepared } = require('./db'); +const { generateId, nowIso } = require('../utils/ids'); const logger = require('../utils/logger'); -const nowIso = () => new Date().toISOString(); - -const generateId = () => - typeof crypto.randomUUID === 'function' - ? crypto.randomUUID() - : `${Date.now().toString(36)}-${crypto.randomBytes(8).toString('hex')}`; - /** * Generate a URL-safe share token * Uses base62 encoding for readability (no special characters) @@ -37,6 +31,13 @@ const toClientShare = (row) => { sourcePath: row.source_path, isDirectory: Boolean(row.is_directory), accessMode: row.access_mode, + allowDelete: row.allow_delete !== 0, + allowCreateFolder: row.allow_create_folder !== 0, + allowCreateFile: row.allow_create_file !== 0, + allowUpload: row.allow_upload !== 0, + allowDownload: row.allow_download !== 0, + versionsVisible: row.versions_visible === 1, + versionsDownload: row.versions_download === 1, sharingType: row.sharing_type, hasPassword: Boolean(row.password_hash), expiresAt: row.expires_at || null, @@ -44,6 +45,9 @@ const toClientShare = (row) => { accessCount: row.access_count || 0, downloadCount: row.download_count || 0, lastAccessedAt: row.last_accessed_at || null, + lastAccessIp: row.last_access_ip || null, + lastDownloadedAt: row.last_downloaded_at || null, + lastDownloadIp: row.last_download_ip || null, createdAt: row.created_at, updatedAt: row.updated_at, }; @@ -58,6 +62,13 @@ const createShare = async ({ sourcePath, isDirectory = false, accessMode = 'readonly', + allowDelete = true, + allowCreateFolder = true, + allowCreateFile = true, + allowUpload = true, + allowDownload = true, + versionsVisible, + versionsDownload, sharingType = 'anyone', password = null, userIds = [], @@ -88,6 +99,20 @@ const createShare = async ({ throw e; } + const operationPermissions = { + allowDelete, + allowCreateFolder, + allowCreateFile, + allowUpload, + }; + for (const [key, value] of Object.entries(operationPermissions)) { + if (typeof value !== 'boolean') { + const e = new Error(`${key} must be a boolean`); + e.status = 400; + throw e; + } + } + if (!['anyone', 'users'].includes(sharingType)) { const e = new Error('Invalid sharing type'); e.status = 400; @@ -100,6 +125,21 @@ const createShare = async ({ throw e; } + // A share with named accounts shows the history they would see anyway; a link + // for anyone shows none until its owner decides otherwise. + const historyByDefault = sharingType === 'users'; + const history = { + versionsVisible: versionsVisible === undefined ? historyByDefault : versionsVisible, + versionsDownload: versionsDownload === undefined ? historyByDefault : versionsDownload, + }; + for (const [key, value] of Object.entries(history)) { + if (typeof value !== 'boolean') { + const e = new Error(`${key} must be a boolean`); + e.status = 400; + throw e; + } + } + const db = await getDb(); const shareId = generateId(); const shareToken = generateShareToken(10); @@ -107,13 +147,15 @@ const createShare = async ({ const passwordHash = password ? await bcrypt.hash(password, 10) : null; // Create share - db.prepare( + prepared( + db, ` INSERT INTO shares ( id, share_token, owner_id, source_space, source_path, is_directory, - access_mode, sharing_type, password_hash, expires_at, label, - download_count, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?) + access_mode, allow_delete, allow_create_folder, allow_create_file, allow_upload, + allow_download, sharing_type, password_hash, expires_at, label, download_count, + created_at, updated_at, versions_visible, versions_download + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?) ` ).run( shareId, @@ -123,20 +165,30 @@ const createShare = async ({ sourcePath, isDirectory ? 1 : 0, accessMode, + allowDelete ? 1 : 0, + allowCreateFolder ? 1 : 0, + allowCreateFile ? 1 : 0, + allowUpload ? 1 : 0, + allowDownload ? 1 : 0, sharingType, passwordHash, expiresAt, label, now, - now + now, + history.versionsVisible ? 1 : 0, + history.versionsDownload ? 1 : 0 ); // Add user permissions if user-specific share if (sharingType === 'users' && Array.isArray(userIds) && userIds.length > 0) { - const insertPerm = db.prepare(` + const insertPerm = prepared( + db, + ` INSERT INTO share_permissions (id, share_id, user_id, created_at) VALUES (?, ?, ?, ?) - `); + ` + ); for (const userId of userIds) { try { @@ -158,7 +210,7 @@ const createShare = async ({ */ const getShareById = async (shareId) => { const db = await getDb(); - const row = db.prepare('SELECT * FROM shares WHERE id = ?').get(shareId); + const row = prepared(db, 'SELECT * FROM shares WHERE id = ?').get(shareId); if (!row) return null; const share = toClientShare(row); @@ -183,7 +235,7 @@ const getShareById = async (shareId) => { */ const getShareByToken = async (token) => { const db = await getDb(); - const row = db.prepare('SELECT * FROM shares WHERE share_token = ?').get(token); + const row = prepared(db, 'SELECT * FROM shares WHERE share_token = ?').get(token); if (!row) return null; const share = toClientShare(row); @@ -274,13 +326,28 @@ const getSharesForUser = async (userId) => { */ const updateShare = async (shareId, updates = {}) => { const db = await getDb(); - const existing = db.prepare('SELECT * FROM shares WHERE id = ?').get(shareId); + const existing = prepared(db, 'SELECT * FROM shares WHERE id = ?').get(shareId); if (!existing) { const e = new Error('Share not found'); e.status = 404; throw e; } + const effectiveSharingType = updates.sharingType || existing.sharing_type; + const hasUserIdsUpdate = 'userIds' in updates; + if (effectiveSharingType === 'users' && hasUserIdsUpdate) { + if (!Array.isArray(updates.userIds) || updates.userIds.length === 0) { + const e = new Error('At least one user is required for user-specific shares'); + e.status = 400; + throw e; + } + } + if (effectiveSharingType === 'users' && existing.sharing_type !== 'users' && !hasUserIdsUpdate) { + const e = new Error('At least one user is required for user-specific shares'); + e.status = 400; + throw e; + } + const fields = []; const values = []; @@ -292,6 +359,26 @@ const updateShare = async (shareId, updates = {}) => { values.push(updates.accessMode); } + const operationPermissionFields = [ + ['allowDelete', 'allow_delete'], + ['allowCreateFolder', 'allow_create_folder'], + ['allowCreateFile', 'allow_create_file'], + ['allowUpload', 'allow_upload'], + ['allowDownload', 'allow_download'], + ['versionsVisible', 'versions_visible'], + ['versionsDownload', 'versions_download'], + ]; + for (const [key, column] of operationPermissionFields) { + if (!(key in updates)) continue; + if (typeof updates[key] !== 'boolean') { + const e = new Error(`${key} must be a boolean`); + e.status = 400; + throw e; + } + fields.push(`${column} = ?`); + values.push(updates[key] ? 1 : 0); + } + if ( typeof updates.sharingType === 'string' && ['anyone', 'users'].includes(updates.sharingType) @@ -326,29 +413,30 @@ const updateShare = async (shareId, updates = {}) => { values.push(updates.label); } - if (fields.length === 0) { - return getShareById(shareId); - } - - fields.push('updated_at = ?'); - values.push(nowIso()); - values.push(shareId); + const permissionsWillChange = + hasUserIdsUpdate || (effectiveSharingType === 'anyone' && existing.sharing_type === 'users'); + if (fields.length > 0 || permissionsWillChange) { + fields.push('updated_at = ?'); + values.push(nowIso()); + values.push(shareId); - db.prepare(`UPDATE shares SET ${fields.join(', ')} WHERE id = ?`).run(...values); - - // Update user permissions if provided and sharing type is 'users' - if ('userIds' in updates && Array.isArray(updates.userIds)) { - const sharingType = updates.sharingType || existing.sharing_type; + prepared(db, `UPDATE shares SET ${fields.join(', ')} WHERE id = ?`).run(...values); + } - if (sharingType === 'users') { - // Remove all existing permissions - db.prepare('DELETE FROM share_permissions WHERE share_id = ?').run(shareId); + // A permission list is meaningful only for user-specific shares. Always clear + // it when switching back to an anyone link so revoked recipients have no stale + // database entries left behind. + if (hasUserIdsUpdate || effectiveSharingType === 'anyone') { + prepared(db, 'DELETE FROM share_permissions WHERE share_id = ?').run(shareId); - // Add new permissions - const insertPerm = db.prepare(` + if (effectiveSharingType === 'users') { + const insertPerm = prepared( + db, + ` INSERT INTO share_permissions (id, share_id, user_id, created_at) VALUES (?, ?, ?, ?) - `); + ` + ); const now = nowIso(); for (const userId of updates.userIds) { @@ -381,7 +469,7 @@ const updateShare = async (shareId, updates = {}) => { */ const deleteShare = async (shareId) => { const db = await getDb(); - const result = db.prepare('DELETE FROM shares WHERE id = ?').run(shareId); + const result = prepared(db, 'DELETE FROM shares WHERE id = ?').run(shareId); return result.changes > 0; }; @@ -391,7 +479,63 @@ const normalizeShareSourcePath = (sourcePath = '') => .replace(/^\/+/, '') .replace(/\/+$/, ''); -const escapeLikePattern = (value = '') => String(value).replace(/[\\%_]/g, '\\$&'); +/** + * The shares inside a folder, found by bounds on the path: every path that + * begins with `prefix/` sorts at or after it and before `prefix0`, `0` being + * the character right after `/`. + * + * `LIKE 'prefix/%'` ignored case, as SQLite's LIKE always does for ASCII: + * deleting `Docs` counted, and then deleted, the share links of `docs/…` — + * another folder on a Linux volume, and somebody else's links as often as not. + */ +const CHILD_SHARES_SQL = + 'SELECT * FROM shares WHERE source_space = ? AND source_path >= ? AND source_path < ?'; +const childRange = (prefix) => [`${prefix}/`, `${prefix}0`]; + +/** + * Shares affected by each target, in one pass. + * + * A bulk delete asked this per file — three thousand round trips through the + * database to answer a question the whole selection could ask once. Returned + * as a map so each entry still knows which shares are its own. + */ +const getSharesBySourceTarget = async (targets = []) => { + const byTarget = new Map(); + const normalized = (Array.isArray(targets) ? targets : []) + .map((target) => ({ + key: `${target?.sourceSpace}:${normalizeShareSourcePath(target?.sourcePath)}`, + sourceSpace: target?.sourceSpace, + sourcePath: normalizeShareSourcePath(target?.sourcePath), + includeChildren: Boolean(target?.includeChildren), + })) + .filter((target) => target.sourceSpace && target.sourcePath); + + if (normalized.length === 0) return byTarget; + + const db = await getDb(); + const exactQuery = prepared( + db, + 'SELECT * FROM shares WHERE source_space = ? AND source_path = ?' + ); + const childQuery = prepared(db, CHILD_SHARES_SQL); + + for (const target of normalized) { + if (byTarget.has(target.key)) continue; + const rows = new Map(); + exactQuery.all(target.sourceSpace, target.sourcePath).forEach((row) => rows.set(row.id, row)); + if (target.includeChildren) { + childQuery + .all(target.sourceSpace, ...childRange(target.sourcePath)) + .forEach((row) => rows.set(row.id, row)); + } + byTarget.set(target.key, Array.from(rows.values()).map(toClientShare)); + } + + return byTarget; +}; + +const shareTargetKey = (target) => + `${target?.sourceSpace}:${normalizeShareSourcePath(target?.sourcePath)}`; const getSharesForSourceTargets = async (targets = []) => { const normalizedTargets = (Array.isArray(targets) ? targets : []) @@ -409,20 +553,18 @@ const getSharesForSourceTargets = async (targets = []) => { const db = await getDb(); const sharesById = new Map(); - const exactQuery = db.prepare('SELECT * FROM shares WHERE source_space = ? AND source_path = ?'); - const childQuery = db.prepare( - "SELECT * FROM shares WHERE source_space = ? AND source_path LIKE ? ESCAPE '\\'" + const exactQuery = prepared( + db, + 'SELECT * FROM shares WHERE source_space = ? AND source_path = ?' ); + const childQuery = prepared(db, CHILD_SHARES_SQL); for (const target of normalizedTargets) { const exactRows = exactQuery.all(target.sourceSpace, target.sourcePath); exactRows.forEach((row) => sharesById.set(row.id, row)); if (target.includeChildren) { - const childRows = childQuery.all( - target.sourceSpace, - `${escapeLikePattern(target.sourcePath)}/%` - ); + const childRows = childQuery.all(target.sourceSpace, ...childRange(target.sourcePath)); childRows.forEach((row) => sharesById.set(row.id, row)); } } @@ -437,7 +579,7 @@ const deleteSharesByIds = async (shareIds = []) => { } const db = await getDb(); - const deleteOne = db.prepare('DELETE FROM shares WHERE id = ?'); + const deleteOne = prepared(db, 'DELETE FROM shares WHERE id = ?'); const transaction = db.transaction((ids) => { let changes = 0; ids.forEach((id) => { @@ -454,7 +596,7 @@ const deleteSharesByIds = async (shareIds = []) => { */ const verifySharePassword = async (shareId, password) => { const db = await getDb(); - const row = db.prepare('SELECT password_hash FROM shares WHERE id = ?').get(shareId); + const row = prepared(db, 'SELECT password_hash FROM shares WHERE id = ?').get(shareId); if (!row) { return false; @@ -469,9 +611,6 @@ const verifySharePassword = async (shareId, password) => { return false; } - // The asynchronous form: this is reachable without an account, and the - // synchronous one stops the server doing anything else for the length of - // the hash. return bcrypt.compare(password, row.password_hash); }; @@ -480,7 +619,7 @@ const verifySharePassword = async (shareId, password) => { */ const hasUserPermission = async (shareId, userId) => { const db = await getDb(); - const share = db.prepare('SELECT sharing_type, owner_id FROM shares WHERE id = ?').get(shareId); + const share = prepared(db, 'SELECT sharing_type, owner_id FROM shares WHERE id = ?').get(shareId); if (!share) { return false; @@ -525,19 +664,15 @@ const isShareExpired = (share) => { }; /** - * Somebody opened the share. - * - * Opening it is not downloading from it: this used to raise the download - * counter, so the number an owner was shown counted page loads, reloads and - * every folder they browsed inside the share. A link opened twenty times and - * never downloaded from read as twenty downloads. + * Update share access tracking */ const trackShareAccess = async (shareId, { ipAddress = null } = {}) => { const db = await getDb(); - db.prepare( + prepared( + db, ` UPDATE shares - SET access_count = COALESCE(access_count, 0) + 1, + SET access_count = access_count + 1, last_accessed_at = ?, last_access_ip = ? WHERE id = ? @@ -545,15 +680,18 @@ const trackShareAccess = async (shareId, { ipAddress = null } = {}) => { ).run(nowIso(), ipAddress, shareId); }; -/** Something was actually sent: a file left through the link. */ +/** + * Update share download tracking + */ const trackShareDownload = async (shareId, { ipAddress = null } = {}) => { const db = await getDb(); - db.prepare( + prepared( + db, ` UPDATE shares - SET download_count = COALESCE(download_count, 0) + 1, - last_accessed_at = ?, - last_access_ip = ? + SET download_count = download_count + 1, + last_downloaded_at = ?, + last_download_ip = ? WHERE id = ? ` ).run(nowIso(), ipAddress, shareId); @@ -564,7 +702,7 @@ const trackShareDownload = async (shareId, { ipAddress = null } = {}) => { */ const getShareStats = async (shareId) => { const db = await getDb(); - const share = db.prepare('SELECT * FROM shares WHERE id = ?').get(shareId); + const share = prepared(db, 'SELECT * FROM shares WHERE id = ?').get(shareId); if (!share) return null; // Count guest sessions @@ -580,6 +718,9 @@ const getShareStats = async (shareId) => { accessCount: share.access_count || 0, downloadCount: share.download_count || 0, lastAccessedAt: share.last_accessed_at || null, + lastAccessIp: share.last_access_ip || null, + lastDownloadedAt: share.last_downloaded_at || null, + lastDownloadIp: share.last_download_ip || null, guestSessionCount: guestSessions?.count || 0, }; }; @@ -606,6 +747,8 @@ module.exports = { getShareByToken, getSharesByOwnerId, getSharesForSourceTargets, + getSharesBySourceTarget, + shareTargetKey, getSharesForUser, updateShare, deleteShare, diff --git a/backend/src/services/storage/jsonStorage.js b/backend/src/services/storage/jsonStorage.js deleted file mode 100644 index b99126eb3..000000000 --- a/backend/src/services/storage/jsonStorage.js +++ /dev/null @@ -1,85 +0,0 @@ -const fs = require('fs/promises'); -const { directories, files } = require('../../config/index'); -const { ensureDir } = require('../../utils/fsUtils'); -const logger = require('../../utils/logger'); - -const CONFIG_FILE = files.passwordConfig; -const ENCODING = 'utf8'; - -let cache = null; -let initialized = false; - -/** - * Default structure for app-config.json - */ -const DEFAULT_DATA = { - version: 4, - settings: { - thumbnails: { enabled: true, size: 200, quality: 70 }, - access: { rules: [] }, - }, - favorites: [], -}; - -/** - * Ensure config directory and file exist - */ -const init = async () => { - if (initialized) return; - - await ensureDir(directories.config); - - try { - await fs.access(CONFIG_FILE); - } catch (error) { - if (error?.code === 'ENOENT') { - logger.info('Creating default config file'); - await fs.writeFile(CONFIG_FILE, JSON.stringify(DEFAULT_DATA, null, 2) + '\n', ENCODING); - } - } - - cache = await read(); - initialized = true; -}; - -/** - * Read from disk - */ -const read = async () => { - try { - const raw = await fs.readFile(CONFIG_FILE, ENCODING); - return JSON.parse(raw); - } catch (error) { - logger.warn({ err: error }, 'Failed to read config, using defaults'); - return { ...DEFAULT_DATA }; - } -}; - -/** - * Write to disk and update cache - */ -const write = async (data) => { - await fs.writeFile(CONFIG_FILE, JSON.stringify(data, null, 2) + '\n', ENCODING); - cache = data; - return data; -}; - -/** - * Get entire config - */ -const get = async () => { - await init(); - return JSON.parse(JSON.stringify(cache)); // Deep clone -}; - -/** - * Update config with an updater function - */ -const update = async (updater) => { - await init(); - const current = await get(); - const next = updater(current); - return write(next); -}; - -module.exports = { get, update }; diff --git a/backend/src/services/storageWritability.js b/backend/src/services/storageWritability.js index e6f1cfe96..a48322b3c 100644 --- a/backend/src/services/storageWritability.js +++ b/backend/src/services/storageWritability.js @@ -35,7 +35,13 @@ const whyNotWritable = async (absoluteDir) => { }; /** The permissions that need to write in the folder itself. */ -const WRITE_PERMISSIONS = ['canWrite', 'canUpload', 'canDelete', 'canCreateFolder']; +const WRITE_PERMISSIONS = [ + 'canWrite', + 'canUpload', + 'canDelete', + 'canCreateFolder', + 'canCreateFile', +]; /** * An access answer with the writes taken away when the storage refuses them, diff --git a/backend/src/services/trash/index.js b/backend/src/services/trash/index.js index c27fbe6ca..ee140fc72 100644 --- a/backend/src/services/trash/index.js +++ b/backend/src/services/trash/index.js @@ -25,8 +25,9 @@ const logger = require('../../utils/logger'); const { normalizeRelativePath } = require('../../utils/pathUtils'); const { ACTIONS, authorizeAndResolve, authorizePath } = require('../authorizationService'); const { getDb } = require('../db'); +const folderSizeHooks = require('../folderSizeHooks'); +const recentDestinations = require('../recentDestinationsService'); const { readTextFile } = require('../textEditorService'); -const folderSizeManager = require('../folderSizeManager'); const maintenance = require('./maintenance'); const operations = require('./operations'); const { DAY_MS, admission } = require('./policy'); @@ -214,7 +215,6 @@ const listItems = async (context) => { if (item.state !== 'trashed' || !visibleTo(item, user)) continue; const zone = zoneRows.get(item.zoneId) || null; if (!availability.has(item.zoneId)) { - // eslint-disable-next-line no-await-in-loop availability.set(item.zoneId, zone ? (await zones.inspectZone(zone)).available : false); } items.push( @@ -250,11 +250,12 @@ const mayRestore = async (item, context, user) => { }; const announceRestored = (item, restorePath) => { - // Best-effort: tell the folder-size index the parent changed, so a restore - // shows the right sizes without waiting for the next reconciliation. Never - // lets a size update fail a restore, and does nothing when the index is off. try { - Promise.resolve(folderSizeManager.touch([path.dirname(restorePath)])).catch(() => {}); + const pending = + item.kind === 'directory' + ? folderSizeHooks.onDirectoryTreeCreated(restorePath) + : folderSizeHooks.onFileWritten(restorePath, item.size); + Promise.resolve(pending).catch(() => {}); } catch (error) { logger.debug({ err: error, restorePath }, 'Folder sizes were not told about a restore'); } @@ -299,7 +300,6 @@ const settleSharesElsewhere = async (kept, choice, { context, user, relativePath let root = null; try { const { resolved } = await authorizeAndResolve(context, relativePath, ACTIONS.read); - // eslint-disable-next-line global-require if (resolved) root = require('../fileTransferService').getShareSourceTarget(resolved, false); } catch (error) { logger.debug({ err: error, relativePath }, 'A restored item could not be named for its shares'); @@ -324,20 +324,16 @@ const restoreItems = async (ids, context, { shares } = {}) => { results.push({ id, status: 'not-found' }); continue; } - // eslint-disable-next-line no-await-in-loop if (!(await mayRestore(item, context, user))) { results.push({ id, status: 'forbidden', name: item.name }); continue; } // Read first: the item takes the shares it kept with it when it leaves. - // eslint-disable-next-line no-await-in-loop const kept = await trashShares.listForItem(id); - // eslint-disable-next-line no-await-in-loop const outcome = await operations.restoreItem(id); if (outcome.status === 'restored') { announceRestored(item, outcome.restorePath); const restoredName = path.basename(outcome.restorePath); - // eslint-disable-next-line no-await-in-loop const sharesOutcome = await settleShares( kept, sharesChoice, @@ -498,7 +494,6 @@ const restoreEntries = async (id, paths, context, { shares } = {}) => { const name = path.posix.basename(entry); let outcome; try { - // eslint-disable-next-line no-await-in-loop outcome = await operations.restoreEntry(item.id, entry); } catch (error) { logger.warn( @@ -511,7 +506,6 @@ const restoreEntries = async (id, paths, context, { shares } = {}) => { announceRestored(outcome, outcome.restorePath); const restoredName = path.basename(outcome.restorePath); const kept = trashShares.underEntry(keptInFolder, entry); - // eslint-disable-next-line no-await-in-loop const sharesOutcome = await settleShares( kept, sharesChoice, @@ -564,7 +558,6 @@ const mayCreateIn = async (context, relativePath, kind) => { const actions = kind === 'directory' ? [ACTIONS.createFolder, ACTIONS.createFile] : [ACTIONS.createFile]; for (const action of actions) { - // eslint-disable-next-line no-await-in-loop const { allowed } = await authorizePath(context, relativePath, action); if (!allowed) return false; } @@ -675,21 +668,18 @@ const executeRestoreTo = async (plan, { onEvent = () => {}, signal } = {}) => { continue; } if (!rightToRestore.has(item.id)) { - // eslint-disable-next-line no-await-in-loop rightToRestore.set(item.id, await mayRestore(item, context, user)); } if (!rightToRestore.get(item.id)) { results.push({ ...key, status: 'forbidden', name }); continue; } - // eslint-disable-next-line no-await-in-loop if (!(await mayCreateIn(context, target.relativePath, described.kind))) { results.push({ ...key, status: 'forbidden', reason: 'destination', name }); continue; } // Read first: a whole item takes the shares it kept with it when it leaves. - // eslint-disable-next-line no-await-in-loop const keptForItem = await trashShares.listForItem(item.id); const kept = task.entry ? trashShares.underEntry(keptForItem, task.entry) : keptForItem; @@ -707,10 +697,8 @@ const executeRestoreTo = async (plan, { onEvent = () => {}, signal } = {}) => { let outcome; try { outcome = task.entry - ? // eslint-disable-next-line no-await-in-loop - await operations.restoreEntry(item.id, task.entry, options) - : // eslint-disable-next-line no-await-in-loop - await operations.restoreItem(item.id, options); + ? await operations.restoreEntry(item.id, task.entry, options) + : await operations.restoreItem(item.id, options); } catch (error) { logger.warn( { err: error, itemId: item.id, entry: task.entry || null }, @@ -725,7 +713,6 @@ const executeRestoreTo = async (plan, { onEvent = () => {}, signal } = {}) => { if (outcome.status === 'restored') { announceRestored(described, outcome.restorePath); const restoredName = path.basename(outcome.restorePath); - // eslint-disable-next-line no-await-in-loop const sharesOutcome = await settleSharesElsewhere(kept, plan.sharesChoice, { context, user, @@ -746,6 +733,13 @@ const executeRestoreTo = async (plan, { onEvent = () => {}, signal } = {}) => { } } + if (results.some((result) => result.status === 'restored')) { + try { + await recentDestinations.record(user.id, target.relativePath); + } catch (error) { + logger.debug({ err: error }, 'The destination was not remembered'); + } + } return { destination: target.relativePath, items: results }; }; @@ -765,10 +759,8 @@ const purgeItems = async (ids, context, { forgetUnavailable = false } = {}) => { results.push({ id, status: 'not-found' }); continue; } - // eslint-disable-next-line no-await-in-loop let outcome = await operations.purgeItem(id); if (outcome.status === 'unavailable' && forgetUnavailable && isAdmin(user)) { - // eslint-disable-next-line no-await-in-loop outcome = await operations.forgetItem(id); } results.push({ id, status: outcome.status, reason: outcome.reason || null, name: item.name }); @@ -786,7 +778,6 @@ const emptyTrash = async (context) => { for (const item of store.listItems(db)) { if (item.state !== 'trashed' || !visibleTo(item, user)) continue; try { - // eslint-disable-next-line no-await-in-loop const outcome = await operations.purgeItem(item.id); if (outcome.status === 'purged') summary.purged += 1; else if (outcome.status === 'unavailable') summary.unavailable += 1; @@ -805,11 +796,8 @@ const verifyAll = async () => { const db = await getDb(); const results = []; for (const zone of store.listZones(db)) { - // eslint-disable-next-line no-await-in-loop const inspection = await zones.inspectZone(zone); - // eslint-disable-next-line no-await-in-loop const limits = inspection.available ? await maintenance.limitsFor(zone.root, settings) : {}; - // eslint-disable-next-line no-await-in-loop const result = await verifyZone(zone, limits); results.push({ ...result, ...zones.describeZoneRoot(zone.root) }); } @@ -828,6 +816,7 @@ module.exports = { listItems, restoreItems, listEntries, + locateTrashFile, readTrashText, restoreEntries, prepareRestoreTo, diff --git a/backend/src/services/trash/maintenance.js b/backend/src/services/trash/maintenance.js index ba3f4498f..e7ab50b42 100644 --- a/backend/src/services/trash/maintenance.js +++ b/backend/src/services/trash/maintenance.js @@ -83,7 +83,6 @@ const applyPlan = async (db, zone, plan, itemsById) => { for (const entry of plan) { const item = itemsById.get(entry.id); try { - // eslint-disable-next-line no-await-in-loop const result = await operations.purgeItem(entry.id); if (result.status === 'unavailable') break; if (result.status !== 'purged') continue; @@ -122,7 +121,6 @@ const applyVersionPlan = async (db, zone, plan) => { const outcome = { versionsPurged: 0, versionBytesPurged: 0, versionsEvictedEarly: 0 }; for (const entry of plan) { try { - // eslint-disable-next-line no-await-in-loop const result = await versionOperations.purgeVersion(entry.id); if (result.status === 'unavailable') break; if (result.status !== 'purged') continue; @@ -227,7 +225,6 @@ const runOnce = async ({ reason }) => { const results = []; for (const zone of store.listZones(db)) { try { - // eslint-disable-next-line no-await-in-loop const summary = await maintainZone(zone, settings); store.pruneEvents(db, { zoneId: zone.id, keep: EVENTS_KEPT_PER_ZONE }); lastPasses.set(zone.id, summary); @@ -275,7 +272,7 @@ const runPass = async ({ reason = 'manual' } = {}) => { let results; do { again = false; - // eslint-disable-next-line no-await-in-loop + results = await runOnce({ reason }); } while (again); return results; @@ -300,7 +297,6 @@ const requestPass = ({ delayMs = REQUEST_DELAY_MS } = {}) => { /** Resolves once no pass is scheduled or running. */ const idle = async () => { while (requested || running) { - // eslint-disable-next-line no-await-in-loop await (running || new Promise((resolve) => setTimeout(resolve, 5))); } }; @@ -322,13 +318,13 @@ const makeRoom = async (directory, requiredBytes) => { // Emptying a trash for an upload that would be refused anyway destroys // people's deleted files for nothing: only when the trash can cover the // shortfall is anything purged. - // eslint-disable-next-line no-await-in-loop + const { freeBytes } = await module.exports.measureVolume(root); const held = trashedItemsOf(db, zone).reduce((total, item) => total + item.size, 0) + versionBytesIn(db, zone); if (Number.isFinite(freeBytes) && freeBytes + held < requiredBytes) continue; - // eslint-disable-next-line no-await-in-loop + const summary = await maintainZone(zone, settings, { floorBytes: requiredBytes }); freed += summary.purgedBytes || 0; } @@ -341,10 +337,9 @@ const zonesOverview = async () => { const db = await getDb(); const overview = []; for (const zone of store.listZones(db)) { - // eslint-disable-next-line no-await-in-loop const inspection = await zones.inspectZone(zone); const items = store.listItemsByZone(db, zone.id); - // eslint-disable-next-line no-await-in-loop + const limits = inspection.available ? await limitsFor(zone.root, settings) : null; overview.push({ id: zone.id, diff --git a/backend/src/services/trash/operations.js b/backend/src/services/trash/operations.js index 819a6c128..e26566d79 100644 --- a/backend/src/services/trash/operations.js +++ b/backend/src/services/trash/operations.js @@ -69,7 +69,6 @@ const measure = async (absolutePath) => { const current = stack.pop(); let dirents; try { - // eslint-disable-next-line no-await-in-loop dirents = await fsp.readdir(current, { withFileTypes: true }); } catch { continue; @@ -82,7 +81,6 @@ const measure = async (absolutePath) => { else if (dirent.isFile()) files.push(child); } for (let index = 0; index < files.length; index += MEASURE_BATCH) { - // eslint-disable-next-line no-await-in-loop const sizes = await Promise.all( files.slice(index, index + MEASURE_BATCH).map((file) => fsp.lstat(file).then( @@ -278,7 +276,6 @@ const entrySegments = (entryPath) => { const walkInside = async (payload, segments) => { let current = payload; for (const segment of segments) { - // eslint-disable-next-line no-await-in-loop const stats = await lstatOrNull(current); if (!stats?.isDirectory()) return null; current = path.join(current, segment); @@ -316,7 +313,6 @@ const prepareDestination = async (zone, parent) => { // The deepest folder that already exists is where a link could stand. let existing = parent; - // eslint-disable-next-line no-await-in-loop while (existing !== zone.root && !(await lstatOrNull(existing))) { existing = path.dirname(existing); } @@ -364,14 +360,33 @@ const STAGING_PREFIX = '.nextexplorer-restoring-'; const stagingPathFor = (restorePath, itemId) => path.join(path.dirname(restorePath), `${STAGING_PREFIX}${itemId}`); +/** + * Bytes copied, as deltas, whichever copy is running. The stream copy reports + * each chunk as a number; rsync reports a running total for its whole run, as + * `{ copiedBytes, percent }`, which is what the container uses. + */ +const bytesReporter = (onBytes) => { + if (typeof onBytes !== 'function') return undefined; + let reported = 0; + return (progress) => { + if (typeof progress === 'number') { + onBytes(progress); + return; + } + const total = Number(progress?.copiedBytes); + if (!Number.isFinite(total) || total <= reported) return; + onBytes(total - reported); + reported = total; + }; +}; + /** Required when used: the transfer service itself requires the trash. */ const copyTree = (source, destination, isDirectory, onBytes, signal) => - // eslint-disable-next-line global-require require('../fileTransferService').copyEntryWithProgress( source, destination, isDirectory, - onBytes, + bytesReporter(onBytes), signal ); @@ -393,23 +408,19 @@ const nameCopy = async ({ db, item, staging, entryPath, restorePath, stoppedSinc for (let attempt = 1; attempt <= MAX_NAMING_ATTEMPTS; attempt += 1) { // A crash between the link and the removal of the hidden name left the // copy under both: it is already named. - // eslint-disable-next-line no-await-in-loop const alreadyNamed = (await linkedUnderBoth(staging, target)) || (stoppedSince !== null && (await sameLinkUnderBoth(staging, target, stoppedSince))); if (!alreadyNamed) { try { - // eslint-disable-next-line no-await-in-loop await moveNoReplace(staging, target); } catch (error) { if (error?.code !== 'EEXIST') throw error; - // eslint-disable-next-line no-await-in-loop target = path.join(directory, await findAvailableName(directory, desired)); store.setItemState(db, item.id, 'copied', { restorePath: target, restoreEntry: entryPath }); continue; } } - // eslint-disable-next-line no-await-in-loop await dropOldName(staging, target); return target; } @@ -792,7 +803,6 @@ const listEntries = async (itemId, entryPath = '') => { const names = await fsp.readdir(found.absolutePath); const entries = []; for (let index = 0; index < names.length; index += LIST_BATCH) { - // eslint-disable-next-line no-await-in-loop const batch = await Promise.all( names.slice(index, index + LIST_BATCH).map(async (name) => { const stats = await lstatOrNull(path.join(found.absolutePath, name)); @@ -1275,7 +1285,6 @@ const recoverZone = async (zone, { breakerRatio = 0.2, breakerMinimum = 5 } = {} ); } else { for (const row of vanished) { - // eslint-disable-next-line no-await-in-loop await drop(row, zones.itemPaths(zone.root, row.id).sidecar, 'lost'); report.lost += 1; } @@ -1285,7 +1294,6 @@ const recoverZone = async (zone, { breakerRatio = 0.2, breakerMinimum = 5 } = {} if (name.endsWith('.json')) { const id = name.slice(0, -'.json'.length); if (!knownIds.has(id) && !onDisk.has(id) && ITEM_ID_PATTERN.test(id)) { - // eslint-disable-next-line no-await-in-loop await fsp.rm(path.join(trashDirectory, name), { force: true }); report.removedSidecars += 1; } @@ -1294,12 +1302,9 @@ const recoverZone = async (zone, { breakerRatio = 0.2, breakerMinimum = 5 } = {} if (knownIds.has(name) || !ITEM_ID_PATTERN.test(name)) continue; const { payload, sidecar } = zones.itemPaths(zone.root, name); - // eslint-disable-next-line no-await-in-loop const described = onDisk.has(`${name}.json`) ? await readSidecar(sidecar) : null; - // eslint-disable-next-line no-await-in-loop const stats = await fsp.lstat(payload).catch(() => null); if (!stats) continue; - // eslint-disable-next-line no-await-in-loop const { bytes } = await measure(payload); const recoveredName = `recovered-${name.slice(0, 8)}`; @@ -1334,7 +1339,6 @@ const recoverZone = async (zone, { breakerRatio = 0.2, breakerMinimum = 5 } = {} : clock.nowIso(), }; store.insertItem(db, adopted); - // eslint-disable-next-line no-await-in-loop await writeSidecar(sidecar, adopted, 'w'); store.insertEvent(db, { zoneId: zone.id, diff --git a/backend/src/services/trash/settings.js b/backend/src/services/trash/settings.js index 340d2e06c..63a2b8b75 100644 --- a/backend/src/services/trash/settings.js +++ b/backend/src/services/trash/settings.js @@ -7,7 +7,7 @@ const getTrashSettings = async () => { // Required lazily: the settings service is a large module that most of the // trash has no other reason to load. - // eslint-disable-next-line global-require + const { getSystemSettings } = require('../settingsService'); return (await getSystemSettings()).trash; }; diff --git a/backend/src/services/trash/verify.js b/backend/src/services/trash/verify.js index 035fcb376..910ab7419 100644 --- a/backend/src/services/trash/verify.js +++ b/backend/src/services/trash/verify.js @@ -61,7 +61,6 @@ const verifyZone = async ( violations.push({ invariant: 'I1', itemId: row.id, detail: 'description missing' }); } if (measureSizes) { - // eslint-disable-next-line no-await-in-loop const { bytes } = await operations.measure(zones.itemPaths(zone.root, row.id).payload); if (bytes !== row.size) { violations.push({ diff --git a/backend/src/services/trash/zones.js b/backend/src/services/trash/zones.js index 76c5cbffc..91a797555 100644 --- a/backend/src/services/trash/zones.js +++ b/backend/src/services/trash/zones.js @@ -21,7 +21,6 @@ * looks exactly like an emptied one from here, and only the marker tells them * apart. */ -const crypto = require('crypto'); const fsp = require('fs/promises'); const path = require('path'); @@ -42,9 +41,6 @@ const trashDirectory = (root) => path.join(root, ZONE_DIRECTORY_NAME, TRASH_DIRE const versionsDirectory = (root) => path.join(root, ZONE_DIRECTORY_NAME, VERSIONS_DIRECTORY); const markerPath = (root) => path.join(root, ZONE_DIRECTORY_NAME, MARKER_FILE); -// link(2) is refused this way where the filesystem has no hard links. -const LINK_UNSUPPORTED = new Set(['EPERM', 'ENOTSUP', 'EOPNOTSUPP', 'ENOSYS', 'EMLINK', 'EINVAL']); - const isWithin = (parent, candidate) => candidate === parent || candidate.startsWith(parent.endsWith(path.sep) ? parent : parent + path.sep); @@ -73,7 +69,7 @@ const userVolumePaths = async () => { */ const locateZoneRoot = async (absolutePath) => { const target = path.resolve(absolutePath); - let root = null; + let root; // The personal root first: by default it sits inside the volume root. if (isWithin(directories.userRoot, target) && target !== directories.userRoot) { @@ -138,43 +134,16 @@ const ensureZone = async (root) => { let marker = await readMarker(root); if (!marker) { const created = { id: generateId(), createdAt: clock.nowIso() }; - // Written to a private temporary first and linked into place: link is - // atomic and fails when the marker already exists, so two deletions racing - // to create the zone agree on one id, and a racer only ever reads a whole - // marker — never the half-written file a plain `wx` create is briefly seen - // as under load. - const temporary = path.join( - zoneDirectory(root), - `.marker-${crypto.randomBytes(8).toString('hex')}.tmp` - ); try { - await fsp.writeFile(temporary, `${JSON.stringify(created)}\n`, { mode: 0o600 }); - try { - await fsp.link(temporary, markerPath(root)); - marker = created; - } catch (error) { - if (error?.code === 'EEXIST') { - marker = await readMarker(root); - } else if (LINK_UNSUPPORTED.has(error?.code)) { - // No hard links here (FAT, exFAT, some SMB shares): fall back to an - // exclusive create, whose brief half-written window is rare and no - // worse than before. - try { - await fsp.writeFile(markerPath(root), `${JSON.stringify(created)}\n`, { - flag: 'wx', - mode: 0o600, - }); - marker = created; - } catch (fallbackError) { - if (fallbackError?.code !== 'EEXIST') throw fallbackError; - marker = await readMarker(root); - } - } else { - throw error; - } - } - } finally { - await fsp.rm(temporary, { force: true }); + // Exclusive: two deletions racing to create the zone agree on one id. + await fsp.writeFile(markerPath(root), `${JSON.stringify(created)}\n`, { + flag: 'wx', + mode: 0o600, + }); + marker = created; + } catch (error) { + if (error?.code !== 'EEXIST') throw error; + marker = await readMarker(root); } } if (!marker || marker.corrupt) { diff --git a/backend/src/services/tusUploadService.js b/backend/src/services/tusUploadService.js index aeae60a1f..5e6a30685 100644 --- a/backend/src/services/tusUploadService.js +++ b/backend/src/services/tusUploadService.js @@ -12,10 +12,12 @@ const { ensureDir } = require('../utils/fsUtils'); const { isTopLevelEntry, normalizeRelativePath } = require('../utils/pathUtils'); const { placeWithoutOverwrite } = require('../utils/placeWithoutOverwrite'); const { ACTIONS, authorizeAndResolve } = require('./authorizationService'); +const activityLog = require('./activityLog'); const { resolveFolderUploadRelativePath } = require('./uploadFolderTargetService'); const { ensureStorageAvailable } = require('./uploadStorageGuard'); const { sweepStaleUploadRemnants, UPLOADING_SUFFIX } = require('./uploadRemnants'); const { getSystemSettings } = require('./settingsService'); +const folderSizeHooks = require('./folderSizeHooks'); const { InsufficientStorageError } = require('../errors/AppError'); const logger = require('../utils/logger'); @@ -34,15 +36,11 @@ let lastCleanupAt = 0; // data this hook has just moved to its destination. Expiry is handled by // cleanupInactiveUploads below, which covers more ground anyway (it also // reclaims data files whose metadata never made it to disk). -/** - * Built on first use, not when this module is required. - * - * `FileStore` creates its directory in its constructor, which turns requiring - * this file into a filesystem write — one that fails outright wherever the - * cache directory is not there yet, including the check that every module - * loads. A server that has never been asked to take an upload has no business - * creating a cache for one either. - */ +// `FileStore` creates its directory in its constructor, which turns requiring +// this file into a filesystem write — one that fails outright wherever the +// cache directory is not there yet, including the check that every module +// loads. A server that has never been asked to take an upload has no business +// creating a cache for one either. let fileStoreInstance = null; const store = () => { if (!fileStoreInstance) fileStoreInstance = new FileStore({ directory: TUS_CACHE_DIR }); @@ -103,7 +101,14 @@ const ensureTusEnabled = async () => { const now = Date.now(); if (tusEnabledCache.enabled === null || now - tusEnabledCache.at >= TUS_ENABLED_TTL_MS) { const settings = await getSystemSettings(); - tusEnabledCache = { enabled: Boolean(settings.uploads?.chunkedEnabled), at: now }; + // TUS serves both forced chunked uploads AND the client-side auto-fallback, + // which uses TUS even though forced chunking (chunkedEnabled) is off. Without + // allowing chunkedAutoFallback here, fallback uploads were rejected with 403 + // (surfacing as a "network error" in the client). + tusEnabledCache = { + enabled: Boolean(settings.uploads?.chunkedEnabled || settings.uploads?.chunkedAutoFallback), + at: now, + }; } if (!tusEnabledCache.enabled) { throw tusError(403, 'Chunked uploads are disabled.'); @@ -660,6 +665,26 @@ const finalizeUpload = async (nodeReq, upload) => { logger.warn({ uploadId: upload.id, err }, 'Failed to remove TUS upload metadata'); } + // Folder sizes hear of the file as they do for a direct upload. A refresh + // that fails leaves them to the periodic reconciliation, never the upload. + try { + const stats = await fs.stat(placed.path); + await folderSizeHooks.onFileWritten(placed.path, stats.size); + } catch (err) { + logger.debug({ err, path: placed.path }, 'Folder sizes were not told about a chunked upload'); + } + + // The one place a chunked upload finishes, whichever request finished it: a + // client that reconnects and asks again gets the result rather than a second + // move, and this line goes with the move. + await activityLog.record({ + action: nodeReq?.user ? 'file.upload' : 'share.upload', + user: nodeReq?.user, + target: target.destinationPath, + detail: { bytes: totalBytes, resumable: true }, + req: nodeReq, + }); + const result = { name: placed.name, path: placed.path, size: totalBytes, owner }; rememberFinished(upload.id, result); await recordFinished(upload.id, result); @@ -734,10 +759,8 @@ const EXPOSED_HEADERS = [ FINALIZE_ERROR_HEADER, ]; -/** - * Same reason as the store below it: the server owns the store, so building it - * eagerly would build the store eagerly too. - */ +// Built on first use, for the same reason the store is: its datastore creates +// a directory, and requiring a module must not write to the disk. let serverInstance = null; const tusServer = () => { if (!serverInstance) serverInstance = buildServer(); diff --git a/backend/src/services/userSearchService.js b/backend/src/services/userSearchService.js index 7351d540a..158b80561 100644 --- a/backend/src/services/userSearchService.js +++ b/backend/src/services/userSearchService.js @@ -81,21 +81,25 @@ const searchUsersForMentions = async (query, limit = 10) => { }; /** - * Who can be mentioned in a comment. + * Everyone who can be mentioned, without a search term. * - * ONLYOFFICE asks for the whole list and filters it in the editor as the - * comment is typed, so this answers with names and addresses rather than to a - * query. + * ONLYOFFICE asks for the list once and filters it in the editor as the comment + * is typed, so there is nothing to search on here — which is why this cannot go + * through `searchLocalUsers`, whose pattern match is what makes it safe to run + * on user input in the first place. Bounded by `limit` for the same reason a + * search is: an unbounded list is a mistake waiting for a large deployment. */ const listUsersForMentions = async (limit = 100) => { try { const db = await getDb(); const rows = db .prepare( - `SELECT id, email, username, display_name - FROM users - ORDER BY display_name ASC, email ASC - LIMIT ?` + ` + SELECT id, email, username, display_name + FROM users + ORDER BY display_name ASC, email ASC + LIMIT ? + ` ) .all(limit); @@ -111,7 +115,7 @@ const listUsersForMentions = async (limit = 100) => { }; module.exports = { - listUsersForMentions, searchUsersForMentions, searchLocalUsers, + listUsersForMentions, }; diff --git a/backend/src/services/userVolumesService.js b/backend/src/services/userVolumesService.js index 82ab52ed5..b82fc6f84 100644 --- a/backend/src/services/userVolumesService.js +++ b/backend/src/services/userVolumesService.js @@ -4,8 +4,6 @@ const fs = require('fs/promises'); const { getDb } = require('./db'); const { generateId, nowIso } = require('../utils/ids'); - - const RESERVED_VOLUME_LABELS = new Set(['personal', 'share', 'volumes']); const assertValidVolumeLabel = (labelRaw) => { diff --git a/backend/src/services/users/index.js b/backend/src/services/users/index.js index 4e9a262da..e52fe267e 100644 --- a/backend/src/services/users/index.js +++ b/backend/src/services/users/index.js @@ -35,8 +35,8 @@ module.exports = { // OIDC authentication getOrCreateOidcUser: oidcAuth.getOrCreateOidcUser, - deriveRolesFromClaims: oidcAuth.deriveRolesFromClaims, rolesFromClaimsAreAuthoritative: oidcAuth.rolesFromClaimsAreAuthoritative, + deriveRolesFromClaims: oidcAuth.deriveRolesFromClaims, // Request user handling getRequestUser: requestUser.getRequestUser, diff --git a/backend/src/services/users/management.js b/backend/src/services/users/management.js index d6d0e6699..eea8cabe2 100644 --- a/backend/src/services/users/management.js +++ b/backend/src/services/users/management.js @@ -1,5 +1,5 @@ const { getDb } = require('../db'); -const { toClientUser, toShareableUser, normalizeEmail, nowIso } = require('./utils'); +const { toClientUser, toShareableUser, normalizeEmail, nowIso, usernameTaken } = require('./utils'); const { countAdmins } = require('./queries'); const listUsers = async () => { @@ -75,6 +75,12 @@ const updateUserProfile = async ({ userId, email, username, displayName }) => { if (typeof username === 'string') { const trimmed = username.trim(); + // A username is something to sign in with, so it has to name one account. + if (trimmed && usernameTaken(db, trimmed, userId)) { + const err = new Error('Username already in use.'); + err.status = 409; + throw err; + } updates.push('username = ?'); values.push(trimmed || null); } @@ -132,8 +138,24 @@ const deleteUser = async ({ userId }) => { /* ignore parse errors */ } - // Delete user (cascade will delete auth_methods) - db.prepare('DELETE FROM users WHERE id = ?').run(userId); + // One transaction. The account goes (its auth_methods cascade with it); its + // folder name stays reserved while its folder is on disk (personalFolders.js); + // and the rows that only ever described this account go too — none of these + // tables points at users through a foreign key, so nothing else removes them. + db.transaction(() => { + if (row.personal_folder_name) { + db.prepare( + 'INSERT OR REPLACE INTO personal_folder_reservations (name, user_id, reserved_at) VALUES (?, ?, ?)' + ).run(row.personal_folder_name, userId, new Date().toISOString()); + } + db.prepare('DELETE FROM folder_preferences WHERE user_id = ?').run(userId); + db.prepare('DELETE FROM recent_destinations WHERE user_id = ?').run(userId); + db.prepare('DELETE FROM auth_locks WHERE key = ?').run(userId); + // Its API tokens do cascade, and are removed here all the same: a way into + // an account is the one leftover worth writing twice. + db.prepare('DELETE FROM api_tokens WHERE user_id = ?').run(userId); + db.prepare('DELETE FROM users WHERE id = ?').run(userId); + })(); return true; }; diff --git a/backend/src/services/users/requestUser.js b/backend/src/services/users/requestUser.js index 5693e9154..379486708 100644 --- a/backend/src/services/users/requestUser.js +++ b/backend/src/services/users/requestUser.js @@ -2,6 +2,22 @@ const { getDb } = require('../db'); const { auth: envAuthConfig } = require('../../config/index'); const { toClientUser, normalizeEmail } = require('./utils'); const { deriveRolesFromClaims } = require('./oidcAuth'); +const { claimPersonalFolderName } = require('../personalFolders'); + +/** + * An account that has no folder name yet gets one here. + * + * The migration gave every account that existed a name; this covers the ones + * created since, wherever they were created from, without every creation path + * having to remember. It writes once in an account's life and reads a column + * that was already loaded, so the cost after that is a null check. + */ +const withPersonalFolder = (db, row) => { + if (row && !row.personal_folder_name) { + row.personal_folder_name = claimPersonalFolderName(db, row); + } + return row; +}; const getRequestUser = async (req) => { // Synthetic or pre-populated user (e.g., AUTH_ENABLED=false) @@ -22,10 +38,10 @@ const getRequestUser = async (req) => { const db = await getDb(); const row = db.prepare('SELECT * FROM users WHERE id = ?').get(req.apiToken.userId); if (!row) return null; - const user = toClientUser(row); + const user = toClientUser(withPersonalFolder(db, row)); if (user) { // Which kind of account this is lives in `auth_methods` rather than on - // the row. + // the row — the column that used to say so was carried there years ago. const local = db .prepare( `SELECT 1 FROM auth_methods @@ -42,7 +58,7 @@ const getRequestUser = async (req) => { if (req?.session?.localUserId) { const db = await getDb(); const row = db.prepare('SELECT * FROM users WHERE id = ?').get(req.session.localUserId); - const user = toClientUser(row); + const user = toClientUser(withPersonalFolder(db, row)); if (user) { user.provider = 'local'; } @@ -73,7 +89,7 @@ const getRequestUser = async (req) => { if (authMethod) { const row = db.prepare('SELECT * FROM users WHERE id = ?').get(authMethod.user_id); - const user = toClientUser(row); + const user = toClientUser(withPersonalFolder(db, row)); if (user) { user.provider = 'oidc'; user.oidcIssuer = issuer; @@ -111,6 +127,21 @@ const getRequestUser = async (req) => { roles, createdAt: null, updatedAt: null, + // The subject, not the username. + // + // This account has no row yet, so there is no claimed folder name to + // carry and nothing to claim one against. Left null, the folder would be + // derived from `USER_FOLDER_NAME_ORDER` — and the order the reference + // recommends for reusing /home puts `username` first, which two + // identities from two providers can share. The claim mechanism exists to + // stop exactly that, and it cannot run here. + // + // The subject is unique to the provider that issued it, so it is a + // folder of this account's own. It is deliberately not the folder the + // account will get once its row exists: that one is claimed, recorded + // and permanent, and guessing at it here would be handing out a name + // nothing had reserved. + personalFolderName: `oidc-${claims.sub}`, }; } catch (_) { return null; diff --git a/backend/src/services/versions/index.js b/backend/src/services/versions/index.js index 2a24cb858..e69c36b59 100644 --- a/backend/src/services/versions/index.js +++ b/backend/src/services/versions/index.js @@ -30,6 +30,8 @@ const logger = require('../../utils/logger'); const { ensureValidName, normalizeRelativePath } = require('../../utils/pathUtils'); const { ACTIONS, authorizeAndResolve, authorizePath } = require('../authorizationService'); const { getDb } = require('../db'); +const folderSizeHooks = require('../folderSizeHooks'); +const recentDestinations = require('../recentDestinationsService'); const { readTextFile } = require('../textEditorService'); const clock = require('../trash/clock'); const trashStore = require('../trash/store'); @@ -287,27 +289,23 @@ const readVersionText = async (context, relativePath, versionId) => { /** * After a restore, an editor still open on the file holds the content it - * replaced. Its next save is set aside as a version of its own rather than - * written over what was just restored, and the document is given a fresh - * identity so whoever opens it next gets what was restored rather than the - * Document Server's cached copy of what it replaced. + * replaced. Its next save is set aside rather than written, and the document is + * given a fresh identity so whoever opens it next gets what was restored. */ const markRestored = async (absolutePath, relative) => { try { const db = await getDb(); const file = await historyOf(db, absolutePath); if (file) store.setRestoredAt(db, file.id, clock.nowIso()); - // Required here rather than at the top: the key service is part of the - // office integration, which reaches back into the versions. - // eslint-disable-next-line global-require await require('../onlyofficeDocumentKeyService').releaseDocumentKey(relative); } catch (error) { logger.warn({ err: error, absolutePath }, 'A restore could not be announced to open editors'); } }; -/** Put a version's content into a file, the way every save does. */ +/** Put a version's content into a file, the way every save does, telling folder sizes. */ const writeVersionInto = async (located, destination, context) => { + const previous = await fsp.stat(destination).catch(() => null); const result = await operations.saveFile( destination, (temporaryPath) => @@ -319,6 +317,16 @@ const writeVersionInto = async (located, destination, context) => { explicit: true, } ); + try { + const after = await fsp.stat(destination); + if (previous?.isFile()) { + await folderSizeHooks.onFileReplaced(destination, previous.size, after.size); + } else { + await folderSizeHooks.onFileWritten(destination, after.size); + } + } catch (error) { + logger.debug({ err: error, destination }, 'Folder sizes were not told about a restore'); + } return result; }; @@ -376,6 +384,23 @@ const copyVersionTo = async (context, relativePath, versionId, { destination, na { purpose: 'restore' } ); const finalName = placed.name; + try { + const after = await fsp.stat(placed.path); + await folderSizeHooks.onFileWritten(placed.path, after.size); + } catch (error) { + logger.debug( + { err: error, destination: placed.path }, + 'Folder sizes were not told about a copy' + ); + } + + if (context?.user?.id) { + try { + await recentDestinations.record(context.user.id, folder.relative); + } catch (error) { + logger.debug({ err: error }, 'The destination was not remembered'); + } + } return { path: `${folder.relative}/${finalName}`, name: finalName }; }; diff --git a/backend/src/services/versions/lifecycle.js b/backend/src/services/versions/lifecycle.js index efcf12c69..016f93617 100644 --- a/backend/src/services/versions/lifecycle.js +++ b/backend/src/services/versions/lifecycle.js @@ -29,11 +29,18 @@ const operations = require('./operations'); const { thinVersions } = require('./policy'); const store = require('./store'); -const escapeLike = (value) => String(value).replace(/[\\%_]/g, '\\$&'); - -/** `column` is `prefix`, or something inside it. */ -const under = (column) => `(${column} = ? OR ${column} LIKE ? ESCAPE '\\')`; -const underValues = (prefix) => [prefix, `${escapeLike(prefix)}/%`]; +/** + * `column` is `prefix`, or something inside it: every path that begins with + * `prefix/` sorts at or after it and before `prefix0`, `0` being the character + * right after `/`. + * + * It was `LIKE 'prefix/%'`, which ignores case for ASCII as SQLite's LIKE + * always does. Moving `Docs` reassigned the histories of `docs/…` to files + * under the new name, and deleting it for good purged them — another folder, + * on a Linux volume, and its versions gone with the wrong one. + */ +const under = (column) => `(${column} = ? OR (${column} >= ? AND ${column} < ?))`; +const underValues = (prefix) => [prefix, `${prefix}/`, `${prefix}0`]; /** What is left of `full` inside `prefix`: '' for the prefix itself. */ const inside = (full, prefix) => (full === prefix ? '' : full.slice(prefix.length + 1)); @@ -158,7 +165,6 @@ const relinkRestored = (db, { itemId, entryPath = null, target, restorePath }) = const purgeFiles = async (fileIds) => { for (const fileId of fileIds) { try { - // eslint-disable-next-line no-await-in-loop await operations.purgeFile(fileId); } catch (error) { logger.warn({ err: error, fileId }, 'A file history could not be purged'); @@ -186,6 +192,58 @@ const historiesUnder = async (db, absolutePath) => { return { root: located.root, prefix, rows }; }; +/** + * What deleting here for good would destroy, before anyone has agreed to it. + * + * Versions are the one thing a deletion takes that cannot be seen from the + * folder: the file is on screen and its history is not, so "delete" reads as + * one file going and takes ten earlier copies of it with it. Asked for a path + * or a whole tree, because a folder is deleted the same way and every file + * under it brings its own. + * + * Counted in one query rather than over the rows, so a folder with a thousand + * versioned files under it is one question to the database and not a thousand + * — and no list of ids long enough to run into the limit on how many a + * statement may carry. + */ +const countUnder = async (absolutePath) => { + const none = { files: 0, versions: 0, bytes: 0 }; + try { + const db = await getDb(); + const located = await zones.locateZoneRoot(path.resolve(absolutePath)); + if (!located.root) return none; + const zoneIds = trashStore + .listZones(db) + .filter((zone) => zone.root === located.root) + .map((zone) => zone.id); + if (zoneIds.length === 0) return none; + + const prefix = toRelative(located.root, path.resolve(absolutePath)); + const row = db + .prepare( + `SELECT COUNT(DISTINCT vf.id) AS files, + COUNT(v.id) AS versions, + COALESCE(SUM(v.size_bytes), 0) AS bytes + FROM version_files vf + JOIN file_versions v ON v.file_id = vf.id AND v.state = 'kept' + WHERE vf.zone_id IN (${zoneIds.map(() => '?').join(', ')}) + AND vf.state IN ('live', 'orphaned') AND ${under('vf.relative_path')}` + ) + .get(...zoneIds, ...underValues(prefix)); + + return { + files: Number(row?.files) || 0, + versions: Number(row?.versions) || 0, + bytes: Number(row?.bytes) || 0, + }; + } catch (error) { + // A count is not worth failing a confirmation over: the dialog says what + // it knows, and the deletion itself is unchanged. + logger.debug({ err: error, absolutePath }, 'File versions were not counted for a deletion'); + return none; + } +}; + /** * A file or folder the application renamed or moved: the histories at the old * path, or under it, now name the new one — in another zone when it went to @@ -235,62 +293,14 @@ const onMoved = async (fromAbsolute, toAbsolute) => { * A file or folder deleted for good by the application: its histories go with * it, now — the space comes back at once rather than at the next pass. */ -/** - * What a deletion at `absolutePath` is about to take with it. - * - * Counted before anything goes, because afterwards there is nothing left to - * count — and a deletion that took ten earlier copies of a file said exactly - * as much as one that took none. Asked for a path or a whole tree, because a - * folder is deleted the same way and every file under it brings its own. - * - * One query rather than one per row, so a folder with a thousand versioned - * files under it is a single question to the database. - */ -const countUnder = async (absolutePath) => { - const none = { files: 0, versions: 0, bytes: 0 }; - try { - const db = await getDb(); - const located = await zones.locateZoneRoot(path.resolve(absolutePath)); - if (!located.root) return none; - const zoneIds = trashStore - .listZones(db) - .filter((zone) => zone.root === located.root) - .map((zone) => zone.id); - if (zoneIds.length === 0) return none; - - const prefix = toRelative(located.root, path.resolve(absolutePath)); - const row = db - .prepare( - `SELECT COUNT(DISTINCT vf.id) AS files, - COUNT(v.id) AS versions, - COALESCE(SUM(v.size_bytes), 0) AS bytes - FROM version_files vf - JOIN file_versions v ON v.file_id = vf.id AND v.state = 'kept' - WHERE vf.zone_id IN (${zoneIds.map(() => '?').join(', ')}) - AND vf.state IN ('live', 'orphaned') AND ${under('vf.relative_path')}` - ) - .get(...zoneIds, ...underValues(prefix)); - - return { - files: Number(row?.files) || 0, - versions: Number(row?.versions) || 0, - bytes: Number(row?.bytes) || 0, - }; - } catch (error) { - // A count is not worth failing a deletion over: the line written down says - // what it knows, and the deletion itself is unchanged. - logger.debug({ err: error, absolutePath }, 'File versions were not counted for a deletion'); - return none; - } -}; - const onDeleted = async (absolutePath) => { const none = { files: 0, versions: 0, bytes: 0 }; try { const db = await getDb(); const { rows } = await historiesUnder(db, absolutePath); if (rows.length === 0) return none; - // Counted before they go, so a deletion can say what it took. + // Counted before they go, so a deletion can say what it took. Afterwards + // there is nothing left to count. const taken = await countUnder(absolutePath); db.transaction(() => { for (const row of rows) store.setFileState(db, row.id, 'purging'); @@ -328,7 +338,6 @@ const reviewZone = async ( .filter((file) => file.state === 'live' || file.state === 'orphaned'); const checked = []; for (let index = 0; index < files.length; index += REVIEW_BATCH) { - // eslint-disable-next-line no-await-in-loop const batch = await Promise.all( files.slice(index, index + REVIEW_BATCH).map(async (file) => { const stats = await lstatOrNull(path.join(zone.root, ...file.relativePath.split('/'))); @@ -367,7 +376,6 @@ const reviewZone = async ( for (const file of store.listFiles(db, { zoneId: zone.id, state: 'orphaned' })) { const orphanedAt = Date.parse(file.orphanedAt || ''); if (Number.isFinite(orphanedAt) && orphanedAt > expiry) continue; - // eslint-disable-next-line no-await-in-loop await purgeFiles([file.id]); report.expired += 1; } @@ -414,7 +422,6 @@ const zoneVersions = (db, zone, { now, settings }) => { }; module.exports = { - countUnder, relinkTrashed, filesInTrashItem, targetForRestore, @@ -422,6 +429,7 @@ module.exports = { purgeFiles, onMoved, onDeleted, + countUnder, reviewZone, zoneVersions, }; diff --git a/backend/src/services/versions/operations.js b/backend/src/services/versions/operations.js index 5a923086a..19e636c97 100644 --- a/backend/src/services/versions/operations.js +++ b/backend/src/services/versions/operations.js @@ -29,9 +29,10 @@ const fsp = require('fs/promises'); const path = require('path'); const { generateId } = require('../../utils/ids'); -const { placeWithoutOverwrite } = require('../../utils/placeWithoutOverwrite'); const logger = require('../../utils/logger'); +const { placeWithoutOverwrite } = require('../../utils/placeWithoutOverwrite'); const { getDb } = require('../db'); +const { track: trackInFlight } = require('../inFlightFiles'); const clock = require('../trash/clock'); const failpoints = require('../trash/failpoints'); const { admission } = require('../trash/policy'); @@ -141,9 +142,7 @@ const placeOf = async (absolutePath) => { /** Whether content of this size could ever fit in the zone's budget. */ const tooLargeFor = async (root, size) => { - // eslint-disable-next-line global-require const maintenance = require('../trash/maintenance'); - // eslint-disable-next-line global-require const { getTrashSettings } = require('../trash/settings'); const { budgetBytes } = await maintenance.limitsFor(root, await getTrashSettings()); return admission({ size, budgetBytes }) === 'too-large'; @@ -151,7 +150,6 @@ const tooLargeFor = async (root, size) => { const requestPass = () => { try { - // eslint-disable-next-line global-require require('../trash/maintenance').requestPass(); } catch (error) { logger.debug({ err: error }, 'No maintenance pass could be requested after a capture'); @@ -212,7 +210,6 @@ const purgeFile = async (fileId) => { states: ['capturing', 'kept', 'purging'], }); for (const version of versions) { - // eslint-disable-next-line no-await-in-loop const outcome = await purgeVersion(version.id); if (outcome.status !== 'purged' && outcome.status !== 'missing') left += 1; } @@ -234,7 +231,6 @@ const thinFile = async (fileId) => { })); const drop = thinVersions({ versions, now: clock.now(), settings }); for (const entry of drop) { - // eslint-disable-next-line no-await-in-loop await purgeVersion(entry.id); } return drop; @@ -498,11 +494,13 @@ const temporaryPathFor = (absolutePath, purpose = 'save') => */ const saveFile = async (absolutePath, writeContent, meta = {}) => { const temporaryPath = temporaryPathFor(absolutePath, meta.purpose || 'save'); + const inFlight = trackInFlight(temporaryPath, 'temporary-file'); try { await writeContent(temporaryPath); return await replaceWithTemporary(absolutePath, temporaryPath, meta); } finally { await fsp.rm(temporaryPath, { force: true }).catch(() => {}); + inFlight.release(); } }; @@ -511,12 +509,12 @@ const saveFile = async (absolutePath, writeContent, meta = {}) => { * free name after it, "notes (1).md". Answers the name and path it took. * * The content is written beside the name by `writeContent`, then put under it - * by a move that never replaces anything. A file that did not exist has no - * history, so nothing is recorded, as `saveFile` records nothing when it - * creates a file. Whatever happens, no temporary file is left behind. - * - * `versions/index.js` has called this since copying a version to a new file - * was added, and it was never written: every such copy answered a 500. + * by a move that never replaces anything. Going through `saveFile` with a name + * chosen beforehand meant a file that arrived under that name while the content + * was being written was replaced, its content kept as an earlier version of a + * file it had nothing to do with. A file that did not exist has no history, so + * nothing is recorded, as `saveFile` records nothing when it creates a file. + * Whatever happens, no temporary file is left behind. * * @param {string} directory * @param {string} desiredName @@ -526,11 +524,13 @@ const saveFile = async (absolutePath, writeContent, meta = {}) => { */ const saveNewFile = async (directory, desiredName, writeContent, meta = {}) => { const temporaryPath = temporaryPathFor(path.join(directory, desiredName), meta.purpose || 'save'); + const inFlight = trackInFlight(temporaryPath, 'temporary-file'); try { await writeContent(temporaryPath); return await placeWithoutOverwrite(temporaryPath, directory, desiredName); } finally { await fsp.rm(temporaryPath, { force: true }).catch(() => {}); + inFlight.release(); } }; @@ -598,20 +598,17 @@ const recoverZone = async ( } const file = store.getFile(db, row.fileId); const livePath = file?.state === 'live' ? absolutePathOf(db, file) : null; - // eslint-disable-next-line no-await-in-loop const [content, live] = await Promise.all([ lstatOrNull(payload), livePath ? lstatOrNull(livePath) : null, ]); if (!row.aside && content && live && content.ino === live.ino && content.dev === live.dev) { - // eslint-disable-next-line no-await-in-loop await fsp.rm(payload, { force: true }); store.deleteVersion(db, row.id); report.undone += 1; } else if (!row.aside && content && livePath && !live) { // Renamed out of the way on a filesystem without hard links, and the // new content never took its place: the file gets its content back. - // eslint-disable-next-line no-await-in-loop await fsp.rename(payload, livePath); store.deleteVersion(db, row.id); report.undone += 1; @@ -620,7 +617,6 @@ const recoverZone = async ( report.finished += 1; } } else if (row.state === 'purging') { - // eslint-disable-next-line no-await-in-loop await fsp.rm(payload, { force: true }); store.deleteVersion(db, row.id); report.purged += 1; @@ -635,7 +631,6 @@ const recoverZone = async ( if (onDisk.has(row.id)) continue; // Looked at again: a capture that finished since the directory was read // is not a loss. - // eslint-disable-next-line no-await-in-loop if (!(await lstatOrNull(path.join(directory, row.id)))) vanished.push(row); } if ( @@ -668,7 +663,6 @@ const recoverZone = async ( for (const version of versions) { if (version.zoneId !== zone.id || inflight.has(version.id)) continue; if (!VERSION_ID_PATTERN.test(version.id)) continue; - // eslint-disable-next-line no-await-in-loop await fsp.rm(contentPath(zone, version.id), { force: true }); store.deleteVersion(db, version.id); report.purged += 1; @@ -686,10 +680,8 @@ const recoverZone = async ( if (known.has(name) || stillKnown.has(name) || inflight.has(name)) continue; if (!VERSION_ID_PATTERN.test(name)) continue; const absolute = path.join(directory, name); - // eslint-disable-next-line no-await-in-loop const stats = await lstatOrNull(absolute); if (!stats || (graceMs > 0 && clock.now() - stats.ctimeMs < graceMs)) continue; - // eslint-disable-next-line no-await-in-loop await fsp.rm(absolute, { recursive: true, force: true }); report.removedContents += 1; } @@ -704,10 +696,10 @@ module.exports = { hashFile, absolutePathOf, placeOf, - saveNewFile, temporaryPathFor, replaceWithTemporary, saveFile, + saveNewFile, authorOf, locateVersion, purgeVersion, diff --git a/backend/src/services/versions/settings.js b/backend/src/services/versions/settings.js index c6a24bb16..164535615 100644 --- a/backend/src/services/versions/settings.js +++ b/backend/src/services/versions/settings.js @@ -5,7 +5,6 @@ * expects the next save to see it. */ const getVersionSettings = async () => { - // eslint-disable-next-line global-require const { getSystemSettings } = require('../settingsService'); return (await getSystemSettings()).versions; }; diff --git a/backend/src/services/versions/verify.js b/backend/src/services/versions/verify.js index f8b1c0447..61fc59e54 100644 --- a/backend/src/services/versions/verify.js +++ b/backend/src/services/versions/verify.js @@ -49,7 +49,6 @@ const verifyVersions = async (zone, { measureSizes = true } = {}) => { continue; } if (measureSizes) { - // eslint-disable-next-line no-await-in-loop const stats = await fsp.lstat(path.join(directory, row.id)); if (stats.size !== row.size) { violations.push({ diff --git a/backend/src/services/wopiLockService.js b/backend/src/services/wopiLockService.js index a0753b3a9..4cb517374 100644 --- a/backend/src/services/wopiLockService.js +++ b/backend/src/services/wopiLockService.js @@ -56,7 +56,6 @@ const resetAllLocks = () => { }; module.exports = { - DEFAULT_LOCK_TTL_MS, getLock, tryLock, tryUnlock, diff --git a/backend/src/utils/bootstrap.js b/backend/src/utils/bootstrap.js index 6a769f2e5..66abdb19e 100644 --- a/backend/src/utils/bootstrap.js +++ b/backend/src/utils/bootstrap.js @@ -49,7 +49,7 @@ const ensureEnvAdminUser = async () => { const username = normalizedEmail.split('@')[0] || 'admin'; const ensureAdminRole = async (userId, rolesJson) => { - let roles = []; + let roles; try { roles = JSON.parse(rolesJson || '[]'); } catch (_) { diff --git a/backend/src/utils/env.js b/backend/src/utils/env.js index dca4b8b92..86feabbfd 100644 --- a/backend/src/utils/env.js +++ b/backend/src/utils/env.js @@ -1,3 +1,52 @@ +const fs = require('node:fs'); + +/** + * A secret, taken from the environment or from the file it names. + * + * `docker inspect` prints every variable a container was started with, so a + * secret passed as `ONLYOFFICE_SECRET=…` is readable by anyone who can reach the + * daemon and stays in the container's stored configuration long after the + * process is gone. The convention around that — Postgres, Nextcloud and most + * images that take credentials — is a companion `_FILE` variable naming a file + * to read instead, so an orchestrator can mount the value from a secret store + * and leave the environment empty. + * + * Names are tried in order, each as a direct value then as a `_FILE` pointer, so + * a legacy alias only answers when the current name says nothing. + * + * A `_FILE` that cannot be read throws rather than resolving to null: the + * operator asked for that file by name, and carrying on would quietly start the + * server with whatever the secret protects turned off. + */ +const readSecret = (...names) => { + for (const name of names) { + const direct = process.env[name]; + if (direct) return direct; + + const file = process.env[`${name}_FILE`]; + if (!file) continue; + + let contents; + try { + contents = fs.readFileSync(file, 'utf8'); + } catch (error) { + throw new Error(`${name}_FILE: cannot read ${file} (${error.code || error.message})`, { + cause: error, + }); + } + + // Trailing newlines are what `echo secret > file` leaves behind, and they + // would travel into signatures and comparisons unnoticed. + const value = contents.trim(); + if (!value) { + throw new Error(`${name}_FILE: ${file} is empty`); + } + return value; + } + + return null; +}; + const normalizeBoolean = (value) => { if (typeof value !== 'string') return null; const normalized = value.trim().toLowerCase(); @@ -22,7 +71,11 @@ const parseByteSize = (value) => { const s = value.trim(); if (!s) return null; - const m = s.match(/^([0-9]+)\s*([kKmMgGtT]?)b?$/); + // `5MB` is how everyone writes it, our own README included, and it used to be + // rejected outright for the capital B — leaving the setting silently at its + // default. Spaces and either case are accepted; the unit is what carries the + // meaning, and `5 mb`, `5MB` and `5m` all mean the same thing to a reader. + const m = s.match(/^([0-9]+)\s*([kKmMgGtT]?)[bB]?$/); if (!m) return null; const num = Number(m[1]); if (!Number.isFinite(num)) return null; @@ -35,4 +88,5 @@ const parseByteSize = (value) => { module.exports = { normalizeBoolean, parseByteSize, + readSecret, }; diff --git a/backend/src/utils/fsUtils.js b/backend/src/utils/fsUtils.js index 2fe3f8806..1194c4a15 100644 --- a/backend/src/utils/fsUtils.js +++ b/backend/src/utils/fsUtils.js @@ -1,14 +1,19 @@ const fs = require('fs/promises'); -const ensureDir = async (targetPath) => { - await fs.mkdir(targetPath, { recursive: true }); -}; +/** + * Make sure a folder is there, and say what had to be created for it. + * + * `mkdir` with `recursive` answers the topmost folder it created, or nothing + * when they all existed already — which is what tells an operation that fails + * later exactly what it added, and nothing more. + */ +const ensureDir = async (targetPath) => fs.mkdir(targetPath, { recursive: true }); const pathExists = async (targetPath) => { try { await fs.access(targetPath); return true; - } catch (error) { + } catch (_) { return false; } }; diff --git a/backend/src/utils/logSanitizer.js b/backend/src/utils/logSanitizer.js index 6b351ce1c..d854d529b 100644 --- a/backend/src/utils/logSanitizer.js +++ b/backend/src/utils/logSanitizer.js @@ -9,6 +9,9 @@ const SENSITIVE_QUERY_PARAMETERS = new Set([ 'logout_token', 'refresh_token', 'state', + // The signature on a /static/thumbnails URL, which unlocks that file for + // anyone holding it until it expires. + 't', 'token', 'jwt', ]); diff --git a/backend/src/utils/logger.js b/backend/src/utils/logger.js index df3c9ebfa..7c41b66f1 100644 --- a/backend/src/utils/logger.js +++ b/backend/src/utils/logger.js @@ -1,17 +1,6 @@ const pino = require('pino'); const loggingConfig = require('../config/logging'); -const prettyOptions = { - colorize: true, - levelFirst: true, - translateTime: 'SYS:standard', - ignore: 'pid,hostname', -}; - -// Keep development formatting in-process. The worker-backed Pino transport can -// crash under Node's watch runner while the worker is starting or stopping. -const prettyStream = loggingConfig.isDebug ? require('pino-pretty')(prettyOptions) : undefined; - /** * Values that must never reach the log files, wherever they are attached. * @@ -35,15 +24,23 @@ const REDACTED_PATHS = [ '*.secret', ]; -const logger = pino( - { - level: loggingConfig.level, - base: { service: 'nextExplorer-backend' }, - redact: { paths: REDACTED_PATHS, censor: '[redacted]' }, - timestamp: pino.stdTimeFunctions.isoTime, - }, - prettyStream -); +const logger = pino({ + level: loggingConfig.level, + base: { service: 'nextExplorer-backend' }, + redact: { paths: REDACTED_PATHS, censor: '[redacted]' }, + timestamp: pino.stdTimeFunctions.isoTime, + transport: loggingConfig.isDebug + ? { + target: 'pino-pretty', + options: { + colorize: true, + levelFirst: true, + translateTime: 'SYS:standard', + ignore: 'pid,hostname', + }, + } + : undefined, +}); logger.debug({ level: loggingConfig.level }, 'Logger initialized'); diff --git a/backend/src/utils/mapWithConcurrency.js b/backend/src/utils/mapWithConcurrency.js index adb87bc4d..fccbefd19 100644 --- a/backend/src/utils/mapWithConcurrency.js +++ b/backend/src/utils/mapWithConcurrency.js @@ -28,7 +28,6 @@ const mapWithConcurrency = async (items, mapper, concurrency = DEFAULT_CONCURREN const index = next; next += 1; if (index >= list.length) return; - // eslint-disable-next-line no-await-in-loop results[index] = await mapper(list[index], index); } }); diff --git a/backend/src/utils/ownedTree.js b/backend/src/utils/ownedTree.js index d6bf5222c..eb824fd71 100644 --- a/backend/src/utils/ownedTree.js +++ b/backend/src/utils/ownedTree.js @@ -40,7 +40,6 @@ const takeInventory = async (root) => { owned.add(identityOf(stats)); if (!stats.isDirectory()) return; for (const name of await fs.readdir(entryPath)) { - // eslint-disable-next-line no-await-in-loop await walk(path.join(entryPath, name)); } }; @@ -73,7 +72,6 @@ const removeInventoried = async (root, inventory) => { } for (const name of await fs.readdir(entryPath)) { - // eslint-disable-next-line no-await-in-loop await visit(path.join(entryPath, name)); } if (!ours) { diff --git a/backend/src/utils/pathUtils.js b/backend/src/utils/pathUtils.js index 158e77a3c..a765b4766 100644 --- a/backend/src/utils/pathUtils.js +++ b/backend/src/utils/pathUtils.js @@ -5,7 +5,7 @@ const { directories, features, personal } = require('../config/index'); const { pathExists } = require('./fsUtils'); const { cachedForRequest, hasRequestContext } = require('./requestContext'); const logger = require('./logger'); -const { ForbiddenError, ValidationError } = require('../errors/AppError'); +const { ForbiddenError, NotFoundError, ValidationError } = require('../errors/AppError'); const { ZONE_DIRECTORY_NAME } = require('../config/constants'); const NAME_INVALID_PATTERN = /[\\/]/; @@ -83,9 +83,6 @@ const normalizeRelativePath = (relativePath = '') => { } if (normalized === '..' || normalized.startsWith('..' + path.sep)) { - // The request's fault, not the server's: a plain Error reached the browser as a - // 500, so a path that leaves the volume read as a server fault rather than a - // refusal — and a 500 is what a caller retries. throw new ValidationError('Invalid path. Traversal outside the volume root is not allowed.'); } @@ -187,7 +184,7 @@ const assertRealPathWithinRoot = async ( const expectedRoot = realRoot(root); const rootWithSep = root.endsWith(path.sep) ? root : `${root}${path.sep}`; const realWithSep = expectedRoot.endsWith(path.sep) ? expectedRoot : `${expectedRoot}${path.sep}`; - const outside = () => new Error(`Resolved path is outside ${label}.`); + const outside = () => new ForbiddenError(`Resolved path is outside ${label}.`); const contained = (candidate) => candidate === expectedRoot || candidate.startsWith(realWithSep); const namedInside = (candidate) => candidate === root || @@ -226,7 +223,6 @@ const assertRealPathWithinRoot = async ( let candidate = absolutePath; for (;;) { - // eslint-disable-next-line no-await-in-loop const realCandidate = await realpathOrNull(candidate); if (realCandidate) { @@ -234,11 +230,10 @@ const assertRealPathWithinRoot = async ( return; } - // eslint-disable-next-line no-await-in-loop const link = await readLinkOrNull(candidate); if (link !== null) { if (hops >= MAX_SYMLINK_HOPS) { - throw new Error('Too many levels of symbolic links.'); + throw new ForbiddenError('Too many levels of symbolic links.'); } const target = path.resolve(path.dirname(candidate), link); // The target of a broken link may not exist anywhere, so there is no real @@ -318,13 +313,15 @@ const resolveVolumePath = async (relativePath = '') => { const absolutePath = path.resolve(directories.volume, safeRelativePath); if (absolutePath !== directories.volume && !absolutePath.startsWith(directories.volumeWithSep)) { - throw new Error('Resolved path is outside the configured volume root.'); + throw new ForbiddenError('Resolved path is outside the configured volume root.'); } await assertRealPathWithinRoot(absolutePath, directories.volume); if (isInsidePersonalRoot(absolutePath)) { - throw new Error('Personal folders are reached through the personal space, not the volume.'); + throw new ForbiddenError( + 'Personal folders are reached through the personal space, not the volume.' + ); } return absolutePath; @@ -373,24 +370,24 @@ const findAvailableFolderName = async (directory, baseName = 'Untitled Folder') const ensureValidName = (rawName) => { if (typeof rawName !== 'string') { - throw new Error('A valid name is required.'); + throw new ValidationError('A valid name is required.'); } const name = rawName; if (!name.trim()) { - throw new Error('Name cannot be empty.'); + throw new ValidationError('Name cannot be empty.'); } if (NAME_INVALID_PATTERN.test(name)) { - throw new Error('Name cannot contain path separators.'); + throw new ValidationError('Name cannot contain path separators.'); } if (name.includes('\0')) { - throw new Error('Name contains invalid characters.'); + throw new ValidationError('Name contains invalid characters.'); } if (RESERVED_NAMES.has(name)) { - throw new Error('This name is not allowed.'); + throw new ValidationError('This name is not allowed.'); } return name; @@ -528,10 +525,10 @@ const getUserFolderName = (user = {}) => { */ const getUserRootDir = async (user) => { if (!PERSONAL_ENABLED) { - throw new Error('Personal directories are disabled.'); + throw new ForbiddenError('Personal directories are disabled.'); } if (!user || !user.id) { - throw new Error('User context is required for personal paths.'); + throw new ForbiddenError('User context is required for personal paths.'); } const base = directories.userRoot; @@ -539,7 +536,7 @@ const getUserRootDir = async (user) => { const userRoot = path.resolve(base, folderName); if (userRoot !== base && !userRoot.startsWith(directories.userRootWithSep)) { - throw new Error('Resolved user directory is outside the configured user root.'); + throw new ForbiddenError('Resolved user directory is outside the configured user root.'); } // Failures are left to the operation that follows: it is the one that knows @@ -557,7 +554,7 @@ const resolvePersonalPath = async (relativePath = '', user) => { const absolutePath = path.resolve(userRoot, safeRelativePath); if (absolutePath !== userRoot && !absolutePath.startsWith(userRoot + path.sep)) { - throw new Error('Resolved path is outside the configured user directory.'); + throw new ForbiddenError('Resolved path is outside the configured user directory.'); } await assertRealPathWithinRoot(absolutePath, userRoot, 'the configured user directory'); @@ -591,10 +588,10 @@ const resolveLogicalPath = async ( if (space === 'personal') { if (!PERSONAL_ENABLED) { - throw new Error('Personal directories are disabled.'); + throw new ForbiddenError('Personal directories are disabled.'); } if (!user) { - throw new Error('User context is required for personal paths.'); + throw new ForbiddenError('User context is required for personal paths.'); } // Awaited, which is the whole point of it: `resolvePersonalPath` checks that @@ -636,13 +633,13 @@ const resolveLogicalPath = async ( : userVolume.path + path.sep; if (absolutePath !== userVolume.path && !absolutePath.startsWith(volumePathWithSep)) { - throw new Error('Resolved path is outside the assigned volume.'); + throw new ForbiddenError('Resolved path is outside the assigned volume.'); } await assertRealPathWithinRoot(absolutePath, userVolume.path, 'the assigned volume'); if (reachesIntoPersonalRoot(userVolume.path, absolutePath)) { - throw new Error( + throw new ForbiddenError( 'Personal folders are reached through the personal space, not an assigned volume.' ); } @@ -693,7 +690,7 @@ const resolveSharePath = async ( if (!shareToken) { logger.debug('resolveSharePath: No shareToken found'); - throw new Error('Share token is required'); + throw new ValidationError('Share token is required'); } // Use pre-fetched share if available (optimization to avoid duplicate DB query) @@ -707,7 +704,7 @@ const resolveSharePath = async ( if (!share) { logger.debug({ shareToken }, 'resolveSharePath share not found in database'); - throw new Error('Share not found'); + throw new NotFoundError('Share not found'); } logger.debug( @@ -731,7 +728,7 @@ const resolveSharePath = async ( if (share.sourceSpace === 'personal') { const owner = await getUserById(share.ownerId); if (!owner) { - throw new Error('Share owner not found'); + throw new NotFoundError('Share owner not found'); } const combinedPath = @@ -743,15 +740,15 @@ const resolveSharePath = async ( .split('/') .filter(Boolean); if (!volumeId) { - throw new Error('Share source volume is invalid'); + throw new NotFoundError('Share source volume is invalid'); } const userVolume = await getUserVolumeById(volumeId); if (!userVolume) { - throw new Error('Share source volume not found'); + throw new NotFoundError('Share source volume not found'); } if (String(userVolume.userId) !== String(share.ownerId)) { - throw new Error('Share source volume mismatch'); + throw new NotFoundError('Share source volume mismatch'); } const baseWithinVolume = rest.join('/'); @@ -766,13 +763,13 @@ const resolveSharePath = async ( ? userVolume.path : userVolume.path + path.sep; if (absolutePath !== userVolume.path && !absolutePath.startsWith(volumePathWithSep)) { - throw new Error('Resolved path is outside the assigned volume.'); + throw new ForbiddenError('Resolved path is outside the assigned volume.'); } await assertRealPathWithinRoot(absolutePath, userVolume.path, 'the assigned volume'); if (reachesIntoPersonalRoot(userVolume.path, absolutePath)) { - throw new Error( + throw new ForbiddenError( 'Personal folders are reached through the personal space, not an assigned volume.' ); } @@ -794,7 +791,7 @@ const resolveSharePath = async ( const resolveItemPaths = async (item = {}, options = {}) => { if (!item || typeof item.name !== 'string') { - throw new Error('Each item must include a name.'); + throw new ValidationError('Each item must include a name.'); } const parentPath = item.path || ''; diff --git a/backend/src/utils/placeWithoutOverwrite.js b/backend/src/utils/placeWithoutOverwrite.js index fa953439d..54bce9db6 100644 --- a/backend/src/utils/placeWithoutOverwrite.js +++ b/backend/src/utils/placeWithoutOverwrite.js @@ -150,7 +150,6 @@ const placeWithoutOverwrite = async (source, directory, desiredName, { style = ' const name = candidateName(desiredName, index, style); const target = path.join(directory, name); try { - // eslint-disable-next-line no-await-in-loop await moveNoReplace(source, target); return { name, path: target }; } catch (error) { @@ -176,12 +175,10 @@ const reserveAvailableName = async ( const target = path.join(directory, name); try { if (isDirectory) { - // eslint-disable-next-line no-await-in-loop await fs.mkdir(target); } else { - // eslint-disable-next-line no-await-in-loop const handle = await fs.open(target, 'wx'); - // eslint-disable-next-line no-await-in-loop + await handle.close(); } return { name, path: target }; diff --git a/backend/src/utils/staticServer.js b/backend/src/utils/staticServer.js index d8978bcd0..789204eac 100644 --- a/backend/src/utils/staticServer.js +++ b/backend/src/utils/staticServer.js @@ -1,49 +1,54 @@ const path = require('path'); const fs = require('fs'); const express = require('express'); -const { auth, directories } = require('../config/index'); +const { directories, auth } = require('../config/index'); const logger = require('./logger'); /** - * A thumbnail is served from /static, which the authentication middleware does - * not cover, and its cache name is derived from the file's path — so anybody - * who can guess a path can ask for the picture of it, and a 200 against a 404 - * answers "does this file exist" besides. + * Thumbnails live outside /api, so the auth middleware never sees them. * - * A session would not settle it either: it says who is asking, not what they - * were cleared to see, so a visitor holding a valid session for one share could - * name a thumbnail belonging to another share or to a private folder. + * Their filenames are derived from the file path (`v-.webp`), which + * makes them guessable by anyone who can guess a path — and a 200 vs 404 also + * answers "does this file exist". A session is not enough to decide here: it + * says who is asking, not what they were cleared to see, so any share visitor + * could ask for a filename belonging to another share or a private folder. * - * The decision is made by /api/thumbnails, which runs the real access check and - * signs the one filename it just cleared. This reads that signature back — no - * database, no session, nothing else to get wrong. + * The answer comes from /api/thumbnails, which runs the real access check and + * signs the one filename it just cleared. This handler only verifies that + * signature — no database read, no session, and nothing to confuse. */ const requireThumbnailToken = (req, res, next) => { if (auth.enabled === false) return next(); - // The cache is flat, so a request names one file and nothing else. Taking the - // basename would let a token for "x.webp" unlock "sub/dir/x.webp". + // The cache is flat, so the request names one file and nothing else. Taking + // the basename instead would let a token for "x.webp" unlock "sub/dir/x.webp". let filename; try { filename = decodeURIComponent((req.path || '').replace(/^\/+/, '')); } catch { - // A malformed escape throws, and matches no thumbnail either way. + // A malformed escape sequence throws; it matches no thumbnail either way. return res.status(401).end(); } const token = typeof req.query?.t === 'string' ? req.query.t : ''; - // eslint-disable-next-line global-require const { verifyThumbnailToken } = require('./thumbnailTokens'); if (filename && !filename.includes('/') && verifyThumbnailToken(filename, token)) return next(); logger.debug({ filename }, 'Thumbnail request without a valid token'); - return res.status(401).end(); + res.status(401).end(); }; /** * Configures static file serving for thumbnails, logos, and frontend */ -const configureStaticFiles = (app) => { +/** + * @param {import('express').Application} app + * @param {string} [frontendDirectory] where the built frontend lives. Defaults + * to where the image puts it; a caller passes its own so this can be + * exercised — the single-page fallback is the one route only production + * registers, and it was the one that stopped the server from starting. + */ +const configureStaticFiles = (app, frontendDirectory) => { // Serve thumbnails app.use('/static/thumbnails', requireThumbnailToken, express.static(directories.thumbnails)); logger.debug('Mounted /static/thumbnails'); @@ -57,19 +62,35 @@ const configureStaticFiles = (app) => { logger.warn('Failed to create logos directory', { error: error.message }); } } - app.use('/static/logos', express.static(logosDir)); + // A branding logo may be an SVG, which the browser executes when opened + // directly. The upload only checks the declared MIME type, so the sandbox is + // what actually keeps it from running on the app origin. + app.use( + '/static/logos', + (_req, res, next) => { + res.setHeader('Content-Security-Policy', 'sandbox'); + next(); + }, + express.static(logosDir) + ); logger.debug('Mounted /static/logos'); // Serve frontend SPA - const frontendDir = path.resolve(__dirname, '..', 'public'); + const frontendDir = frontendDirectory || path.resolve(__dirname, '..', 'public'); const indexFile = path.join(frontendDir, 'index.html'); if (fs.existsSync(frontendDir) && fs.existsSync(indexFile)) { app.use(express.static(frontendDir)); logger.debug({ frontendDir, indexFile }, 'Mounted static frontend'); - // SPA fallback - serve index.html for all non-API routes - app.get('/{*splat}', (req, res, next) => { + // SPA fallback - serve index.html for all non-API routes. + // + // `{*splat}` and not `*`: path-to-regexp 8, which Express 5 uses, refuses a + // bare wildcard outright — and it refuses it while the route is being + // registered, so the server does not start at all. The braces are what + // keep `/` itself matching, which is the address the application is + // usually opened at. + app.get('{*splat}', (req, res, next) => { // Skip API routes and static asset routes if (req.path.startsWith('/api') || req.path.startsWith('/static/')) { return next(); diff --git a/backend/tests/README.md b/backend/tests/README.md index 712e97fd6..29b5b60d8 100644 --- a/backend/tests/README.md +++ b/backend/tests/README.md @@ -89,15 +89,16 @@ describe('GET /api/items', () => { ### `setupTestEnv(options)` Creates an isolated test environment with: + - Temporary directories for config, cache, and volume - Environment variable overrides - Module cache management for fresh requires ```javascript const envContext = await setupTestEnv({ - tag: 'my-test-', // Prefix for temp directory + tag: 'my-test-', // Prefix for temp directory modules: ['src/services/db'], // Modules to clear from cache - env: { MY_VAR: 'value' }, // Additional env vars + env: { MY_VAR: 'value' }, // Additional env vars }); // Use envContext.requireFresh() for clean module imports @@ -148,5 +149,6 @@ npm run test:coverage ``` Coverage reports are generated in: + - `coverage/` - HTML report (open `coverage/index.html`) - Terminal output with summary diff --git a/backend/tests/config/archive-bounds.test.js b/backend/tests/config/archive-bounds.test.js new file mode 100644 index 000000000..bbc37163a --- /dev/null +++ b/backend/tests/config/archive-bounds.test.js @@ -0,0 +1,133 @@ +import { describe, it, expect, afterEach } from 'vitest'; + +import { modulePath } from '../helpers/env-test-utils.js'; + +/** + * The two bounds on browsing inside an archive. + * + * `archiveCacheService` reads both of them, and nothing defined either. In + * JavaScript that is not an error, it is `undefined`, and every comparison + * against `undefined` is false — so the guards were the wrong way round in two + * different directions at once: + * + * - `innerSize > browseMaxBytes` was never true, so no archive was ever too + * large to look inside. A .tar.gz of any size was decompressed whole into + * the cache so its listing could be shown, which is the one thing that + * guard exists to prevent. + * - `total <= cacheMaxBytes` was never true either, so the sweep never + * returned early and never stopped: it removed every cached copy it found, + * on every pass, and each archive was decompressed again from scratch the + * next time somebody opened it. + * + * Asserted against the configuration rather than the service, because that is + * where the numbers were missing, and a service test would have passed just as + * happily against `undefined`. + */ + +const load = () => { + delete require.cache[require.resolve(modulePath('src/config/index.js'))]; + delete require.cache[require.resolve(modulePath('src/config/env.js'))]; + return require(modulePath('src/config/index.js')); +}; + +const withEnv = (values, run) => { + const previous = {}; + for (const [key, value] of Object.entries(values)) { + previous[key] = process.env[key]; + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + try { + return run(); + } finally { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } +}; + +afterEach(() => load()); + +describe('the bounds on browsing inside an archive', () => { + it('gives both of them a number, so every comparison against them means something', () => { + const { archives } = withEnv( + { MAX_BROWSABLE_ARCHIVE_SIZE: undefined, ARCHIVE_CACHE_MAX_SIZE: undefined }, + load + ); + + expect(Number.isFinite(archives.browseMaxBytes)).toBe(true); + expect(Number.isFinite(archives.cacheMaxBytes)).toBe(true); + expect(archives.browseMaxBytes).toBeGreaterThan(0); + expect(archives.cacheMaxBytes).toBeGreaterThan(0); + }); + + /** + * The comparisons as `archiveCacheService` writes them. Undefined passes the + * first assertion above in no version of this — but it also has to be said + * the way the code says it, because that is where it went wrong. + */ + it('refuses an archive larger than the bound, and admits one under it', () => { + const { archives } = load(); + + expect(archives.browseMaxBytes + 1 > archives.browseMaxBytes).toBe(true); + expect(1 > archives.browseMaxBytes).toBe(false); + }); + + it('lets a cache under the bound alone, and sweeps one over it', () => { + const { archives } = load(); + + expect(1 <= archives.cacheMaxBytes).toBe(true); + expect(archives.cacheMaxBytes + 1 <= archives.cacheMaxBytes).toBe(false); + }); + + it('takes the size an administrator sets', () => { + const { archives } = withEnv( + { MAX_BROWSABLE_ARCHIVE_SIZE: '512MB', ARCHIVE_CACHE_MAX_SIZE: '1GB' }, + load + ); + + expect(archives.browseMaxBytes).toBe(512 * 1024 * 1024); + expect(archives.cacheMaxBytes).toBe(1024 * 1024 * 1024); + }); + + it('falls back rather than failing the start when the value is not a size', () => { + const { archives } = withEnv( + { MAX_BROWSABLE_ARCHIVE_SIZE: 'as much as it takes', ARCHIVE_CACHE_MAX_SIZE: '-1' }, + load + ); + + expect(archives.browseMaxBytes).toBe(2 * 1024 * 1024 * 1024); + expect(archives.cacheMaxBytes).toBe(8 * 1024 * 1024 * 1024); + }); +}); + +/** + * Every size an administrator can set, and the capital B. + * + * `5MB` is how the README writes it and how everyone writes it, and the parser + * rejected it for the capital B alone — returning null, which every setting + * reads as "not set" and answers with its default. Ten settings took their + * default from a value that had been given: the upload chunk size, the search + * and editor ceilings, the JSON body limit, the direct-upload limit, the + * storage reserve, both archive bounds, the trash quota and the preview ceiling. + */ +describe('a size, however it is written', () => { + const sizes = () => require(modulePath('src/utils/env.js')).parseByteSize; + + it.each([ + ['512M', 512 * 1024 * 1024], + ['512MB', 512 * 1024 * 1024], + ['512mb', 512 * 1024 * 1024], + ['512 MB', 512 * 1024 * 1024], + ['2GB', 2 * 1024 * 1024 * 1024], + ['1k', 1024], + ['4096', 4096], + ])('reads %s', (written, bytes) => { + expect(sizes()(written)).toBe(bytes); + }); + + it.each(['512Mo', 'as much as it takes', '', '5 5MB'])('refuses %s', (written) => { + expect(sizes()(written)).toBe(null); + }); +}); diff --git a/backend/tests/config/editor-body-limit.test.js b/backend/tests/config/editor-body-limit.test.js new file mode 100644 index 000000000..35709d076 --- /dev/null +++ b/backend/tests/config/editor-body-limit.test.js @@ -0,0 +1,88 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { clearModuleCache, overrideEnv } from '../helpers/env-test-utils.js'; + +/** + * The editor sends a file back through a JSON body when it saves. A body limit + * under the size the editor opens therefore makes a file that opens and cannot + * be saved — answered with "request entity too large", which names neither of + * the two settings involved. + * + * These are the guarantee that the pair cannot be left in that state, and that + * it is always the editor that gives way where someone has set a body ceiling + * on purpose. + */ +const requireFreshConfig = () => { + clearModuleCache('src/config/env'); + clearModuleCache('src/config/index'); + return require('../../src/config/index'); +}; + +const MB = 1024 * 1024; + +describe('the body limit and what the editor opens', () => { + let restoreEnv; + + afterEach(() => { + if (restoreEnv) { + restoreEnv(); + restoreEnv = null; + } + }); + + it('leaves both defaults alone — they already clear each other', () => { + restoreEnv = overrideEnv({ EDITOR_MAX_FILESIZE: undefined, MAX_JSON_BODY_SIZE: undefined }); + + const config = requireFreshConfig(); + + expect(config.editor.maxFileSizeBytes).toBe(2 * MB); + expect(config.uploads.maxJsonBodyBytes).toBe(8 * MB); + }); + + // The configuration our own FAQ recommends for editing large documents. + it('raises the body limit when the editor is told to open more', () => { + restoreEnv = overrideEnv({ EDITOR_MAX_FILESIZE: '10M', MAX_JSON_BODY_SIZE: undefined }); + + const config = requireFreshConfig(); + + expect(config.editor.maxFileSizeBytes).toBe(10 * MB); + expect(config.uploads.maxJsonBodyBytes).toBeGreaterThan(2 * config.editor.maxFileSizeBytes); + }); + + // Escaping can double the text, and the path travels in the same body. + it('keeps room for a file that escapes badly', () => { + restoreEnv = overrideEnv({ EDITOR_MAX_FILESIZE: '4M', MAX_JSON_BODY_SIZE: undefined }); + + const config = requireFreshConfig(); + + const worstCase = JSON.stringify({ + path: 'Documents/notes.md', + content: '"\n'.repeat((4 * MB) / 2), + }); + expect(Buffer.byteLength(worstCase, 'utf-8')).toBeLessThanOrEqual( + config.uploads.maxJsonBodyBytes + ); + }); + + // A ceiling someone set is a guard, not a detail to be talked out of. The + // editor is what gives way — by refusing to open what it could not save. + it('honours a body ceiling that was chosen, and lowers the editor to fit', () => { + restoreEnv = overrideEnv({ EDITOR_MAX_FILESIZE: '2M', MAX_JSON_BODY_SIZE: '1M' }); + + const config = requireFreshConfig(); + + expect(config.uploads.maxJsonBodyBytes).toBe(1 * MB); + expect(config.editor.maxFileSizeBytes).toBeLessThan(1 * MB); + // Whatever it opens, it can send back. + expect(config.editor.maxFileSizeBytes * 2).toBeLessThanOrEqual(config.uploads.maxJsonBodyBytes); + }); + + // A body limit raised for its original reason — thousands of paths in one + // delete — is not pulled back down to the editor's floor. + it('keeps a larger limit that was asked for', () => { + restoreEnv = overrideEnv({ EDITOR_MAX_FILESIZE: '2M', MAX_JSON_BODY_SIZE: '64M' }); + + const config = requireFreshConfig(); + + expect(config.uploads.maxJsonBodyBytes).toBe(64 * MB); + }); +}); diff --git a/backend/tests/config/editor-config.test.js b/backend/tests/config/editor-config.test.js index 0c98d0200..bd7d7a706 100644 --- a/backend/tests/config/editor-config.test.js +++ b/backend/tests/config/editor-config.test.js @@ -4,7 +4,7 @@ import { clearModuleCache, overrideEnv } from '../helpers/env-test-utils.js'; const requireFreshConfig = () => { clearModuleCache('src/config/env'); clearModuleCache('src/config/index'); - // eslint-disable-next-line global-require + return require('../../src/config/index'); }; @@ -46,4 +46,3 @@ describe('Editor config', () => { expect(config.editor.extensions).toEqual(['toml', 'proto', 'graphql']); }); }); - diff --git a/backend/tests/config/hidden-files-config.test.js b/backend/tests/config/hidden-files-config.test.js index 9bcb0dd04..c2ca38ab6 100644 --- a/backend/tests/config/hidden-files-config.test.js +++ b/backend/tests/config/hidden-files-config.test.js @@ -4,7 +4,7 @@ import { clearModuleCache, overrideEnv } from '../helpers/env-test-utils.js'; const requireFreshConfig = () => { clearModuleCache('src/config/env'); clearModuleCache('src/config/index'); - // eslint-disable-next-line global-require + return require('../../src/config/index'); }; @@ -24,8 +24,13 @@ describe('Hidden files config', () => { }); const config = requireFreshConfig(); - expect(config.hiddenFiles.patterns).toEqual(['.']); + // The dot, and the two suffixes the application's own in-flight files + // carry: a download being fetched and an upload being written. They are + // hidden for the same reason a dot-file is — nobody put them there and + // nobody should open them — and they disappear when the operation ends. + expect(config.hiddenFiles.patterns).toEqual(['.', 'regex:\\.download$', 'regex:\\.uploading$']); expect(config.hiddenFiles.isHiddenName('.env')).toBe(true); + expect(config.hiddenFiles.isHiddenName('holiday.mp4.download')).toBe(true); expect(config.hiddenFiles.isHiddenName('visible.txt')).toBe(false); }); diff --git a/backend/tests/config/secret-files.test.js b/backend/tests/config/secret-files.test.js new file mode 100644 index 000000000..b662731a0 --- /dev/null +++ b/backend/tests/config/secret-files.test.js @@ -0,0 +1,155 @@ +import { describe, it, expect, afterEach, beforeAll, afterAll } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { clearModuleCache, overrideEnv } from '../helpers/env-test-utils.js'; + +/** + * Secrets are readable from a file, not only from the environment. + * + * A secret passed as an environment variable is printed back by `docker inspect` + * and kept in the container's stored configuration, which is why orchestrators + * mount secrets as files instead. Every credential the server takes therefore + * accepts a companion `_FILE` variable naming the file to read. + * + * The failure that makes this worth pinning is quiet: a file written with + * `echo secret > file` ends in a newline, and a secret carrying an invisible + * trailing newline signs tokens the Document Server then rejects, with nothing + * in either log to say why. + */ + +const requireFreshConfig = () => { + clearModuleCache('src/utils/env'); + clearModuleCache('src/config/env'); + clearModuleCache('src/config/index'); + return require('../../src/config/index'); +}; + +const requireFreshEnv = () => { + clearModuleCache('src/utils/env'); + clearModuleCache('src/config/env'); + return require('../../src/config/env'); +}; + +describe('Secrets from files', () => { + let dir; + let restoreEnv; + + beforeAll(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nextexplorer-secret-files-')); + }); + + afterAll(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + afterEach(() => { + if (restoreEnv) { + restoreEnv(); + restoreEnv = null; + } + }); + + /** Write a secret file and return its path. */ + const secretFile = (name, contents) => { + const file = path.join(dir, name); + fs.writeFileSync(file, contents); + return file; + }; + + it('reads a secret from the file the _FILE variable names', () => { + restoreEnv = overrideEnv({ + ONLYOFFICE_SECRET: undefined, + ONLYOFFICE_SECRET_FILE: secretFile('onlyoffice', 'from-the-file'), + }); + + expect(requireFreshEnv().ONLYOFFICE_SECRET).toBe('from-the-file'); + }); + + it('drops the trailing newline a shell redirection leaves behind', () => { + // `echo secret > file` is how these files get written, and the newline + // would otherwise travel into every signature the secret produces. + restoreEnv = overrideEnv({ + ONLYOFFICE_SECRET: undefined, + ONLYOFFICE_SECRET_FILE: secretFile('onlyoffice-newline', 'from-the-file\n'), + }); + + expect(requireFreshEnv().ONLYOFFICE_SECRET).toBe('from-the-file'); + }); + + it('prefers the variable when both are set', () => { + restoreEnv = overrideEnv({ + ONLYOFFICE_SECRET: 'from-the-environment', + ONLYOFFICE_SECRET_FILE: secretFile('onlyoffice-ignored', 'from-the-file'), + }); + + expect(requireFreshEnv().ONLYOFFICE_SECRET).toBe('from-the-environment'); + }); + + it('covers every credential the server takes', () => { + restoreEnv = overrideEnv({ + SESSION_SECRET: undefined, + AUTH_SESSION_SECRET: undefined, + AUTH_ADMIN_PASSWORD: undefined, + ADMIN_PASSWORD: undefined, + OIDC_CLIENT_SECRET: undefined, + ONLYOFFICE_SECRET: undefined, + COLLABORA_SECRET: undefined, + SESSION_SECRET_FILE: secretFile('session', 'session-value'), + AUTH_ADMIN_PASSWORD_FILE: secretFile('admin', 'admin-value'), + OIDC_CLIENT_SECRET_FILE: secretFile('oidc', 'oidc-value'), + ONLYOFFICE_SECRET_FILE: secretFile('oo', 'onlyoffice-value'), + COLLABORA_SECRET_FILE: secretFile('collabora', 'collabora-value'), + }); + + const env = requireFreshEnv(); + expect(env.SESSION_SECRET).toBe('session-value'); + expect(env.AUTH_ADMIN_PASSWORD).toBe('admin-value'); + expect(env.OIDC_CLIENT_SECRET).toBe('oidc-value'); + expect(env.ONLYOFFICE_SECRET).toBe('onlyoffice-value'); + expect(env.COLLABORA_SECRET).toBe('collabora-value'); + }); + + it('answers on a legacy alias when the current name says nothing', () => { + restoreEnv = overrideEnv({ + SESSION_SECRET: undefined, + SESSION_SECRET_FILE: undefined, + AUTH_SESSION_SECRET: undefined, + AUTH_SESSION_SECRET_FILE: secretFile('legacy-session', 'legacy-value'), + }); + + expect(requireFreshEnv().SESSION_SECRET).toBe('legacy-value'); + }); + + it('refuses to start when the file cannot be read', () => { + // Resolving to null instead would start the server with document signing + // silently disabled — the operator named that file for a reason. + restoreEnv = overrideEnv({ + ONLYOFFICE_SECRET: undefined, + ONLYOFFICE_SECRET_FILE: path.join(dir, 'does-not-exist'), + }); + + expect(() => requireFreshEnv()).toThrow(/ONLYOFFICE_SECRET_FILE/); + }); + + it('refuses to start when the file is empty', () => { + restoreEnv = overrideEnv({ + ONLYOFFICE_SECRET: undefined, + ONLYOFFICE_SECRET_FILE: secretFile('blank', ' \n'), + }); + + expect(() => requireFreshEnv()).toThrow(/empty/); + }); + + it('carries the file-borne secret through to the editor configuration', () => { + // What the rest of the server actually reads is the resolved config, so the + // value has to survive the layer above env.js. + restoreEnv = overrideEnv({ + ONLYOFFICE_SECRET: undefined, + ONLYOFFICE_URL: 'https://documents.example.com', + ONLYOFFICE_SECRET_FILE: secretFile('onlyoffice-config', 'signing-key'), + }); + + expect(requireFreshConfig().onlyoffice.secret).toBe('signing-key'); + }); +}); diff --git a/backend/tests/fixtures/half-red-half-blue.heic b/backend/tests/fixtures/half-red-half-blue.heic new file mode 100644 index 000000000..7e50ee5df Binary files /dev/null and b/backend/tests/fixtures/half-red-half-blue.heic differ diff --git a/backend/tests/helpers/env-test-utils.js b/backend/tests/helpers/env-test-utils.js index 7c7421ff6..44fdb027a 100644 --- a/backend/tests/helpers/env-test-utils.js +++ b/backend/tests/helpers/env-test-utils.js @@ -165,6 +165,15 @@ const quiesceLoadedServices = async () => { /* nothing queued is nothing to drain */ } + // The RAW previews keep a cleanup timer of their own, and an extraction in + // progress writes into the same cache directory. + const rawPreviews = loadedModule('src/services/rawPreviewService'); + try { + await rawPreviews?.stopRawPreviewWork?.(); + } catch { + /* nothing extracting and nothing scheduled */ + } + // The trash maintenance schedules a pass shortly after a deletion; one landing // after the database is closed would fail on the next test's time. const trashMaintenance = loadedModule('src/services/trash/maintenance'); @@ -175,6 +184,31 @@ const quiesceLoadedServices = async () => { /* nothing scheduled is nothing to stop */ } + const indexDb = loadedModule('src/services/indexDb'); + try { + indexDb?.closeIndexDb?.(); + } catch { + /* an unopened index has no handle to close */ + } + + // The chunked-upload cache sweep runs on a timer once started, and recreates + // the cache directory it inspects. + const tusUploads = loadedModule('src/services/tusUploadService'); + try { + await tusUploads?.stopCacheSweep?.(); + } catch { + /* a sweep that never started has nothing to stop */ + } + + // Changing a password opens sessions.db to end the account's sessions, and + // the store keeps its handle and a daily cleanup timer until closed. + const sessionStore = loadedModule('src/utils/sessionStore'); + try { + sessionStore?.localStore?.close?.(); + } catch { + /* already closed by the test */ + } + const db = loadedModule('src/services/db'); try { db?.closeDb?.(); diff --git a/backend/tests/helpers/fake-seven-zip.js b/backend/tests/helpers/fake-seven-zip.js new file mode 100644 index 000000000..4c8762946 --- /dev/null +++ b/backend/tests/helpers/fake-seven-zip.js @@ -0,0 +1,171 @@ +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +/** + * A stand-in for 7-Zip, so what this application does around it can be run. + * + * Browsing an archive is two things: what 7-Zip says, and what is made of it. + * The second is all of the interesting part — a name that climbs out of the + * archive, a level cut out of a full listing, an archive that refuses to open + * — and on a machine without 7-Zip installed none of it could be run at all, + * which is most machines somebody develops on. + * + * The stand-in answers `i` with a format list, `l` by printing the listing a + * test wrote into the archive file, and `x -so` by printing the bytes that + * test filed under the name being asked for. So the route is decided by this + * application rather than by an archive somebody had to commit. + * + * It refuses an `x` that does not carry `-spd`, which is the switch that stops + * 7-Zip reading a name as a pattern. Nothing else would notice its absence + * until an archive held a file called `report*.txt`. + * + * That is the limit of it, and it is why the suites that use a real 7-Zip stay: + * this proves what is done with a listing, never that 7-Zip prints one. + */ + +const SCRIPT = `#!/bin/sh +# Every call, for a test that needs to know how much work was asked of 7-Zip +# rather than only what came back. +if [ -n "$FAKE_7Z_LOG" ]; then + echo "$*" >> "$FAKE_7Z_LOG" +fi +case "$1" in + i) + echo "7-Zip (z) 26.03 (x64) : Copyright (c) 1999-2026 Igor Pavlov" + echo "Formats:" + echo " 7z zip tar gz bz2 xz rar iso cab wim" + exit 0 + ;; + l) + # The last argument is the archive; everything before it is switches and -- + for last; do :; done + if [ ! -f "$last" ]; then + echo "ERROR: $last : The system cannot find the file specified." >&2 + exit 2 + fi + if head -n 1 "$last" | grep -q '^FAKE-7Z-ENCRYPTED$'; then + echo "ERROR: Can not open encrypted archive. Wrong password?" >&2 + exit 2 + fi + if head -n 1 "$last" | grep -q '^FAKE-7Z-BROKEN$'; then + echo "ERROR: Unexpected end of archive" >&2 + exit 2 + fi + sed '/^%%FAKE-7Z-CONTENT%%$/,$d' "$last" + exit 0 + ;; + x) + # Everything into a directory: how a solid archive is put in the cache. + out="" + for arg; do case "$arg" in -o*) out="\${arg#-o}";; esac; done + if [ -n "$out" ]; then + for last; do :; done + mkdir -p "$out" + tab=$(printf '\\t') + sed -n '/^%%FAKE-7Z-CONTENT%%$/,$p' "$last" | tail -n +2 | while IFS="$tab" read -r name body; do + [ -n "$name" ] || continue + mkdir -p "$out/$(dirname "$name")" + printf '%s' "$body" > "$out/$name" + done + exit 0 + fi + # The last two arguments are the archive and the entry inside it — unless + # there is no entry, which is how one layer of a compound archive is peeled: + # then the archive is last, and what it decompresses to is the file a test + # filed beside it. + for entry; do archive="$previous"; previous="$entry"; done + if [ "$archive" = "--" ]; then + cat "$entry.inner" + exit $? + fi + # Only an extraction that names an entry needs the switch that stops 7-Zip + # reading that name as a pattern; peeling a whole archive names nothing. + case " $* " in + *" -spd "*) ;; + *) echo "fake 7z: x of one entry without -spd would read its name as a pattern" >&2; exit 1 ;; + esac + awk -v want="$entry" ' + seen && index($0, want "\t") == 1 { printf "%s", substr($0, length(want) + 2); found = 1 } + /^%%FAKE-7Z-CONTENT%%$/ { seen = 1 } + END { exit(found ? 0 : 2) } + ' "$archive" || { echo "ERROR: No files to process" >&2; exit 2; } + exit 0 + ;; +esac +echo "fake 7z: unsupported command $1" >&2 +exit 1 +`; + +/** + * Put the stand-in first on PATH for the duration of a test. + * + * @returns {() => void} restores PATH and removes the stand-in + */ +const useFakeSevenZip = () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fake-7z-')); + const binary = path.join(dir, '7z'); + fs.writeFileSync(binary, SCRIPT, { mode: 0o755 }); + + const previousPath = process.env.PATH; + const previousBin = process.env.SEVEN_ZIP_PATH; + process.env.PATH = `${dir}${path.delimiter}${previousPath || ''}`; + // The services read this once, at load: a test that set it for another + // reason would otherwise reach a 7-Zip that is not this one. + delete process.env.SEVEN_ZIP_PATH; + + return () => { + process.env.PATH = previousPath; + if (previousBin === undefined) delete process.env.SEVEN_ZIP_PATH; + else process.env.SEVEN_ZIP_PATH = previousBin; + fs.rmSync(dir, { recursive: true, force: true }); + }; +}; + +/** + * An archive the stand-in can answer for: the listing 7-Zip would print, and + * then the bytes of each entry that has any, behind a marker `l` never reads. + */ +const fakeListing = (entries, { solid = false } = {}) => + [ + '7-Zip (z) 26.03 (x64) : Copyright (c) 1999-2026 Igor Pavlov', + '', + 'Listing archive: pack.zip', + '', + '--', + 'Path = pack.zip', + `Type = ${solid ? '7z' : 'zip'}`, + // What 7-Zip prints for an archive whose files share one compressed + // stream, and the whole reason a cached tree exists. + ...(solid ? ['Solid = +', 'Blocks = 1'] : []), + '', + '----------', + ...entries.map((entry) => + [ + `Path = ${entry.path}`, + `Size = ${entry.size ?? 0}`, + `Modified = ${entry.modified ?? '2026-09-16 11:22:33'}`, + `Attributes = ${entry.directory ? 'D_ drwxr-xr-x' : 'A_ -rw-r--r--'}`, + `Encrypted = ${entry.encrypted ? '+' : '-'}`, + '', + ].join('\n') + ), + '%%FAKE-7Z-CONTENT%%', + ...entries + .filter((entry) => typeof entry.content === 'string') + .map((entry) => `${entry.path}\t${entry.content}`), + '', + ].join('\n'); + +/** + * A compound archive: one that decompresses to another archive rather than to + * a file, which is what a .tar.gz is. The stand-in answers `l` with the single + * entry a real 7-Zip reports, and a whole-archive `x` with the inner listing — + * so what the test writes as `inner` is what the cache ends up holding. + */ +const fakeCompound = ({ innerName = 'backup.tar', innerSize = 4096, entries = [] } = {}) => ({ + outer: fakeListing([{ path: innerName, size: innerSize }]), + inner: fakeListing(entries), +}); + +module.exports = { useFakeSevenZip, fakeListing, fakeCompound }; diff --git a/backend/tests/helpers/legacy-app-db.js b/backend/tests/helpers/legacy-app-db.js new file mode 100644 index 000000000..4c0f8a698 --- /dev/null +++ b/backend/tests/helpers/legacy-app-db.js @@ -0,0 +1,466 @@ +/** + * The application database as earlier releases left it. + * + * An upgrade test is only as good as the database it upgrades. These schemas + * are not reconstructed from the current migrations — those create today's + * tables, which is exactly what an older installation does not have — but from + * what each tagged release creates on a clean start, table by table and column + * by column. Where two releases differ only by a few columns, the difference is + * spelled out below rather than hidden in a copy. + * + * The shapes that matter, and why: + * + * - schema 2 (1.1.x): one `users` table carrying the sign-in method itself; + * schema 3 split it into accounts and sign-in methods. + * - schema 3 (1.1.8 – 1.2.0): favorites still lived in app-config.json. + * - schema 6 (2.0.3 – 2.1.1): settings still lived in app-config.json, and a + * share counted its visits in `download_count`. + * - schema 8 (2.1.2a – 2.2.7, the last upstream release): no per-operation + * share permissions, no audit columns, none of the later feature tables. + * - schema 13 (3.0.0): share permissions but no `allow_download` column, which + * arrived after schema 10 had already been recorded. + * - schema 14 (3.0.1 – 3.1.1): folder preferences as rows, no personal folder + * names. + * - schema 16 (a build between 3.1.2 and 3.2.0): the first search index, whose + * documents had no `dir` column, and still no `allow_download`. + * - schema 17 (3.2.0 – 3.5.0, the last release): everything but the trash and + * the file versions. + * - schema 18 (integration builds before the release): the trash as it first + * shipped, before a restore recorded which entry of a folder it was taking. + */ +const fs = require('node:fs'); +const path = require('node:path'); +const Database = require('better-sqlite3'); + +const META = ` + CREATE TABLE meta ( + key TEXT PRIMARY KEY, + value TEXT + ); +`; + +// Schema 1 and 2: the sign-in method is a column of the account. +const ACCOUNTS_V2 = ` + CREATE TABLE users ( + id TEXT PRIMARY KEY, + provider TEXT NOT NULL CHECK(provider IN ('local','oidc')), + username TEXT UNIQUE, + password_hash TEXT, + password_algo TEXT, + oidc_issuer TEXT, + oidc_sub TEXT, + display_name TEXT, + email TEXT, + roles TEXT DEFAULT '[]', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE UNIQUE INDEX idx_users_oidc ON users(oidc_issuer, oidc_sub); + CREATE INDEX idx_users_username ON users(username); + CREATE TABLE auth_locks ( + key TEXT PRIMARY KEY, -- normalized username or subject key + failed_count INTEGER NOT NULL DEFAULT 0, + locked_until TEXT + ); +`; + +// Schema 3 onwards: accounts and the ways to sign in to them. +const accountsV3 = ({ personalFolder = false } = {}) => ` + CREATE TABLE auth_locks ( + key TEXT PRIMARY KEY, -- normalized username or subject key + failed_count INTEGER NOT NULL DEFAULT 0, + locked_until TEXT + ); + CREATE TABLE users ( + id TEXT PRIMARY KEY, + email TEXT UNIQUE NOT NULL, + email_verified INTEGER DEFAULT 0, + username TEXT, + display_name TEXT, + roles TEXT DEFAULT '[]', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL${personalFolder ? ',\n personal_folder_name TEXT' : ''} + ); + CREATE TABLE auth_methods ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + method_type TEXT NOT NULL CHECK(method_type IN ('local_password', 'oidc')), + password_hash TEXT, + password_algo TEXT DEFAULT 'bcrypt', + provider_issuer TEXT, + provider_sub TEXT, + provider_name TEXT, + enabled INTEGER DEFAULT 1, + last_used_at TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ); + CREATE INDEX idx_users_email ON users(email); + CREATE INDEX idx_auth_methods_user ON auth_methods(user_id); + CREATE UNIQUE INDEX idx_auth_methods_oidc ON auth_methods(provider_issuer, provider_sub) WHERE method_type = 'oidc'; + CREATE INDEX idx_auth_methods_type ON auth_methods(method_type); + ${personalFolder ? 'CREATE UNIQUE INDEX idx_users_personal_folder ON users(personal_folder_name);' : ''} +`; + +const FAVORITES = ` + CREATE TABLE favorites ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + path TEXT NOT NULL, + label TEXT, + icon TEXT DEFAULT 'outline:StarIcon', + color TEXT DEFAULT NULL, + position INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ); + CREATE UNIQUE INDEX idx_favorites_user_path ON favorites(user_id, path); + CREATE INDEX idx_favorites_user ON favorites(user_id); +`; + +/** + * The shares table of each era. `operations` are the four per-operation + * permissions and the audit columns that shipped with 3.0.0; `download` is the + * `allow_download` column that followed them. + */ +const shares = ({ operations = false, download = false } = {}) => ` + CREATE TABLE shares ( + id TEXT PRIMARY KEY, + share_token TEXT UNIQUE NOT NULL, + owner_id TEXT NOT NULL, + source_space TEXT NOT NULL, + source_path TEXT NOT NULL, + is_directory INTEGER NOT NULL, + access_mode TEXT NOT NULL CHECK(access_mode IN ('readonly', 'readwrite')), + ${ + operations + ? `allow_delete INTEGER NOT NULL DEFAULT 1, + allow_create_folder INTEGER NOT NULL DEFAULT 1, + allow_create_file INTEGER NOT NULL DEFAULT 1, + allow_upload INTEGER NOT NULL DEFAULT 1,` + : '' + } + ${download ? 'allow_download INTEGER NOT NULL DEFAULT 1,' : ''} + sharing_type TEXT NOT NULL CHECK(sharing_type IN ('anyone', 'users')), + password_hash TEXT, + expires_at TEXT, + label TEXT, + ${operations ? 'access_count INTEGER DEFAULT 0,' : ''} + download_count INTEGER DEFAULT 0, + last_accessed_at TEXT, + ${ + operations + ? `last_access_ip TEXT, + last_downloaded_at TEXT, + last_download_ip TEXT,` + : '' + } + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE + ); + CREATE TABLE share_permissions ( + id TEXT PRIMARY KEY, + share_id TEXT NOT NULL, + user_id TEXT NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY (share_id) REFERENCES shares(id) ON DELETE CASCADE, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE(share_id, user_id) + ); + CREATE TABLE guest_sessions ( + id TEXT PRIMARY KEY, + share_id TEXT NOT NULL, + ip_address TEXT, + user_agent TEXT, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + last_activity_at TEXT NOT NULL, + FOREIGN KEY (share_id) REFERENCES shares(id) ON DELETE CASCADE + ); + CREATE INDEX idx_shares_owner ON shares(owner_id); + CREATE INDEX idx_shares_token ON shares(share_token); + CREATE INDEX idx_shares_expires ON shares(expires_at); + CREATE INDEX idx_shares_source ON shares(source_space, source_path); + CREATE INDEX idx_share_permissions_share ON share_permissions(share_id); + CREATE INDEX idx_share_permissions_user ON share_permissions(user_id); + CREATE INDEX idx_guest_sessions_share ON guest_sessions(share_id); + CREATE INDEX idx_guest_sessions_expires ON guest_sessions(expires_at); +`; + +const USER_VOLUMES = ` + CREATE TABLE user_volumes ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + label TEXT NOT NULL, + path TEXT NOT NULL, + access_mode TEXT NOT NULL CHECK(access_mode IN ('readonly', 'readwrite')), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ); + CREATE INDEX idx_user_volumes_user ON user_volumes(user_id); + CREATE UNIQUE INDEX idx_user_volumes_user_path ON user_volumes(user_id, path); +`; + +const SETTINGS = ` + CREATE TABLE system_settings ( + id TEXT PRIMARY KEY, + category TEXT NOT NULL CHECK(category IN ('branding', 'system')), + key TEXT NOT NULL, + value TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE(category, key) + ); + CREATE TABLE user_settings ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + UNIQUE(user_id, key) + ); + CREATE INDEX idx_system_settings_category ON system_settings(category); + CREATE INDEX idx_user_settings_user ON user_settings(user_id); +`; + +// Schemas 9 to 13, as 3.0.0 created them. +const FEATURES_V13 = ` + CREATE TABLE folder_size_index ( + path_hash TEXT PRIMARY KEY, + parent_hash TEXT, + volume TEXT NOT NULL, + relative_path TEXT NOT NULL, + size_bytes INTEGER NOT NULL DEFAULT 0, + entry_count INTEGER NOT NULL DEFAULT 0, + last_delta_at DATETIME, + last_full_scan_at DATETIME, + dirty INTEGER NOT NULL DEFAULT 0 + ); + CREATE INDEX idx_folder_size_parent ON folder_size_index(parent_hash); + CREATE INDEX idx_folder_size_volume ON folder_size_index(volume); + CREATE TABLE onlyoffice_document_keys ( + relative_path TEXT PRIMARY KEY, + document_key TEXT NOT NULL, + signature TEXT NOT NULL, + created_at DATETIME, + expires_at DATETIME + ); + CREATE TABLE onlyoffice_editor_sessions ( + id TEXT PRIMARY KEY, + document_key TEXT NOT NULL, + relative_path TEXT NOT NULL, + absolute_path TEXT NOT NULL, + user_id TEXT, + guest_session_id TEXT, + expires_at DATETIME NOT NULL + ); + CREATE INDEX idx_onlyoffice_sessions_expiry ON onlyoffice_editor_sessions(expires_at); + CREATE TABLE recent_destinations ( + user_id TEXT NOT NULL, + path TEXT NOT NULL, + used_at DATETIME NOT NULL, + PRIMARY KEY (user_id, path) + ); +`; + +const FOLDER_PREFERENCES = ` + CREATE TABLE folder_preferences ( + user_id TEXT NOT NULL, + path TEXT NOT NULL, + sort_by TEXT, + sort_order TEXT, + view_mode TEXT, + updated_at DATETIME NOT NULL, + PRIMARY KEY (user_id, path) + ); + CREATE INDEX idx_folder_preferences_path ON folder_preferences(path); +`; + +// The first search index: documents known by path alone. +const SEARCH_V16 = ` + CREATE TABLE search_documents ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL UNIQUE, + mtime_ms INTEGER NOT NULL, + size INTEGER NOT NULL, + indexed_at TEXT NOT NULL + ); + CREATE INDEX idx_search_documents_path ON search_documents(path); + CREATE VIRTUAL TABLE search_terms + USING fts5(text, content='', contentless_delete=1, tokenize='unicode61 remove_diacritics 2'); +`; + +// Schema 17: the same index, by folder. +const SEARCH_V17 = ` + CREATE TABLE search_documents ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL UNIQUE, + dir TEXT NOT NULL DEFAULT '', + mtime_ms INTEGER NOT NULL, + size INTEGER NOT NULL, + indexed_at TEXT NOT NULL + ); + CREATE INDEX idx_search_documents_path ON search_documents(path); + CREATE INDEX idx_search_documents_dir ON search_documents(dir); + CREATE VIRTUAL TABLE search_terms + USING fts5(text, content='', contentless_delete=1, tokenize='unicode61 remove_diacritics 2'); +`; + +// Schema 18 as the trash first shipped on integration: no `restore_entry`, no +// `trash_shares`. +const EARLY_TRASH = ` + CREATE TABLE trash_zones ( + id TEXT PRIMARY KEY, + root TEXT NOT NULL, + created_at TEXT NOT NULL + ); + CREATE INDEX idx_trash_zones_root ON trash_zones(root); + CREATE TABLE trash_items ( + id TEXT PRIMARY KEY, + zone_id TEXT NOT NULL, + state TEXT NOT NULL, + name TEXT NOT NULL, + kind TEXT NOT NULL, + size_bytes INTEGER NOT NULL DEFAULT 0, + original_path TEXT NOT NULL, + relative_path TEXT NOT NULL, + logical_path TEXT, + space TEXT, + deleted_by TEXT, + deleted_by_label TEXT, + owner_user_id TEXT, + restore_path TEXT, + deleted_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE INDEX idx_trash_items_zone ON trash_items(zone_id, deleted_at); + CREATE INDEX idx_trash_items_deleted_by ON trash_items(deleted_by); + CREATE INDEX idx_trash_items_owner ON trash_items(owner_user_id); + CREATE TABLE trash_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + zone_id TEXT NOT NULL, + item_id TEXT, + item_name TEXT, + kind TEXT NOT NULL, + detail TEXT, + created_at TEXT NOT NULL + ); + CREATE INDEX idx_trash_events_zone ON trash_events(zone_id, id); +`; + +const SCHEMAS = { + 2: { release: '1.1.x', ddl: [META, ACCOUNTS_V2] }, + 3: { release: '1.2.0', ddl: [META, accountsV3()] }, + 6: { release: '2.1.1', ddl: [META, accountsV3(), FAVORITES, shares(), USER_VOLUMES] }, + 8: { release: '2.2.7', ddl: [META, accountsV3(), FAVORITES, shares(), USER_VOLUMES, SETTINGS] }, + 13: { + release: '3.0.0', + ddl: [ + META, + accountsV3(), + FAVORITES, + shares({ operations: true }), + USER_VOLUMES, + SETTINGS, + FEATURES_V13, + ], + }, + 14: { + release: '3.1.0', + ddl: [ + META, + accountsV3(), + FAVORITES, + shares({ operations: true }), + USER_VOLUMES, + SETTINGS, + FEATURES_V13, + FOLDER_PREFERENCES, + ], + }, + 16: { + release: 'a build between 3.1.2 and 3.2.0', + ddl: [ + META, + accountsV3({ personalFolder: true }), + FAVORITES, + shares({ operations: true }), + USER_VOLUMES, + SETTINGS, + FEATURES_V13, + FOLDER_PREFERENCES, + SEARCH_V16, + ], + }, + 17: { + release: '3.5.0', + ddl: [ + META, + accountsV3({ personalFolder: true }), + FAVORITES, + shares({ operations: true, download: true }), + USER_VOLUMES, + SETTINGS, + FEATURES_V13, + FOLDER_PREFERENCES, + SEARCH_V17, + ], + }, + 18: { + release: 'the first integration builds with the trash', + ddl: [ + META, + accountsV3({ personalFolder: true }), + FAVORITES, + shares({ operations: true, download: true }), + USER_VOLUMES, + SETTINGS, + FEATURES_V13, + FOLDER_PREFERENCES, + SEARCH_V17, + EARLY_TRASH, + ], + }, +}; + +/** + * Create `app.db` in a configuration directory as the release that recorded + * `schemaVersion` left it, and hand back an open connection so the test can put + * that installation's rows in. Close it before the application opens the file. + */ +const createLegacyDatabase = (configDir, schemaVersion) => { + const schema = SCHEMAS[schemaVersion]; + if (!schema) throw new Error(`No recorded schema for version ${schemaVersion}`); + + fs.mkdirSync(configDir, { recursive: true }); + const db = new Database(path.join(configDir, 'app.db')); + db.exec(schema.ddl.join('\n')); + db.prepare('INSERT INTO meta(key, value) VALUES (?, ?)').run( + 'schema_version', + String(schemaVersion) + ); + return db; +}; + +/** A connection of the test's own to a database, independent of the application's. */ +const openDatabaseFile = (configDir) => new Database(path.join(configDir, 'app.db')); + +/** Every table, index and trigger, with its definition, for comparing two states. */ +const describeSchema = (db) => + db + .prepare( + `SELECT type, name, tbl_name, sql FROM sqlite_master + WHERE name NOT LIKE 'sqlite_%' + ORDER BY type, name` + ) + .all(); + +module.exports = { + SCHEMAS, + createLegacyDatabase, + openDatabaseFile, + describeSchema, +}; diff --git a/backend/tests/helpers/media-tools.js b/backend/tests/helpers/media-tools.js new file mode 100644 index 000000000..6e7826906 --- /dev/null +++ b/backend/tests/helpers/media-tools.js @@ -0,0 +1,127 @@ +import { execFile } from 'node:child_process'; +import path from 'node:path'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +/** + * Which media tools this machine has, for the suites that need them. + * + * Several suites need ffmpeg, and the HEIC ones need an ffmpeg that can read + * HEIF, which it only could from 7.1. Where a tool is missing, those suites + * skip — and say so, through skipIf, so the run reports them as skipped. They + * used to return early from the test body instead, which the runner counts as + * a pass: on every CI run, whose Ubuntu ffmpeg is 6.1, the HEIC decode was + * reported green having asserted nothing. + * + * Skipping is right on a laptop that lacks a tool, and wrong where the tool + * was installed on purpose. REQUIRE_MEDIA_TOOLS lists the ones that must be + * there (`ffmpeg`, `heif`, comma-separated); a missing one then fails the + * suite at load, by name, rather than skipping it — so a CI image that stops + * carrying ffmpeg turns red instead of quietly running less. + */ + +export const HEIC_FIXTURE = path.join( + import.meta.dirname, + '..', + 'fixtures', + 'half-red-half-blue.heic' +); + +const required = new Set( + (process.env.REQUIRE_MEDIA_TOOLS || '') + .split(',') + .map((name) => name.trim()) + .filter(Boolean) +); + +const probe = async (name, check) => { + let available; + try { + available = await check(); + } catch (_) { + available = false; + } + if (!available && required.has(name)) { + throw new Error( + `REQUIRE_MEDIA_TOOLS asks for ${name}, and this machine does not have it; ` + + 'the suites that need it would otherwise skip without anyone noticing.' + ); + } + return available; +}; + +export const hasFfmpeg = () => + probe('ffmpeg', async () => { + await execFileAsync('ffmpeg', ['-version']); + return true; + }); + +/** HEIF is read when ffprobe finds the HEVC picture inside the fixture. */ +export const ffmpegReadsHeif = () => + probe('heif', async () => { + const { stdout } = await execFileAsync('ffprobe', [ + '-v', + 'error', + '-show_entries', + 'stream=codec_name', + '-of', + 'default=nw=1:nk=1', + HEIC_FIXTURE, + ]); + return stdout.trim() === 'hevc'; + }); + +/** + * 7-Zip, which is what reads an archive's table of contents. + * + * Named here rather than probed inline for the same reason as the rest: a + * machine without it skips those suites, and CI asks for it by name so that a + * skip there is a failure instead of a quiet gap. + */ +export const hasSevenZip = () => + probe('7z', async () => { + await execFileAsync(process.env.SEVEN_ZIP_PATH || '7z', ['i']); + return true; + }); + +/** + * Something that can build an ISO image, so the format can be covered without + * a binary fixture in the repository: `genisoimage` or `mkisofs` on Linux, + * `hdiutil` on a Mac. An ISO is worth covering because it is the one format in + * the offered list that is a filesystem rather than an archive, and 7-Zip is + * what makes it look like the others. + */ +export const isoBuilder = () => + probe('iso', async () => { + for (const [command, args] of [ + ['genisoimage', ['--version']], + ['mkisofs', ['-version']], + ['hdiutil', ['help']], + ]) { + try { + await execFileAsync(command, args); + return command; + } catch (_) { + // The next one, or none. + } + } + return false; + }); + +/** Write `directory` as an ISO image at `file`, with whichever builder there is. */ +export const buildIso = async (builder, directory, file) => { + if (builder === 'hdiutil') { + await execFileAsync('hdiutil', [ + 'makehybrid', + '-iso', + '-joliet', + '-o', + file, + directory, + '-quiet', + ]); + return; + } + await execFileAsync(builder, ['-quiet', '-J', '-r', '-o', file, directory]); +}; diff --git a/backend/tests/helpers/mounted-routes.js b/backend/tests/helpers/mounted-routes.js index b45d3ecd5..e0c61f3e3 100644 --- a/backend/tests/helpers/mounted-routes.js +++ b/backend/tests/helpers/mounted-routes.js @@ -32,10 +32,7 @@ const mountedRoutes = (requireFresh) => { const guards = handlers.slice(0, -1).filter(Boolean); for (const routePath of paths) { for (const method of methods) { - // A router mounted at `/` gives a prefix that would double the - // slash of every path under it. - const at = `${prefix === '/' ? '' : prefix}${routePath}`; - routes.push({ method, path: at, router: label, guards }); + routes.push({ method, path: `${prefix}${routePath}`, router: label, guards }); } } } else if (layer.handle && Array.isArray(layer.handle.stack)) { @@ -52,6 +49,8 @@ const mountedRoutes = (requireFresh) => { if (typeof prefix === 'string') walk(prefix, router, prefix); }, }); + walk('', requireFresh('src/routes/health'), '/'); + return routes; }; diff --git a/backend/tests/helpers/native-rsync.js b/backend/tests/helpers/native-rsync.js new file mode 100644 index 000000000..41d6385f7 --- /dev/null +++ b/backend/tests/helpers/native-rsync.js @@ -0,0 +1,44 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +/** + * Whether the native copy can run here: an rsync that understands + * `--info=progress2`, the flag the transfer service asks it for. + * + * The flag arrived in rsync 3.1. macOS ships openrsync, which refuses it, so a + * Mac falls back to the in-application copy and the native suites skip there — + * honestly, with `skipIf`. Yet FILE_TRANSFER_ENGINE is native on Linux, so + * rsync is what copies in nearly every deployment. CI installs rsync and sets + * `REQUIRE_NATIVE_RSYNC=1`, which turns an rsync that cannot run into a failure + * rather than a quiet skip. + */ +const probeNativeRsync = () => { + let directory = null; + try { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'nextexplorer-rsync-probe-')); + const source = path.join(directory, 'source.txt'); + const copy = path.join(directory, 'copy.txt'); + fs.writeFileSync(source, 'probe'); + const result = spawnSync('rsync', ['-a', '--info=progress2', source, copy], { + encoding: 'utf8', + timeout: 10000, + }); + return result.status === 0 && fs.readFileSync(copy, 'utf8') === 'probe'; + } catch { + return false; + } finally { + if (directory) fs.rmSync(directory, { recursive: true, force: true }); + } +}; + +const NATIVE_RSYNC = probeNativeRsync(); + +if (process.env.REQUIRE_NATIVE_RSYNC && !NATIVE_RSYNC) { + throw new Error( + 'REQUIRE_NATIVE_RSYNC is set but no rsync that understands --info=progress2 was found.' + ); +} + +module.exports = { NATIVE_RSYNC }; diff --git a/backend/tests/helpers/soft-authenticator.js b/backend/tests/helpers/soft-authenticator.js new file mode 100644 index 000000000..3f3011e34 --- /dev/null +++ b/backend/tests/helpers/soft-authenticator.js @@ -0,0 +1,219 @@ +const crypto = require('node:crypto'); + +/** + * An authenticator made of software, for the tests. + * + * It writes what a real one writes — the CBOR, the authenticator data, the + * signature over both halves — with an encoder of its own, written from the + * format rather than from the reader it is used to test. A test that encoded + * with the production code would only prove the reader agrees with the writer + * beside it; this one can disagree, which is the point. + */ + +const encode = (value) => { + if (Buffer.isBuffer(value)) return Buffer.concat([head(2, value.length), value]); + if (typeof value === 'string') { + const bytes = Buffer.from(value, 'utf8'); + return Buffer.concat([head(3, bytes.length), bytes]); + } + if (typeof value === 'number') { + if (!Number.isInteger(value)) throw new Error('This writes whole numbers only.'); + return value >= 0 ? head(0, value) : head(1, -1 - value); + } + if (value === false) return Buffer.from([0xf4]); + if (value === true) return Buffer.from([0xf5]); + if (value === null) return Buffer.from([0xf6]); + if (Array.isArray(value)) { + return Buffer.concat([head(4, value.length), ...value.map(encode)]); + } + if (value instanceof Map) { + const parts = [head(5, value.size)]; + for (const [key, item] of value) parts.push(encode(key), encode(item)); + return Buffer.concat(parts); + } + throw new Error(`Nothing here writes ${typeof value}.`); +}; + +const head = (major, argument) => { + const prefix = major << 5; + if (argument < 24) return Buffer.from([prefix | argument]); + if (argument < 0x100) return Buffer.from([prefix | 24, argument]); + if (argument < 0x10000) { + const buffer = Buffer.alloc(3); + buffer.writeUInt8(prefix | 25, 0); + buffer.writeUInt16BE(argument, 1); + return buffer; + } + const buffer = Buffer.alloc(5); + buffer.writeUInt8(prefix | 26, 0); + buffer.writeUInt32BE(argument, 1); + return buffer; +}; + +const b64u = (bytes) => Buffer.from(bytes).toString('base64url'); +const fromB64u = (value) => Buffer.from(value, 'base64url'); + +const ALGORITHMS = { + ES256: -7, + EdDSA: -8, + RS256: -257, +}; + +const generateKeyPair = (algorithm) => { + if (algorithm === ALGORITHMS.ES256) { + return crypto.generateKeyPairSync('ec', { namedCurve: 'P-256' }); + } + if (algorithm === ALGORITHMS.EdDSA) return crypto.generateKeyPairSync('ed25519'); + if (algorithm === ALGORITHMS.RS256) { + return crypto.generateKeyPairSync('rsa', { modulusLength: 2048 }); + } + throw new Error(`No key for algorithm ${algorithm}.`); +}; + +/** The public half, as a COSE key (RFC 9052). */ +const coseKey = (publicKey, algorithm) => { + const jwk = publicKey.export({ format: 'jwk' }); + if (algorithm === ALGORITHMS.ES256) { + return new Map([ + [1, 2], + [3, algorithm], + [-1, 1], + [-2, fromB64u(jwk.x)], + [-3, fromB64u(jwk.y)], + ]); + } + if (algorithm === ALGORITHMS.EdDSA) { + return new Map([ + [1, 1], + [3, algorithm], + [-1, 6], + [-2, fromB64u(jwk.x)], + ]); + } + return new Map([ + [1, 3], + [3, algorithm], + [-1, fromB64u(jwk.n)], + [-2, fromB64u(jwk.e)], + ]); +}; + +const FLAGS = { + userPresent: 0x01, + userVerified: 0x04, + backedUp: 0x10, + attestedCredential: 0x40, + extensionData: 0x80, +}; + +const authenticatorData = ({ + rpId, + flags, + signCount = 0, + aaguid = null, + credentialId = null, + publicKey = null, + extensions = null, +}) => { + const parts = [crypto.createHash('sha256').update(rpId, 'utf8').digest(), Buffer.from([flags])]; + const counter = Buffer.alloc(4); + counter.writeUInt32BE(signCount); + parts.push(counter); + + if (credentialId) { + const length = Buffer.alloc(2); + length.writeUInt16BE(credentialId.length); + parts.push(aaguid || Buffer.alloc(16), length, credentialId, encode(publicKey)); + } + if (extensions) parts.push(encode(extensions)); + return Buffer.concat(parts); +}; + +const clientData = ({ type, challenge, origin, crossOrigin = false }) => + Buffer.from(JSON.stringify({ type, challenge, origin, crossOrigin }), 'utf8'); + +/** + * Make a passkey, the way a browser would hand one over. + * + * @returns what the registration route receives, plus the key to sign with later. + */ +const createCredential = ({ + rpId = 'files.example.com', + origin = 'https://files.example.com', + challenge, + algorithm = ALGORITHMS.ES256, + userVerified = true, + signCount = 0, + format = 'none', + credentialId = crypto.randomBytes(32), + aaguid = Buffer.alloc(16), + flags = null, +} = {}) => { + const { publicKey, privateKey } = generateKeyPair(algorithm); + const key = coseKey(publicKey, algorithm); + const bits = + flags ?? FLAGS.userPresent | FLAGS.attestedCredential | (userVerified ? FLAGS.userVerified : 0); + + const authData = authenticatorData({ + rpId, + flags: bits, + signCount, + aaguid, + credentialId, + publicKey: key, + }); + + return { + credentialId, + privateKey, + algorithm, + coseKey: encode(key), + attestationObject: encode( + new Map([ + ['fmt', format], + ['attStmt', new Map()], + ['authData', authData], + ]) + ), + clientDataJSON: clientData({ type: 'webauthn.create', challenge, origin }), + }; +}; + +/** Sign in with a passkey already made. */ +const signAssertion = ({ + credential, + rpId = 'files.example.com', + origin = 'https://files.example.com', + challenge, + signCount = 1, + userVerified = true, + crossOrigin = false, + type = 'webauthn.get', + flags = null, +}) => { + const bits = flags ?? FLAGS.userPresent | (userVerified ? FLAGS.userVerified : 0); + const authData = authenticatorData({ rpId, flags: bits, signCount }); + const clientDataJSON = clientData({ type, challenge, origin, crossOrigin }); + const signedData = Buffer.concat([ + authData, + crypto.createHash('sha256').update(clientDataJSON).digest(), + ]); + const signature = + credential.algorithm === ALGORITHMS.EdDSA + ? crypto.sign(null, signedData, credential.privateKey) + : crypto.sign('sha256', signedData, credential.privateKey); + + return { authenticatorData: authData, clientDataJSON, signature }; +}; + +module.exports = { + ALGORITHMS, + FLAGS, + authenticatorData, + b64u, + clientData, + coseKey, + createCredential, + encode, + signAssertion, +}; diff --git a/backend/tests/helpers/substitute-module.js b/backend/tests/helpers/substitute-module.js new file mode 100644 index 000000000..38c6b659f --- /dev/null +++ b/backend/tests/helpers/substitute-module.js @@ -0,0 +1,36 @@ +const { createRequire } = require('node:module'); + +/** + * Stand `exports` in for what `fromFile` receives when it requires `request`, + * until the returned function is called. + * + * For the rare collaborator a test cannot otherwise bring into the state it + * needs — a write that has begun and not yet finished. The request is resolved + * from `fromFile`, exactly as the code under test resolves it, so the stand-in + * is the module that code actually gets. Require the code under test after + * this, and restore once whatever it started has been stopped. + */ +const substituteModule = (fromFile, request, exports) => { + const localRequire = createRequire(fromFile); + const resolved = localRequire.resolve(request); + const previous = localRequire.cache[resolved]; + + localRequire.cache[resolved] = { + id: resolved, + filename: resolved, + loaded: true, + exports, + children: [], + paths: [], + }; + + return () => { + if (previous) { + localRequire.cache[resolved] = previous; + } else { + delete localRequire.cache[resolved]; + } + }; +}; + +module.exports = { substituteModule }; diff --git a/backend/tests/middleware/auth-gates.test.js b/backend/tests/middleware/auth-gates.test.js new file mode 100644 index 000000000..f0fcf188a --- /dev/null +++ b/backend/tests/middleware/auth-gates.test.js @@ -0,0 +1,191 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Every gate in the middleware that decides whether a request is let through. + * + * Forty-six paths through one function, exercised only sideways by route + * suites that were testing something else. What it lets past without a session + * is the list worth being able to read: the health of the process, a feature + * flag, a branding logo on the login page, two integration callbacks that + * guard themselves with their own tokens, and a share link. + * + * Everything else is a 401, and that is the assertion this file exists for. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +/** + * The middleware and a way to ask it about one request. + * + * `next()` with no argument is "let through"; a 401 written to the response is + * "refused"; `next(error)` is a refusal the error handler will dress up. + */ +const gate = async (env = {}) => { + currentEnv = await setupTestEnv({ tag: 'auth-gates-', env }); + const authMiddleware = currentEnv.requireFresh('src/middleware/authMiddleware'); + + return async (request = {}) => { + const req = { + path: '/api/browse', + method: 'GET', + headers: {}, + cookies: {}, + ...request, + }; + + let status = null; + let body = null; + const res = { + status: (code) => { + status = code; + return res; + }, + json: (payload) => { + body = payload; + return res; + }, + }; + + let passed = false; + let failure = null; + await authMiddleware(req, res, (error) => { + if (error) failure = error; + else passed = true; + }); + + return { passed, status, body, failure, req }; + }; +}; + +describe('what goes through without any session at all', () => { + it('lets anything outside the API alone', async () => { + const ask = await gate(); + + expect((await ask({ path: '/healthz' })).passed).toBe(true); + expect((await ask({ path: '/index.html' })).passed).toBe(true); + expect((await ask({ path: '/' })).passed).toBe(true); + }); + + it('lets a preflight through, since it carries no credentials to check', async () => { + const ask = await gate(); + + expect((await ask({ path: '/api/browse', method: 'OPTIONS' })).passed).toBe(true); + }); + + it('lets the feature flags and the branding through', async () => { + const ask = await gate(); + + expect((await ask({ path: '/api/features' })).passed).toBe(true); + expect((await ask({ path: '/api/features/anything' })).passed).toBe(true); + // The login page draws itself before anyone has signed in. + expect((await ask({ path: '/api/branding' })).passed).toBe(true); + }); + + it('lets the sign-in routes through', async () => { + const ask = await gate(); + + expect((await ask({ path: '/api/auth/status' })).passed).toBe(true); + expect((await ask({ path: '/api/auth/login', method: 'POST' })).passed).toBe(true); + }); + + /** + * A share link is opened by someone with no account, which is the whole + * point of it. Browsing inside one is not on this list: that needs either an + * account or a guest session proving the password was typed. + */ + it('lets a share link through, but not browsing inside it', async () => { + const ask = await gate(); + + expect((await ask({ path: '/api/share/abc123/access' })).passed).toBe(true); + expect((await ask({ path: '/api/share/abc123/browse/Docs' })).passed).toBe(false); + }); + + it('refuses everything else', async () => { + const ask = await gate(); + + for (const path of ['/api/browse', '/api/users', '/api/settings', '/api/shares']) { + const answer = await ask({ path }); + expect(answer.passed, path).toBe(false); + expect(answer.status, path).toBe(401); + } + }); +}); + +describe('the integrations that guard themselves', () => { + /** + * An editor's server calls back with its own signed token, which the route + * checks. Those two paths are open only while the integration is configured + * — otherwise they are two unauthenticated endpoints for no reason. + */ + it('opens the ONLYOFFICE callbacks only once a server is configured', async () => { + const closed = await gate(); + expect((await closed({ path: '/api/onlyoffice/callback' })).passed).toBe(false); + expect((await closed({ path: '/api/onlyoffice/file' })).passed).toBe(false); + + const open = await gate({ ONLYOFFICE_URL: 'https://office.example.com' }); + expect((await open({ path: '/api/onlyoffice/callback' })).passed).toBe(true); + expect((await open({ path: '/api/onlyoffice/file' })).passed).toBe(true); + // Not the whole integration: only the two paths that carry a token. + expect((await open({ path: '/api/onlyoffice/users' })).passed).toBe(false); + }); + + it('opens the Collabora endpoints only once its URL and secret are both set', async () => { + const noSecret = await gate({ COLLABORA_URL: 'https://collabora.example.com' }); + expect((await noSecret({ path: '/api/collabora/wopi/files/x' })).passed).toBe(false); + + const configured = await gate({ + COLLABORA_URL: 'https://collabora.example.com', + COLLABORA_SECRET: 'a-secret', + }); + expect((await configured({ path: '/api/collabora/wopi/files/x' })).passed).toBe(true); + expect((await configured({ path: '/api/collabora/anything-else' })).passed).toBe(false); + }); +}); + +describe('when authentication is switched off', () => { + /** + * A deployment behind its own front door runs with no accounts at all. Every + * request then arrives as the same synthetic administrator, because the rest + * of the application asks who is calling and has to be told something. + */ + it('lets everything through as one anonymous administrator', async () => { + const ask = await gate({ AUTH_ENABLED: 'false' }); + + const answer = await ask({ path: '/api/users' }); + + expect(answer.passed).toBe(true); + expect(answer.req.user).toMatchObject({ id: 'anonymous', roles: ['admin'] }); + }); +}); + +describe('a guest session', () => { + it('is ignored when it names a session that does not exist', async () => { + const ask = await gate(); + + const answer = await ask({ + path: '/api/browse', + headers: { 'x-guest-session': 'not-a-real-session' }, + }); + + expect(answer.passed).toBe(false); + expect(answer.status).toBe(401); + expect(answer.req.guestSession).toBeUndefined(); + }); + + it('is looked for in the cookie as well as the header', async () => { + const ask = await gate(); + + // Both are refused because the session is not real; what is asserted is + // that neither route into the middleware crashes it. + expect((await ask({ cookies: { guestSession: 'nope' } })).status).toBe(401); + expect((await ask({ headers: { 'x-guest-session': 'nope' } })).status).toBe(401); + }); +}); diff --git a/backend/tests/middleware/guest-session-with-user.test.js b/backend/tests/middleware/guest-session-with-user.test.js new file mode 100644 index 000000000..9b1976ce4 --- /dev/null +++ b/backend/tests/middleware/guest-session-with-user.test.js @@ -0,0 +1,154 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * These mount the real auth middleware on purpose. + * + * The password check on protected shares reads req.guestSession, which is the + * only proof a signed-in visitor typed the password. An earlier build deleted + * that session as soon as a user was attached, so the check could never be + * satisfied: the visitor typed the right password and still got 401 on every + * request. Test harnesses that fake the middleware cannot see that — this one + * runs the middleware itself. + */ + +let env; +let owner; +let visitor; +let share; + +beforeAll(async () => { + env = await setupTestEnv({ + tag: 'guest-session-with-user-', + modules: [ + 'src/config/env', + 'src/config/index', + 'src/utils/pathUtils', + 'src/services/db', + 'src/services/users', + 'src/services/sharesService', + 'src/services/guestSessionService', + 'src/services/accessManager', + 'src/middleware/authMiddleware', + 'src/middleware/errorHandler', + 'src/routes/shares', + ], + }); + + const usersService = env.requireFresh('src/services/users'); + const sharesService = env.requireFresh('src/services/sharesService'); + + owner = await usersService.createLocalUser({ + email: 'owner@example.com', + username: 'owner', + displayName: 'Owner', + password: 'secret123', + roles: ['user'], + }); + visitor = await usersService.createLocalUser({ + email: 'visitor@example.com', + username: 'visitor', + displayName: 'Visitor', + password: 'secret123', + roles: ['user'], + }); + + await fs.mkdir(path.join(env.volumeDir, 'shared'), { recursive: true }); + share = await sharesService.createShare({ + ownerId: owner.id, + sourcePath: 'shared', + sourceSpace: 'volume', + isDirectory: true, + accessMode: 'readonly', + sharingType: 'anyone', + password: 'link-password', + }); +}); + +afterAll(async () => { + if (env) await env.cleanup(); +}); + +const buildApp = ({ sessionUserId } = {}) => { + const authMiddleware = env.requireFresh('src/middleware/authMiddleware'); + const sharesRoutes = env.requireFresh('src/routes/shares'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + + const app = express(); + app.use(express.json()); + app.use(cookieParser()); + // express-session normally provides this; only the shape matters here. + app.use((req, _res, next) => { + req.session = sessionUserId ? { localUserId: sessionUserId } : {}; + next(); + }); + app.use(authMiddleware); + app.use('/api/share', sharesRoutes); + app.use(errorHandler); + return app; +}; + +describe('Protected share, signed-in visitor', () => { + it('grants access once the password has been verified', async () => { + const app = buildApp({ sessionUserId: visitor.id }); + + // Without verification the visitor is refused, signed in or not. + const before = await request(app).get(`/api/share/${share.shareToken}/access`); + expect(before.status).toBe(401); + + // Verifying the password issues the guest session... + const verify = await request(app) + .post(`/api/share/${share.shareToken}/verify`) + .send({ password: 'link-password' }); + expect(verify.status).toBe(200); + // A browser applies the deletion and sends back only what is left; supertest + // would replay the emptied cookie as-is. + const cookies = verify.headers['set-cookie'].filter((value) => !/^guestSession=;/.test(value)); + + // ...and the very next request has to be accepted. It used to 401 forever. + const after = await request(app) + .get(`/api/share/${share.shareToken}/access`) + .set('Cookie', cookies); + expect(after.status).toBe(200); + }); + + it('drops the /api-scoped cookie an earlier build left behind', async () => { + const verify = await request(buildApp()) + .post(`/api/share/${share.shareToken}/verify`) + .send({ password: 'link-password' }); + + const cookies = verify.headers['set-cookie']; + // Same name on two paths means the stale /api one wins on /api requests, + // so it has to be deleted, not just overwritten. + expect(cookies.some((value) => /^guestSession=;/.test(value) && /Path=\/api/.test(value))).toBe( + true + ); + expect( + cookies.some((value) => /^guestSession=[^;]+/.test(value) && /Path=\//.test(value)) + ).toBe(true); + }); + + it('lets the owner in without the password', async () => { + const app = buildApp({ sessionUserId: owner.id }); + + const response = await request(app).get(`/api/share/${share.shareToken}/access`); + expect(response.status).toBe(200); + + // And the router is told, so it does not send them to the prompt either. + const info = await request(app).get(`/api/share/${share.shareToken}/info`); + expect(info.body.hasPassword).toBe(true); + expect(info.body.requiresPassword).toBe(false); + }); + + it('still asks a signed-in stranger', async () => { + const info = await request(buildApp({ sessionUserId: visitor.id })).get( + `/api/share/${share.shareToken}/info` + ); + expect(info.body.requiresPassword).toBe(true); + }); +}); diff --git a/backend/tests/middleware/held-requests.test.js b/backend/tests/middleware/held-requests.test.js new file mode 100644 index 000000000..943c610a3 --- /dev/null +++ b/backend/tests/middleware/held-requests.test.js @@ -0,0 +1,145 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import express from 'express'; +import request from 'supertest'; + +/** + * A request that is accepted and never answered leaves nothing behind: the + * event loop is idle, the log is quiet, and an orchestrator declares the + * container dead with no more to go on than anyone reading afterwards had. + */ +const logger = require('../../src/utils/logger'); +const { createHeldRequestLogger, markLongPoll } = require('../../src/middleware/heldRequests'); + +// The real logger with its `warn` watched. What this middleware writes is the +// entire point of it, so a stand-in that never sees the real call would be +// testing the stand-in. Real timers throughout: faking the clock stops +// supertest's own sockets from progressing. +let warn; + +beforeEach(() => { + warn = vi.spyOn(logger, 'warn').mockImplementation(() => {}); +}); + +afterEach(() => { + warn.mockRestore(); +}); + +const appHolding = (holdMs, options) => { + const app = express(); + app.use(createHeldRequestLogger({ heldAfterMs: 20, ...options })); + app.get('/slow', (_req, res) => { + setTimeout(() => res.status(200).json({ ok: true }), holdMs); + }); + app.get('/fast', (_req, res) => res.status(200).json({ ok: true })); + return app; +}; + +describe('reporting a request that is held', () => { + it('says nothing about a request that answers straight away', async () => { + await request(appHolding(0)).get('/fast'); + await new Promise((resolve) => setTimeout(resolve, 60)); + + expect(warn).not.toHaveBeenCalled(); + }); + + it('names the path, and says the route was never reached', async () => { + await request(appHolding(120)).get('/slow'); + + expect(warn).toHaveBeenCalledWith( + expect.objectContaining({ method: 'GET', path: '/slow', headersSent: false }), + 'Request accepted and not yet answered' + ); + }); + + // The half that tells a redirect apart from a hang: a request held for + // eleven seconds and then answered 302 is an identity provider, not a + // deadlock, and the status is the only thing that says which. + it('says what it answered in the end', async () => { + await request(appHolding(120)).get('/slow'); + + expect(warn).toHaveBeenCalledWith( + expect.objectContaining({ path: '/slow', statusCode: 200 }), + 'A held request finally answered' + ); + }); + + /** + * An open editor long-polls every thirty seconds and is answered after + * twenty-five. Reporting that is not merely noise: at ten reports it spends + * the whole ceiling in five minutes and leaves the instrument silent for the + * rest of the process's life — the noise would switch the thing off. + */ + it('says nothing about a request a route means to hold', async () => { + const app = express(); + app.use(createHeldRequestLogger({ heldAfterMs: 20 })); + app.get('/poll', (req, res) => { + markLongPoll(req); + setTimeout(() => res.status(200).json({ ok: true }), 120); + }); + + await request(app).get('/poll'); + + expect(warn).not.toHaveBeenCalled(); + }); + + it('still reports one that nobody meant to hold', async () => { + await request(appHolding(120)).get('/slow'); + + expect(warn).toHaveBeenCalled(); + }); + + // A diagnostic for a stuck server must not become the loudest thing in its + // log: a hundred held requests are the same fact reported a hundred times. + it('stops reporting once it has said enough', async () => { + const app = appHolding(80, { maxReported: 2 }); + + await Promise.all([ + request(app).get('/slow'), + request(app).get('/slow'), + request(app).get('/slow'), + request(app).get('/slow'), + ]); + + const held = warn.mock.calls.filter( + ([, message]) => message === 'Request accepted and not yet answered' + ); + expect(held).toHaveLength(2); + }); +}); + +/** + * A request reported as held and then never mentioned again reads as a hang + * that never ended. Someone typing a search abandons one request per + * keystroke, and telling those apart from a stuck server is the whole purpose + * of the instrument. + */ +describe('a held request whose client gives up', () => { + it('says the client stopped waiting', async () => { + const app = express(); + app.use(createHeldRequestLogger({ heldAfterMs: 20 })); + app.get('/slow', (_req, res) => { + setTimeout(() => res.status(200).json({ ok: true }), 5000); + }); + + const pending = request(app).get('/slow'); + pending.end(() => {}); + await new Promise((resolve) => setTimeout(resolve, 80)); + pending.abort(); + await new Promise((resolve) => setTimeout(resolve, 60)); + + expect(warn).toHaveBeenCalledWith( + expect.objectContaining({ path: '/slow' }), + 'A held request was abandoned by its client' + ); + }); + + it('says nothing of the kind about one that answered', async () => { + await request(appHolding(120)).get('/slow'); + await new Promise((resolve) => setTimeout(resolve, 60)); + + const abandoned = warn.mock.calls.filter( + ([, message]) => message === 'A held request was abandoned by its client' + ); + expect(abandoned).toHaveLength(0); + }); +}); diff --git a/backend/tests/middleware/response-end-compat.test.js b/backend/tests/middleware/response-end-compat.test.js new file mode 100644 index 000000000..c387ed2b9 --- /dev/null +++ b/backend/tests/middleware/response-end-compat.test.js @@ -0,0 +1,126 @@ +import { describe, expect, it } from 'vitest'; +import { createRequire } from 'node:module'; + +const require = createRequire(import.meta.url); +const express = require('express'); +const session = require('express-session'); +const request = require('supertest'); + +const { responseEndCompat } = require('../../src/middleware/responseEndCompat'); + +/** + * A store that implements touch, like the SQLite one the application uses. + * With `resave: false` and an established session, that is what sends + * express-session down the branch where it writes the body itself before + * ending — the branch srvx's `res.end(callback)` breaks. + */ +const createTouchingStore = () => { + const sessions = new Map(); + const store = new session.Store(); + store.get = (sid, cb) => setImmediate(() => cb(null, sessions.get(sid) || null)); + store.set = (sid, data, cb) => + setImmediate(() => { + sessions.set(sid, JSON.parse(JSON.stringify(data))); + cb(null); + }); + store.destroy = (sid, cb) => + setImmediate(() => { + sessions.delete(sid); + cb(null); + }); + store.touch = (sid, data, cb) => setImmediate(() => cb(null)); + return store; +}; + +/** + * `whenSent` is what srvx awaits: @tus/server ends the response with a + * callback and waits for it. 201 rather than 204 because Node silently drops + * writes on a response that must not have a body — TUS answers 201 when it + * creates an upload, which is where this bites first. + */ +const createApp = ({ withCompat }) => { + let settle; + const whenSent = new Promise((resolve) => { + settle = resolve; + }); + + const app = express(); + app.use( + session({ + secret: 'test-secret', + resave: false, + saveUninitialized: false, + store: createTouchingStore(), + cookie: { httpOnly: true, maxAge: 60_000 }, + }) + ); + + app.get('/sign-in', (req, res) => { + req.session.user = 'demo'; + res.status(204).end(); + }); + + const handler = (req, res) => { + res.writeHead(201); + new Promise((resolve) => res.end(resolve)).then( + () => settle({ ok: true }), + (error) => settle({ ok: false, code: error?.code }) + ); + }; + + if (withCompat) app.get('/upload/tus', responseEndCompat, handler); + else app.get('/upload/tus', handler); + + return { app, whenSent }; +}; + +describe('res.end compatibility for TUS responses', () => { + it('throws into the caller without the wrapper, once a session exists', async () => { + const { app, whenSent } = createApp({ withCompat: false }); + const agent = request.agent(app); + await agent.get('/sign-in').expect(204); + + // The response never completes, so there is nothing to await here. In + // production this rejection is not handled at all and the process exits. + agent.get('/upload/tus').end(() => {}); + + expect(await whenSent).toEqual({ ok: false, code: 'ERR_INVALID_ARG_TYPE' }); + }); + + it('sends the response and resolves the caller with the wrapper', async () => { + const { app, whenSent } = createApp({ withCompat: true }); + const agent = request.agent(app); + await agent.get('/sign-in').expect(204); + + await agent.get('/upload/tus').expect(201); + + expect(await whenSent).toEqual({ ok: true }); + }); + + it('still sends a body when one is given', async () => { + const app = express(); + app.get('/echo', responseEndCompat, (req, res) => { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('hello'); + }); + + const response = await request(app).get('/echo').expect(200); + expect(response.text).toBe('hello'); + }); + + it('accepts the (chunk, encoding, callback) form', async () => { + let called = false; + const app = express(); + app.get('/echo', responseEndCompat, (req, res) => { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('hello', 'utf8', () => { + called = true; + }); + }); + + const response = await request(app).get('/echo').expect(200); + expect(response.text).toBe('hello'); + await new Promise((resolve) => setImmediate(resolve)); + expect(called).toBe(true); + }); +}); diff --git a/backend/tests/middleware/sanitize-client-message.test.js b/backend/tests/middleware/sanitize-client-message.test.js new file mode 100644 index 000000000..3db21296d --- /dev/null +++ b/backend/tests/middleware/sanitize-client-message.test.js @@ -0,0 +1,81 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Error messages reach the browser, and the ones coming from the filesystem + * carry absolute paths that describe the server's layout. They are reduced to + * a basename — without eating the sentence around them, which an earlier + * pattern did whenever a message mentioned two paths. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const load = async () => { + currentEnv = await setupTestEnv({ + tag: 'sanitize-message-', + modules: ['src/config/env', 'src/config/index', 'src/middleware/errorHandler'], + }); + return currentEnv.requireFresh('src/middleware/errorHandler').sanitizeClientMessage; +}; + +describe('sanitizeClientMessage', () => { + it('keeps the sentence intact when a message names two paths', async () => { + const sanitize = await load(); + + // The whole point: the words between the two paths must survive. + expect(sanitize('Failed to copy /srv/data/a.txt to /srv/data/b.txt')).toBe( + 'Failed to copy …/a.txt to …/b.txt' + ); + }); + + it('reduces a path to its basename wherever it appears', async () => { + const sanitize = await load(); + + expect(sanitize('/var/lib/app/report.pdf is locked')).toBe('…/report.pdf is locked'); + expect(sanitize('cannot read (/var/lib/x/y.txt)')).toBe('cannot read (…/y.txt)'); + // A space in the file name itself is fine: the user typed that one. + expect(sanitize('ENOENT: /srv/data/my file.txt not found')).toBe( + 'ENOENT: …/my file.txt not found' + ); + }); + + it('leaves a message with no path alone', async () => { + const sanitize = await load(); + + expect(sanitize('Permission denied')).toBe('Permission denied'); + expect(sanitize('')).toBe(''); + expect(sanitize(undefined)).toBe(undefined); + }); +}); + +describe('OIDC error redirect', () => { + it('redacts the path before putting it in the URL', async () => { + currentEnv = await setupTestEnv({ + tag: 'oidc-redirect-redaction-', + modules: ['src/config/env', 'src/config/index', 'src/middleware/errorHandler'], + }); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + + const app = express(); + app.get('/callback', (_req, _res, next) => { + next(Object.assign(new Error('ENOENT: /srv/data/secret.txt missing'), { status: 500 })); + }); + app.use(errorHandler); + + // A browser navigation to /callback is what triggers the redirect branch. + const response = await request(app).get('/callback').set('Accept', 'text/html'); + + expect(response.status).toBe(302); + expect(response.headers.location).toContain(encodeURIComponent('…/secret.txt')); + expect(response.headers.location).not.toContain('srv'); + }); +}); diff --git a/backend/tests/middleware/security-hardening.test.js b/backend/tests/middleware/security-hardening.test.js index a69719b69..c8777f5ac 100644 --- a/backend/tests/middleware/security-hardening.test.js +++ b/backend/tests/middleware/security-hardening.test.js @@ -92,7 +92,6 @@ describe('The application', () => { const app = await createApp({ skipOidc: true, skipStaticFiles: true }); for (const route of ['/api/features', '/api/volumes', '/healthz']) { - // eslint-disable-next-line no-await-in-loop const response = await request(app).get(route); expect(response.headers['x-frame-options']).toBe('SAMEORIGIN'); expect(response.headers['x-content-type-options']).toBe('nosniff'); diff --git a/backend/tests/openapi/tour.js b/backend/tests/openapi/tour.js index 4fc6a46bc..c63ab2b0c 100644 --- a/backend/tests/openapi/tour.js +++ b/backend/tests/openapi/tour.js @@ -353,6 +353,7 @@ const tour = async (app, { volume, requireFresh }) => { destination: 'Documents/Projets', }) ); + await call('GET /api/files/recent-destinations', admin.get('/api/files/recent-destinations')); await call( 'POST /api/files/delete-impact', admin diff --git a/backend/tests/routes/account-language.test.js b/backend/tests/routes/account-language.test.js index 3a589e037..5c833192c 100644 --- a/backend/tests/routes/account-language.test.js +++ b/backend/tests/routes/account-language.test.js @@ -74,8 +74,8 @@ describe('a language on the account', () => { }); it('is a preference the settings route accepts', async () => { - const { USER_SETTING_KEYS } = settings(); + const { WRITABLE_USER_SETTINGS } = settings(); - expect(USER_SETTING_KEYS.has('locale')).toBe(true); + expect(WRITABLE_USER_SETTINGS.has('locale')).toBe(true); }); }); diff --git a/backend/tests/routes/admin-guards.test.js b/backend/tests/routes/admin-guards.test.js new file mode 100644 index 000000000..513454cfd --- /dev/null +++ b/backend/tests/routes/admin-guards.test.js @@ -0,0 +1,106 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Every route file used to carry its own copy of the admin check, which is how + * /permissions/chmod and /permissions/chown ended up with none at all. They now + * share one middleware — these pin that the routes actually refuse a regular + * user, so a future refactor cannot quietly drop the guard again. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const REGULAR_USER = { id: 'user-1', username: 'regular', roles: ['user'] }; +const ADMIN_USER = { id: 'admin-1', username: 'admin', roles: ['admin'] }; + +const buildApp = (env, routes, user) => { + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + if (user) req.user = user; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +describe('Admin-only routes', () => { + it('refuses a regular user on settings and user volumes', async () => { + const env = await setupTestEnv({ + tag: 'admin-guards-', + env: { USER_VOLUMES: 'true' }, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/utils/pathUtils', + 'src/services/db', + 'src/services/users', + 'src/services/userVolumesService', + 'src/services/settingsService', + 'src/middleware/ensureAdmin', + 'src/middleware/errorHandler', + 'src/routes/settings', + 'src/routes/userVolumes', + ], + }); + currentEnv = env; + + const settingsRoutes = env.requireFresh('src/routes/settings'); + const volumeRoutes = env.requireFresh('src/routes/userVolumes'); + + const logoUpload = await request(buildApp(env, settingsRoutes, REGULAR_USER)).post( + '/api/settings/upload-logo' + ); + expect(logoUpload.status).toBe(403); + + const listVolumes = await request(buildApp(env, volumeRoutes, REGULAR_USER)).get( + '/api/users/user-2/volumes' + ); + expect(listVolumes.status).toBe(403); + + const browse = await request(buildApp(env, volumeRoutes, REGULAR_USER)).get( + '/api/admin/browse-directories' + ); + expect(browse.status).toBe(403); + + // And the refusal is shaped like every other API error, not a bare string. + expect(listVolumes.body?.error?.message).toMatch(/admin/i); + }); + + it('lets an admin through the same guard', async () => { + const env = await setupTestEnv({ + tag: 'admin-guards-allow-', + env: { USER_VOLUMES: 'true' }, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/utils/pathUtils', + 'src/services/db', + 'src/services/users', + 'src/services/userVolumesService', + 'src/middleware/ensureAdmin', + 'src/middleware/errorHandler', + 'src/routes/userVolumes', + ], + }); + currentEnv = env; + + const volumeRoutes = env.requireFresh('src/routes/userVolumes'); + const response = await request(buildApp(env, volumeRoutes, ADMIN_USER)).get( + '/api/admin/browse-directories' + ); + + expect(response.status).toBe(200); + }); +}); diff --git a/backend/tests/routes/archive-browse-real.test.js b/backend/tests/routes/archive-browse-real.test.js new file mode 100644 index 000000000..463ad8794 --- /dev/null +++ b/backend/tests/routes/archive-browse-real.test.js @@ -0,0 +1,370 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import fss from 'node:fs'; +import express from 'express'; +import request from 'supertest'; +import { TarArchive, ZipArchive } from 'archiver'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { hasSevenZip, isoBuilder, buildIso } from '../helpers/media-tools.js'; + +/** + * The same browsing, against a real 7-Zip and a real archive. + * + * The suite beside this one stands 7-Zip in, which proves what is made of a + * listing and nothing about the listing itself: the shape of that output is + * 7-Zip's to decide, and it is what every guard here is written against. So + * this builds an archive, asks the real tool, and reads the answer through the + * route — the whole chain, once. + * + * Skipped where 7-Zip is not installed. CI asks for it by name through + * REQUIRE_MEDIA_TOOLS, so a skip there is a failure rather than a quiet gap. + */ + +const sevenZip = await hasSevenZip(); +const iso = sevenZip ? await isoBuilder() : false; + +let currentEnv; + +afterEach(async () => { + try { + await currentEnv?.requireFresh('src/services/archiveCacheService').stopArchiveCacheWork(); + } catch (_) { + // Never loaded, which is as stopped as it gets. + } + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +/** A real zip, written by the same library the application writes zips with. */ +const writeZip = (file, entries) => + new Promise((resolve, reject) => { + const output = fss.createWriteStream(file); + const archive = new ZipArchive({ zlib: { level: 1 } }); + output.on('close', resolve); + archive.on('error', reject); + archive.pipe(output); + for (const entry of entries) { + if (entry.directory) archive.append(null, { name: `${entry.path}/`, type: 'directory' }); + else archive.append(entry.content ?? 'x', { name: entry.path }); + } + archive.finalize(); + }); + +/** A real .tar.gz: gzip around a tar, which is two archives rather than one. */ +const writeTarGz = (file, entries) => + new Promise((resolve, reject) => { + const output = fss.createWriteStream(file); + const archive = new TarArchive({ gzip: true }); + output.on('close', resolve); + archive.on('error', reject); + archive.pipe(output); + for (const entry of entries) archive.append(entry.content ?? 'x', { name: entry.path }); + archive.finalize(); + }); + +const buildApp = () => { + const routes = currentEnv.requireFresh('src/routes/archive'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'u1', roles: ['admin'] }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'archive-real-' }); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1','u@example.com',1,'u','U','["admin"]', ?, ?)` + ).run(now, now); + return currentEnv.volumeDir; +}; + +describe.skipIf(!sevenZip)('browsing a real archive with the real 7-Zip', () => { + it('reads the top level, then a folder inside it', async () => { + const volume = await seed(); + await writeZip(path.join(volume, 'backup.zip'), [ + { path: 'notes.txt', content: 'twelve bytes' }, + { path: 'docs/report.txt', content: 'a report' }, + { path: 'docs/deep/inner.txt', content: 'deeper' }, + { path: 'photos', directory: true }, + ]); + const app = buildApp(); + + const top = await request(app).get('/api/archive/list').query({ path: 'backup.zip' }); + + expect(top.status).toBe(200); + expect(top.body.entries.map((entry) => entry.name)).toEqual(['docs', 'photos', 'notes.txt']); + const notes = top.body.entries.find((entry) => entry.name === 'notes.txt'); + expect(notes).toMatchObject({ isDirectory: false, size: 12 }); + expect(notes.modified).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/); + + const inside = await request(app) + .get('/api/archive/list') + .query({ path: 'backup.zip', inside: 'docs' }); + + expect(inside.status).toBe(200); + expect(inside.body.entries.map((entry) => entry.name)).toEqual(['deep', 'report.txt']); + }); + + /** + * Nothing is written to disk to answer: the whole point of this, and the + * thing a later refactor would quietly lose. + */ + it('leaves the folder holding the archive exactly as it was', async () => { + const volume = await seed(); + await writeZip(path.join(volume, 'backup.zip'), [ + { path: 'docs/report.txt', content: 'a report' }, + ]); + const before = await fs.readdir(volume); + + const response = await request(buildApp()) + .get('/api/archive/list') + .query({ path: 'backup.zip', inside: 'docs' }); + + expect(response.status).toBe(200); + expect(await fs.readdir(volume)).toEqual(before); + }); + + /** + * The read, against the real tool. Two things only a real 7-Zip can prove: + * that `-so` writes the bytes rather than a file, and that a name holding a + * character 7-Zip reads as a pattern comes back as one file rather than + * every file it matches. + */ + it('takes one file out, by its exact name', async () => { + const volume = await seed(); + await writeZip(path.join(volume, 'backup.zip'), [ + { path: 'docs/report.txt', content: 'the report itself' }, + { path: 'docs/other.txt', content: 'not this one' }, + ]); + + const response = await request(buildApp()) + .get('/api/archive/entry') + .query({ path: 'backup.zip', entry: 'docs/report.txt' }) + .buffer(true) + .parse((res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks))); + }); + + expect(response.status).toBe(200); + expect(response.body.toString()).toBe('the report itself'); + expect(response.headers['content-disposition']).toContain('report.txt'); + }); + + it('reads a name 7-Zip would otherwise take for a pattern', async () => { + const volume = await seed(); + await writeZip(path.join(volume, 'backup.zip'), [ + { path: 'report*.txt', content: 'the literal one' }, + { path: 'report1.txt', content: 'not this' }, + { path: 'report2.txt', content: 'nor this' }, + ]); + + const response = await request(buildApp()) + .get('/api/archive/entry') + .query({ path: 'backup.zip', entry: 'report*.txt' }) + .buffer(true) + .parse((res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks))); + }); + + expect(response.status).toBe(200); + expect(response.body.toString()).toBe('the literal one'); + }); + + it('writes nothing to disk to hand a file over', async () => { + const volume = await seed(); + await writeZip(path.join(volume, 'backup.zip'), [ + { path: 'docs/report.txt', content: 'the report itself' }, + ]); + const before = await fs.readdir(volume); + + await request(buildApp()) + .get('/api/archive/entry') + .query({ path: 'backup.zip', entry: 'docs/report.txt' }); + + expect(await fs.readdir(volume)).toEqual(before); + }); + + /** + * The compound case, against the real tool. What the stand-in cannot prove is + * the assumption the whole path rests on: that 7-Zip reports a .tar.gz as one + * entry whose name ends in .tar. If that is ever untrue, this is where it + * shows, rather than in somebody's backup. + */ + it('goes inside a real .tar.gz, and keeps one copy of the tar', async () => { + const volume = await seed(); + await writeTarGz(path.join(volume, 'backup.tar.gz'), [ + { path: 'docs/report.txt', content: 'a report' }, + { path: 'notes.txt', content: 'twelve bytes' }, + ]); + const app = buildApp(); + + const top = await request(app).get('/api/archive/list').query({ path: 'backup.tar.gz' }); + + expect(top.status).toBe(200); + expect(top.body.entries.map((entry) => entry.name)).toEqual(['docs', 'notes.txt']); + + const inside = await request(app) + .get('/api/archive/list') + .query({ path: 'backup.tar.gz', inside: 'docs' }); + + expect(inside.body.entries.map((entry) => entry.name)).toEqual(['report.txt']); + + const entry = await request(app) + .get('/api/archive/entry') + .query({ path: 'backup.tar.gz', entry: 'docs/report.txt' }) + .buffer(true) + .parse((res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks))); + }); + + expect(entry.body.toString()).toBe('a report'); + + // One copy, in the cache directory, and nothing beside the archive. + const cached = await fs.readdir(path.join(currentEnv.cacheDir, 'archives')); + expect(cached.filter((name) => name.endsWith('.inner'))).toHaveLength(1); + expect(await fs.readdir(volume)).toEqual(['backup.tar.gz']); + }); + + /** + * `.tgz` is the same two archives under one extension, and 7-Zip is the one + * that decides what the entry inside is called. The rule this rests on is + * that the name ends in `.tar` — which is a claim about 7-Zip's own extension + * mapping, not about our code, so only the real tool can answer it. + */ + it('goes inside a real .tgz, whose inner name only 7-Zip decides', async () => { + const volume = await seed(); + await writeTarGz(path.join(volume, 'backup.tgz'), [ + { path: 'docs/report.txt', content: 'a report' }, + { path: 'notes.txt', content: 'twelve bytes' }, + ]); + const app = buildApp(); + + const top = await request(app).get('/api/archive/list').query({ path: 'backup.tgz' }); + + expect(top.status).toBe(200); + expect(top.body.entries.map((entry) => entry.name)).toEqual(['docs', 'notes.txt']); + + const entry = await request(app) + .get('/api/archive/entry') + .query({ path: 'backup.tgz', entry: 'notes.txt' }) + .buffer(true) + .parse((res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks))); + }); + + expect(entry.body.toString()).toBe('twelve bytes'); + expect(await fs.readdir(volume)).toEqual(['backup.tgz']); + }); + + /** + * The formats that are not zip. + * + * Everything offered here goes through the same two commands, so what this + * checks is that assumption rather than each format's own business: a .7z + * made solid, where reading one entry means decompressing the ones before + * it; a plain .tar, which is a filesystem laid end to end; and an ISO, which + * is a filesystem full stop and only looks like an archive because 7-Zip + * makes it. + */ + it.each([ + ['a solid .7z', 'backup.7z'], + ['a plain .tar', 'backup.tar'], + ])('reads %s the same way', async (_name, filename) => { + const volume = await seed(); + const source = path.join(currentEnv.tmpRoot, 'source'); + await fs.mkdir(path.join(source, 'docs'), { recursive: true }); + await fs.writeFile(path.join(source, 'notes.txt'), 'twelve bytes'); + await fs.writeFile(path.join(source, 'docs', 'report.txt'), 'a report'); + + const { execFile } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const run = promisify(execFile); + const archivePath = path.join(volume, filename); + // Solid is 7-Zip's default for .7z, which is the case worth covering: the + // entries share one compressed stream. + await run('7z', ['a', '-y', archivePath, '.'], { cwd: source }); + + const app = buildApp(); + const top = await request(app).get('/api/archive/list').query({ path: filename }); + + expect(top.status).toBe(200); + expect(top.body.entries.map((entry) => entry.name)).toEqual(['docs', 'notes.txt']); + + const entry = await request(app) + .get('/api/archive/entry') + .query({ path: filename, entry: 'docs/report.txt' }) + .buffer(true) + .parse((res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks))); + }); + + expect(entry.body.toString()).toBe('a report'); + }); + + it.skipIf(!iso)('reads an ISO, which is a filesystem rather than an archive', async () => { + const volume = await seed(); + const source = path.join(currentEnv.tmpRoot, 'source'); + await fs.mkdir(path.join(source, 'docs'), { recursive: true }); + await fs.writeFile(path.join(source, 'notes.txt'), 'twelve bytes'); + await fs.writeFile(path.join(source, 'docs', 'report.txt'), 'a report'); + await buildIso(iso, source, path.join(volume, 'disc.iso')); + + const app = buildApp(); + const top = await request(app).get('/api/archive/list').query({ path: 'disc.iso' }); + + expect(top.status).toBe(200); + expect(top.body.entries.map((entry) => entry.name)).toEqual(['docs', 'notes.txt']); + + const inside = await request(app) + .get('/api/archive/list') + .query({ path: 'disc.iso', inside: 'docs' }); + + expect(inside.body.entries.map((entry) => entry.name)).toEqual(['report.txt']); + + const entry = await request(app) + .get('/api/archive/entry') + .query({ path: 'disc.iso', entry: 'docs/report.txt' }) + .buffer(true) + .parse((res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks))); + }); + + expect(entry.body.toString()).toBe('a report'); + }); + + it('refuses a file that is not an archive, whatever it is called', async () => { + const volume = await seed(); + await fs.writeFile(path.join(volume, 'pretend.zip'), 'not a zip at all'); + + const response = await request(buildApp()) + .get('/api/archive/list') + .query({ path: 'pretend.zip' }); + + expect(response.status).toBe(422); + expect(response.body.error.code).toBe('ARCHIVE_UNREADABLE'); + }); +}); diff --git a/backend/tests/routes/archive-browse-route.test.js b/backend/tests/routes/archive-browse-route.test.js new file mode 100644 index 000000000..c3d06ec5d --- /dev/null +++ b/backend/tests/routes/archive-browse-route.test.js @@ -0,0 +1,399 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { useFakeSevenZip, fakeListing } from '../helpers/fake-seven-zip.js'; + +/** + * Looking inside an archive over the API. + * + * The archive is addressed like any other file — resolved and authorized the + * same way — and where the caller is looking inside it is a separate + * parameter. What this pins is the boundary: who may look, at what, and that + * nothing inside an archive is ever taken for a path on disk. + * + * What 7-Zip itself prints is stood in for; what is made of it is covered in + * `services/archive-browse.test.js`, and against a real 7-Zip in + * `archive-browse-real.test.js`. + */ + +let currentEnv; +let restoreSevenZip; + +afterEach(async () => { + restoreSevenZip?.(); + restoreSevenZip = null; + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async ({ env = {}, rules = [] } = {}) => { + restoreSevenZip = useFakeSevenZip(); + currentEnv = await setupTestEnv({ tag: 'archive-browse-', env }); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1','u@example.com',1,'u','U','["admin"]', ?, ?)` + ).run(now, now); + if (rules.length) { + await currentEnv.requireFresh('src/services/accessControlService').setRules(rules); + } + return currentEnv.volumeDir; +}; + +const buildApp = (user = { id: 'u1', email: 'u@example.com', roles: ['admin'] }) => { + const routes = currentEnv.requireFresh('src/routes/archive'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = user; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +/** An archive whose listing is the one given, written where 7-Zip will read it. */ +const writeArchive = async (volume, name, entries) => { + const file = path.join(volume, name); + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, typeof entries === 'string' ? entries : fakeListing(entries)); + return file; +}; + +const list = (query) => request(buildApp()).get('/api/archive/list').query(query); + +const named = (response) => response.body.entries.map((entry) => entry.name); + +describe('listing what is in an archive', () => { + it('answers the top level, folders before files', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [ + { path: 'notes.txt', size: 12 }, + { path: 'docs/report.txt', size: 4096 }, + { path: 'photos', directory: true }, + ]); + + const response = await list({ path: 'pack.zip' }); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ path: 'pack.zip', name: 'pack.zip', inside: '' }); + expect(named(response)).toEqual(['docs', 'photos', 'notes.txt']); + expect(response.body.entries[2]).toMatchObject({ size: 12, isDirectory: false }); + }); + + it('goes down a level without showing what is below it', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [ + { path: 'docs/report.txt', size: 1 }, + { path: 'docs/deep/inner.txt', size: 2 }, + { path: 'elsewhere.txt', size: 3 }, + ]); + + const response = await list({ path: 'pack.zip', inside: 'docs' }); + + expect(response.status).toBe(200); + expect(response.body.inside).toBe('docs'); + expect(named(response)).toEqual(['deep', 'report.txt']); + }); + + it('says how many entries the archive holds in all', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [ + { path: 'a.txt', size: 1 }, + { path: 'b/c.txt', size: 2 }, + { path: 'b/d.txt', size: 3 }, + ]); + + const response = await list({ path: 'pack.zip' }); + + expect(response.body.total).toBe(3); + }); + + /** + * The names a crafted archive carries are counted rather than shown: the + * panel can say so, and nothing in the answer claims a place for them. + */ + it('leaves out what points outside the archive, and says how much', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [ + { path: '../../etc/passwd', size: 1 }, + { path: 'C:/Windows/notepad.exe', size: 2 }, + { path: 'safe.txt', size: 3 }, + ]); + + const response = await list({ path: 'pack.zip' }); + + expect(named(response)).toEqual(['safe.txt']); + expect(response.body.outside).toBe(2); + }); + + it('refuses a position that points outside the archive', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [{ path: 'docs/report.txt', size: 1 }]); + + const response = await list({ path: 'pack.zip', inside: '../../etc' }); + + expect(response.status).toBe(400); + expect(response.body.error.code).toBe('ARCHIVE_BAD_POSITION'); + }); + + it('says a folder is not there rather than answering an empty one', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [{ path: 'docs/report.txt', size: 1 }]); + + const response = await list({ path: 'pack.zip', inside: 'nowhere' }); + + expect(response.status).toBe(404); + expect(response.body.error.code).toBe('ARCHIVE_ENTRY_NOT_FOUND'); + }); +}); + +describe('an archive that cannot be browsed', () => { + it('says so when its table of contents is behind a password', async () => { + const volume = await seed(); + await writeArchive(volume, 'secret.zip', 'FAKE-7Z-ENCRYPTED\n'); + + const response = await list({ path: 'secret.zip' }); + + expect(response.status).toBe(409); + expect(response.body.error.code).toBe('ARCHIVE_ENCRYPTED'); + }); + + it('says so when it is damaged', async () => { + const volume = await seed(); + await writeArchive(volume, 'broken.zip', 'FAKE-7Z-BROKEN\n'); + + const response = await list({ path: 'broken.zip' }); + + expect(response.status).toBe(422); + expect(response.body.error.code).toBe('ARCHIVE_UNREADABLE'); + }); + + it('refuses to open a file that is not an archive at all', async () => { + const volume = await seed(); + await fs.writeFile(path.join(volume, 'notes.txt'), 'not an archive'); + + const response = await list({ path: 'notes.txt' }); + + expect(response.status).toBe(400); + }); + + it('refuses a folder', async () => { + const volume = await seed(); + await fs.mkdir(path.join(volume, 'Docs.zip'), { recursive: true }); + + const response = await list({ path: 'Docs.zip' }); + + expect(response.status).toBe(400); + }); + + it('says not found for an archive that is not there', async () => { + await seed(); + + const response = await list({ path: 'absent.zip' }); + + expect(response.status).toBe(404); + }); + + it('needs a path at all', async () => { + await seed(); + + const response = await list({}); + + expect(response.status).toBe(400); + }); +}); + +describe('who may look inside an archive', () => { + /** + * The archive is read through the same resolution as any other file, so a + * folder an administrator hid or made unreadable hides its archives too. + * Without this, browsing would be a way to read what listing refuses. + */ + it('refuses one in a folder the caller may not read', async () => { + const volume = await seed({ + env: { USER_VOLUMES: 'true' }, + rules: [{ path: 'Private', permissions: 'hidden', recursive: true }], + }); + await writeArchive(volume, 'Private/pack.zip', [{ path: 'a.txt', size: 1 }]); + + const routes = currentEnv.requireFresh('src/routes/archive'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'restricted', roles: [] }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + + const response = await request(app) + .get('/api/archive/list') + .query({ path: 'Private/pack.zip' }); + + expect([403, 404]).toContain(response.status); + }); + + it('refuses a path that climbs out of the volume', async () => { + await seed(); + + const response = await list({ path: '../../etc/passwd' }); + + expect([400, 403, 404]).toContain(response.status); + expect(response.status).not.toBe(200); + }); +}); + +/** + * Taking one file out of an archive. + * + * The name is the caller's, so the answer is decided by the archive: it is + * looked up in the listing, and what comes back is that entry or nothing. And + * it always arrives as an attachment — a file inside somebody's archive is + * their HTML as easily as their photograph, and served inline it would run on + * this application's origin. + */ +describe('reading one entry of an archive', () => { + /** Served as an attachment, so nothing here is a type superagent parses. */ + const binary = (res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks))); + }; + + const read = (query) => + request(buildApp()).get('/api/archive/entry').query(query).buffer(true).parse(binary); + + /** A refusal is JSON, and is read as JSON. */ + const refuse = (query) => request(buildApp()).get('/api/archive/entry').query(query); + + it('answers the bytes of the entry that was asked for', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [ + { path: 'notes.txt', size: 12, content: 'twelve bytes' }, + { path: 'docs/report.txt', size: 8, content: 'a report' }, + ]); + + const response = await read({ path: 'pack.zip', entry: 'docs/report.txt' }); + + expect(response.status).toBe(200); + expect(response.body.toString()).toBe('a report'); + }); + + it('names and types the file it hands over, as an attachment', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [ + { path: 'docs/report.txt', size: 8, content: 'a report' }, + ]); + + const response = await read({ path: 'pack.zip', entry: 'docs/report.txt' }); + + expect(response.headers['content-disposition']).toContain('attachment'); + expect(response.headers['content-disposition']).toContain('report.txt'); + // The type table this application keeps is about media; everything else is + // handed over as bytes, which for an attachment is the safe answer anyway. + expect(response.headers['content-type']).toContain('application/octet-stream'); + expect(response.headers['x-content-type-options']).toBe('nosniff'); + expect(response.headers['content-length']).toBe('8'); + }); + + /** Even a page, which is the one this rule is about. */ + it('hands over a web page as an attachment too', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [{ path: 'page.html', size: 5, content: '

' }]); + + const response = await read({ path: 'pack.zip', entry: 'page.html' }); + + expect(response.status).toBe(200); + expect(response.headers['content-disposition']).toContain('attachment'); + }); + + it('refuses a name that points outside the archive', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [{ path: 'notes.txt', size: 1, content: 'x' }]); + + const response = await refuse({ path: 'pack.zip', entry: '../../etc/passwd' }); + + expect(response.status).toBe(400); + expect(response.body.error.code).toBe('ARCHIVE_BAD_POSITION'); + }); + + /** + * The lookup is the guard: a name the archive does not hold is refused here + * rather than handed to 7-Zip to be interpreted. + */ + it('refuses a name the archive does not hold', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [{ path: 'notes.txt', size: 1, content: 'x' }]); + + const response = await refuse({ path: 'pack.zip', entry: 'invented.txt' }); + + expect(response.status).toBe(404); + expect(response.body.error.code).toBe('ARCHIVE_ENTRY_NOT_FOUND'); + }); + + it('refuses a folder', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [ + { path: 'docs', directory: true }, + { path: 'docs/report.txt', size: 1, content: 'x' }, + ]); + + const response = await refuse({ path: 'pack.zip', entry: 'docs' }); + + expect(response.status).toBe(400); + expect(response.body.error.code).toBe('ARCHIVE_ENTRY_IS_FOLDER'); + }); + + it('refuses an entry whose contents are encrypted', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [ + { path: 'secret.txt', size: 6, encrypted: true, content: 'hidden' }, + ]); + + const response = await refuse({ path: 'pack.zip', entry: 'secret.txt' }); + + expect(response.status).toBe(409); + expect(response.body.error.code).toBe('ARCHIVE_ENCRYPTED'); + }); + + it('needs an entry to read', async () => { + const volume = await seed(); + await writeArchive(volume, 'pack.zip', [{ path: 'notes.txt', size: 1, content: 'x' }]); + + const response = await refuse({ path: 'pack.zip' }); + + expect(response.status).toBe(400); + }); + + it('is refused for an archive the caller may not read', async () => { + const volume = await seed({ + env: { USER_VOLUMES: 'true' }, + rules: [{ path: 'Private', permissions: 'hidden', recursive: true }], + }); + await writeArchive(volume, 'Private/pack.zip', [{ path: 'notes.txt', size: 1, content: 'x' }]); + + const routes = currentEnv.requireFresh('src/routes/archive'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'restricted', roles: [] }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + + const response = await request(app) + .get('/api/archive/entry') + .query({ path: 'Private/pack.zip', entry: 'notes.txt' }); + + expect([403, 404]).toContain(response.status); + }); +}); diff --git a/backend/tests/routes/archive-compound.test.js b/backend/tests/routes/archive-compound.test.js new file mode 100644 index 000000000..5da172c8d --- /dev/null +++ b/backend/tests/routes/archive-compound.test.js @@ -0,0 +1,225 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { useFakeSevenZip, fakeListing, fakeCompound } from '../helpers/fake-seven-zip.js'; + +/** + * Looking inside a .tar.gz, which is two archives rather than one. + * + * 7-Zip peels one layer per run, so listing `backup.tar.gz` answers with a + * single entry called `backup.tar` — true, and no use at all to somebody + * looking for a file inside it. And a tar cannot be read from the middle: gzip + * has no index, so reaching the last entry means decompressing everything + * before it. Doing that per request would mean decompressing the whole backup + * to list one folder, and again for the next click. + * + * So it is decompressed once, into the cache directory, and both the listing + * and the reads go to that copy. What this pins is the once, the ceiling above + * which it is not done at all, and that nothing is written where somebody's + * files are. + */ + +let currentEnv; +let restoreSevenZip; + +afterEach(async () => { + try { + const cache = currentEnv?.requireFresh('src/services/archiveCacheService'); + await cache?.stopArchiveCacheWork(); + } catch (_) { + // Never loaded, which is as stopped as it gets. + } + restoreSevenZip?.(); + restoreSevenZip = null; + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async (env = {}) => { + restoreSevenZip = useFakeSevenZip(); + currentEnv = await setupTestEnv({ tag: 'archive-compound-', env }); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1','u@example.com',1,'u','U','["admin"]', ?, ?)` + ).run(now, now); + return currentEnv.volumeDir; +}; + +const buildApp = () => { + const routes = currentEnv.requireFresh('src/routes/archive'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'u1', roles: ['admin'] }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +/** An archive that decompresses to another archive, as a .tar.gz does. */ +const writeCompound = async (volume, name, options) => { + const { outer, inner } = fakeCompound(options); + const file = path.join(volume, name); + await fs.writeFile(file, outer); + await fs.writeFile(`${file}.inner`, inner); + return file; +}; + +const list = (query) => request(buildApp()).get('/api/archive/list').query(query); + +const cachedFiles = async () => { + try { + return (await fs.readdir(path.join(currentEnv.cacheDir, 'archives'))).sort(); + } catch (_) { + return []; + } +}; + +describe('a compound archive', () => { + it('shows what is inside the tar, not the tar itself', async () => { + const volume = await seed(); + await writeCompound(volume, 'backup.tar.gz', { + entries: [ + { path: 'docs/report.txt', size: 8, content: 'a report' }, + { path: 'notes.txt', size: 12, content: 'twelve bytes' }, + ], + }); + + const response = await list({ path: 'backup.tar.gz' }); + + expect(response.status).toBe(200); + expect(response.body.entries.map((entry) => entry.name)).toEqual(['docs', 'notes.txt']); + }); + + it('goes down a level inside it', async () => { + const volume = await seed(); + await writeCompound(volume, 'backup.tar.gz', { + entries: [{ path: 'docs/report.txt', size: 8, content: 'a report' }], + }); + + const response = await list({ path: 'backup.tar.gz', inside: 'docs' }); + + expect(response.status).toBe(200); + expect(response.body.entries.map((entry) => entry.name)).toEqual(['report.txt']); + }); + + it('reads one file out of it', async () => { + const volume = await seed(); + await writeCompound(volume, 'backup.tar.gz', { + entries: [{ path: 'docs/report.txt', size: 8, content: 'a report' }], + }); + + const response = await request(buildApp()) + .get('/api/archive/entry') + .query({ path: 'backup.tar.gz', entry: 'docs/report.txt' }) + .buffer(true) + .parse((res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks))); + }); + + expect(response.status).toBe(200); + expect(response.body.toString()).toBe('a report'); + }); + + /** Decompressed once: the second look reads the copy the first one made. */ + it('keeps one copy, and uses it again', async () => { + const volume = await seed(); + const file = await writeCompound(volume, 'backup.tar.gz', { + entries: [{ path: 'notes.txt', size: 12, content: 'twelve bytes' }], + }); + + await list({ path: 'backup.tar.gz' }); + const afterFirst = await cachedFiles(); + // Taking the source away proves the second listing did not go back to it. + await fs.rm(`${file}.inner`); + + const second = await list({ path: 'backup.tar.gz' }); + + expect(afterFirst).toHaveLength(1); + expect(await cachedFiles()).toEqual(afterFirst); + expect(second.status).toBe(200); + expect(second.body.entries.map((entry) => entry.name)).toEqual(['notes.txt']); + }); + + /** Nowhere near the volume: a file there would be listed, indexed and backed up. */ + it('writes its copy under the cache directory and nowhere else', async () => { + const volume = await seed(); + await writeCompound(volume, 'backup.tar.gz', { + entries: [{ path: 'notes.txt', size: 12, content: 'twelve bytes' }], + }); + const before = (await fs.readdir(volume)).sort(); + + await list({ path: 'backup.tar.gz' }); + + expect((await fs.readdir(volume)).sort()).toEqual(before); + expect(await cachedFiles()).toHaveLength(1); + }); + + /** + * The refusal comes before anything is written: the size is what the outer + * archive already declares, so it costs a listing rather than a disk. + */ + it('refuses to open one too large to hold, and writes nothing', async () => { + const volume = await seed({ MAX_BROWSABLE_ARCHIVE_SIZE: '1K' }); + await writeCompound(volume, 'huge.tar.gz', { + innerSize: 4096, + entries: [{ path: 'notes.txt', size: 12, content: 'twelve bytes' }], + }); + + const response = await list({ path: 'huge.tar.gz' }); + + expect(response.status).toBe(413); + expect(response.body.error.code).toBe('ARCHIVE_TOO_LARGE_TO_BROWSE'); + expect(await cachedFiles()).toEqual([]); + }); + + /** + * A gzipped text file is one entry too, and it is not an archive to go + * inside: it is a file to hand over, which the listing already offers. + */ + it('leaves a gzipped file as the one file it is', async () => { + const volume = await seed(); + await fs.writeFile( + path.join(volume, 'notes.txt.gz'), + fakeListing([{ path: 'notes.txt', size: 12, content: 'twelve bytes' }]) + ); + + const response = await list({ path: 'notes.txt.gz' }); + + expect(response.status).toBe(200); + expect(response.body.entries.map((entry) => entry.name)).toEqual(['notes.txt']); + expect(await cachedFiles()).toEqual([]); + }); + + /** An archive replaced by another of the same name is a different archive. */ + it('makes a new copy when the archive itself changes', async () => { + const volume = await seed(); + const file = await writeCompound(volume, 'backup.tar.gz', { + entries: [{ path: 'first.txt', size: 5, content: 'first' }], + }); + await list({ path: 'backup.tar.gz' }); + + const { outer, inner } = fakeCompound({ + entries: [{ path: 'second.txt', size: 6, content: 'second' }], + }); + await fs.writeFile(file, `${outer}\n`); + await fs.writeFile(`${file}.inner`, inner); + await fs.utimes(file, new Date(Date.now() + 60_000), new Date(Date.now() + 60_000)); + + const response = await list({ path: 'backup.tar.gz' }); + + expect(response.body.entries.map((entry) => entry.name)).toEqual(['second.txt']); + expect(await cachedFiles()).toHaveLength(2); + }); +}); diff --git a/backend/tests/routes/archive-extract-entry.test.js b/backend/tests/routes/archive-extract-entry.test.js new file mode 100644 index 000000000..2257076ca --- /dev/null +++ b/backend/tests/routes/archive-extract-entry.test.js @@ -0,0 +1,305 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { hasSevenZip } from '../helpers/media-tools.js'; + +/** + * Taking part of an archive out onto the volume. + * + * The other half of looking inside one: a folder of photographs in a backup is + * found here and wanted *there*, and downloading it to put it back is not an + * answer on a server somebody reaches from a phone. + * + * A real 7-Zip throughout, because what is being pinned is what lands on disk: + * which entries came out, which did not, and what happens to the name when + * something already holds it. A stand-in would only prove the arguments. + */ + +const sevenZip = await hasSevenZip(); + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'archive-extract-' }); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1','u@example.com',1,'u','U','["admin"]', ?, ?)` + ).run(now, now); + return currentEnv.volumeDir; +}; + +const buildApp = (user = { id: 'u1', roles: ['admin'] }) => { + const routes = currentEnv.requireFresh('src/routes/archive'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = user; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +/** A real zip, built with 7-Zip itself so nothing here depends on a library. */ +const writeArchive = async (volume, name, entries) => { + const { execFile } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const run = promisify(execFile); + const source = path.join(currentEnv.tmpRoot, `source-${name}`); + for (const entry of entries) { + const file = path.join(source, entry.path); + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, entry.content ?? 'x'); + } + await run('7z', ['a', '-y', path.join(volume, name), '.'], { cwd: source }); +}; + +/** The events the extraction writes, read as the interface reads them. */ +const extract = async (body) => { + const response = await request(buildApp()).post('/api/archive/extract').send(body); + const events = response.text + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)); + return { response, events, done: events.find((event) => event.type === 'done') }; +}; + +const tree = async (directory) => { + const found = await fs.readdir(directory, { recursive: true }); + return found.map((entry) => entry.split(path.sep).join('/')).sort(); +}; + +describe.skipIf(!sevenZip)('taking one entry out of an archive', () => { + it('writes the file beside the archive, and nothing else', async () => { + const volume = await seed(); + await writeArchive(volume, 'backup.zip', [ + { path: 'notes.txt', content: 'twelve bytes' }, + { path: 'docs/report.txt', content: 'a report' }, + { path: 'docs/deep/inner.txt', content: 'deeper' }, + ]); + + const { done } = await extract({ path: 'backup.zip', entries: ['notes.txt'] }); + + expect(done).toMatchObject({ success: true }); + expect(await tree(volume)).toEqual(['backup.zip', 'notes.txt']); + expect(await fs.readFile(path.join(volume, 'notes.txt'), 'utf8')).toBe('twelve bytes'); + }); + + it('takes a folder with everything under it', async () => { + const volume = await seed(); + await writeArchive(volume, 'backup.zip', [ + { path: 'notes.txt', content: 'twelve bytes' }, + { path: 'docs/report.txt', content: 'a report' }, + { path: 'docs/deep/inner.txt', content: 'deeper' }, + ]); + + const { done } = await extract({ path: 'backup.zip', entries: ['docs'] }); + + expect(done.success).toBe(true); + expect(await tree(volume)).toEqual([ + 'backup.zip', + 'docs', + 'docs/deep', + 'docs/deep/inner.txt', + 'docs/report.txt', + ]); + }); + + it('takes several at once', async () => { + const volume = await seed(); + await writeArchive(volume, 'backup.zip', [ + { path: 'one.txt', content: '1' }, + { path: 'two.txt', content: '2' }, + { path: 'three.txt', content: '3' }, + ]); + + const { done } = await extract({ path: 'backup.zip', entries: ['one.txt', 'three.txt'] }); + + expect(done.items.map((item) => item.name).sort()).toEqual(['one.txt', 'three.txt']); + expect(await tree(volume)).toEqual(['backup.zip', 'one.txt', 'three.txt']); + }); + + /** + * The rule the whole application is held to: nothing is ever replaced. The + * file that was there keeps its name and its contents, and what comes out of + * the archive takes the next one. + */ + it('never replaces a file that already holds the name', async () => { + const volume = await seed(); + await writeArchive(volume, 'backup.zip', [{ path: 'notes.txt', content: 'from the archive' }]); + await fs.writeFile(path.join(volume, 'notes.txt'), 'the original'); + + const { done } = await extract({ path: 'backup.zip', entries: ['notes.txt'] }); + + expect(done.success).toBe(true); + expect(await fs.readFile(path.join(volume, 'notes.txt'), 'utf8')).toBe('the original'); + expect(await fs.readFile(path.join(volume, 'notes (1).txt'), 'utf8')).toBe('from the archive'); + }); + + it('leaves nothing hidden behind it', async () => { + const volume = await seed(); + await writeArchive(volume, 'backup.zip', [{ path: 'notes.txt', content: 'twelve bytes' }]); + + await extract({ path: 'backup.zip', entries: ['notes.txt'] }); + + const hidden = (await fs.readdir(volume)).filter((name) => name.startsWith('.')); + expect(hidden).toEqual([]); + }); + + it.each([ + ['a name that points outside the archive', ['../../etc/passwd']], + ['a name the archive does not hold', ['invented.txt']], + ['nothing at all', []], + ])('refuses %s', async (_name, entries) => { + const volume = await seed(); + await writeArchive(volume, 'backup.zip', [{ path: 'notes.txt', content: 'twelve bytes' }]); + + const response = await request(buildApp()) + .post('/api/archive/extract') + .send({ path: 'backup.zip', entries }); + + expect([400, 404]).toContain(response.status); + expect(await tree(volume)).toEqual(['backup.zip']); + }); + + /** + * Reading an archive is not the right to write beside it: a share that only + * lets somebody look must not become a way to put files on the volume. + */ + it('refuses a caller who may read the archive and not write in its folder', async () => { + const volume = await seed(); + await fs.mkdir(path.join(volume, 'ReadOnly'), { recursive: true }); + await writeArchive(path.join(volume, 'ReadOnly'), 'backup.zip', [ + { path: 'notes.txt', content: 'twelve bytes' }, + ]); + await currentEnv + .requireFresh('src/services/accessControlService') + .setRules([{ path: 'ReadOnly', permissions: 'ro', recursive: true }]); + + const routes = currentEnv.requireFresh('src/routes/archive'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: 'reader', roles: [] }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + + const response = await request(app) + .post('/api/archive/extract') + .send({ path: 'ReadOnly/backup.zip', entries: ['notes.txt'] }); + + expect(response.status).toBe(403); + expect(await tree(path.join(volume, 'ReadOnly'))).toEqual(['backup.zip']); + }); +}); + +/** + * Where it comes out. + * + * The folder the archive sits in is the answer when nobody says otherwise, and + * it is the one the dialog offers first. A named destination changes where the + * files land and nothing else: the same right to write is asked for, in the + * same way, so pointing somewhere is never a way around what a read-only + * folder means. + */ +describe.skipIf(!sevenZip)('choosing where it comes out', () => { + it('puts what comes out in the folder that was asked for', async () => { + const volume = await seed(); + await writeArchive(volume, 'backup.zip', [{ path: 'notes.txt', content: 'twelve bytes' }]); + await fs.mkdir(path.join(volume, 'Elsewhere')); + + const { done } = await extract({ + path: 'backup.zip', + entries: ['notes.txt'], + destination: 'Elsewhere', + }); + + expect(done).toMatchObject({ success: true }); + expect(await tree(volume)).toEqual(['Elsewhere', 'Elsewhere/notes.txt', 'backup.zip']); + expect(done.items[0].path).toBe('Elsewhere'); + }); + + it('falls back to the folder the archive is in, not to the root', async () => { + const volume = await seed(); + await fs.mkdir(path.join(volume, 'Backups')); + await writeArchive(volume, path.join('Backups', 'backup.zip'), [ + { path: 'notes.txt', content: 'twelve bytes' }, + ]); + + const { done } = await extract({ path: 'Backups/backup.zip', entries: ['notes.txt'] }); + + expect(done.success).toBe(true); + expect(await tree(volume)).toEqual(['Backups', 'Backups/backup.zip', 'Backups/notes.txt']); + }); + + /** + * Which layer refuses it is not the point: this route normalises the name and + * the path layer under it refuses the same thing again. What is pinned is + * that the answer is a refusal, before anything is written. + */ + it('refuses a destination that climbs out of the volume', async () => { + const volume = await seed(); + await writeArchive(volume, 'backup.zip', [{ path: 'notes.txt', content: 'twelve bytes' }]); + + const { response } = await extract({ + path: 'backup.zip', + entries: ['notes.txt'], + destination: '../../etc', + }); + + expect(response.status).toBe(400); + expect(await tree(volume)).toEqual(['backup.zip']); + }); + + /** + * Said before the stream starts, rather than as a failed event halfway + * through it: the first thing the extraction does is make a staging folder + * inside the destination, and that error would arrive after "start". + */ + it('refuses a destination that is not there, as an ordinary error', async () => { + const volume = await seed(); + await writeArchive(volume, 'backup.zip', [{ path: 'notes.txt', content: 'twelve bytes' }]); + + const { response, events } = await extract({ + path: 'backup.zip', + entries: ['notes.txt'], + destination: 'Nowhere', + }); + + expect(response.status).toBe(404); + expect(events.some((event) => event.type === 'start')).toBe(false); + expect(await tree(volume)).toEqual(['backup.zip']); + }); + + it('refuses a destination that is a file rather than a folder', async () => { + const volume = await seed(); + await writeArchive(volume, 'backup.zip', [{ path: 'notes.txt', content: 'twelve bytes' }]); + await fs.writeFile(path.join(volume, 'target.txt'), 'not a folder'); + + const { response } = await extract({ + path: 'backup.zip', + entries: ['notes.txt'], + destination: 'target.txt', + }); + + expect(response.status).toBe(404); + expect(await tree(volume)).toEqual(['backup.zip', 'target.txt']); + }); +}); diff --git a/backend/tests/routes/archive-solid-cache.test.js b/backend/tests/routes/archive-solid-cache.test.js new file mode 100644 index 000000000..7c86bbed7 --- /dev/null +++ b/backend/tests/routes/archive-solid-cache.test.js @@ -0,0 +1,257 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv, clearModuleCache } from '../helpers/env-test-utils.js'; +import { useFakeSevenZip, fakeListing } from '../helpers/fake-seven-zip.js'; + +/** + * Reading a solid archive more than once. + * + * A solid `.7z` compresses every file into one stream, so reaching the last + * entry means decompressing the ones before it. Measured on a runner with a + * real 7-Zip (`scripts/measure-solid-7z.mjs`), on fifty megabytes that + * compress about two to one: the first entry takes 0.02 s, the last 1.37 s, + * and extracting the whole archive 1.41 s — about what that one read costs. + * Ten entries read one at a time take 6.95 s. + * + * So the rule this pins: leave the first read alone, and on the second extract + * once and serve every read after it from disk. Anything else — a zip, an + * archive too large to hold — reads from the archive as it always did. + */ + +let currentEnv; +let restoreSevenZip; +let logFile; + +afterEach(async () => { + try { + await currentEnv?.requireFresh('src/services/archiveCacheService').stopArchiveCacheWork(); + } catch (_) { + // Never loaded, which is as stopped as it gets. + } + restoreSevenZip?.(); + restoreSevenZip = null; + delete process.env.FAKE_7Z_LOG; + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async (env = {}) => { + restoreSevenZip = useFakeSevenZip(); + logFile = path.join(await fs.mkdtemp(path.join(os.tmpdir(), 'fake-7z-log-')), 'calls'); + process.env.FAKE_7Z_LOG = logFile; + + currentEnv = await setupTestEnv({ tag: 'archive-solid-', env }); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1','u@example.com',1,'u','U','["admin"]', ?, ?)` + ).run(now, now); + return currentEnv.volumeDir; +}; + +/** + * One application, for the whole of a test. + * + * Built once rather than per request on purpose: how many times an archive has + * been read is something a running server remembers, and rebuilding the module + * graph between two reads would be a restart — which forgets, exactly as a + * real restart does. + */ +const buildApp = () => { + const routes = currentEnv.requireFresh('src/routes/archive'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'u1', roles: ['admin'] }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +const ENTRIES = [ + { path: 'first.txt', size: 5, content: 'first' }, + { path: 'docs/second.txt', size: 6, content: 'second' }, + { path: 'docs/third.txt', size: 5, content: 'third' }, +]; + +const writeArchive = async (volume, name, { solid = true, entries = ENTRIES } = {}) => { + const file = path.join(volume, name); + await fs.writeFile(file, fakeListing(entries, { solid })); + return file; +}; + +const read = async (app, archive, entry) => { + const response = await request(app) + .get('/api/archive/entry') + .query({ path: archive, entry }) + .buffer(true) + .parse((res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks))); + }); + return response; +}; + +/** What 7-Zip was asked to do, one line per call. */ +const calls = async () => + (await fs.readFile(logFile, 'utf8').catch(() => '')).split('\n').filter(Boolean); + +const extractions = async () => + (await calls()).filter((line) => line.startsWith('x ') && line.includes(' -o')); + +const cached = async () => { + const directory = currentEnv.requireFresh('src/services/archiveCacheService').cacheDirectory(); + return (await fs.readdir(directory).catch(() => [])).sort(); +}; + +describe('the second read of a solid archive', () => { + it('leaves the first read alone', async () => { + const volume = await seed(); + await writeArchive(volume, 'solid.7z'); + + const app = buildApp(); + + const response = await read(app, 'solid.7z', 'first.txt'); + + expect(response.status).toBe(200); + expect(response.body.toString()).toBe('first'); + expect(await extractions()).toHaveLength(0); + expect(await cached()).toEqual([]); + }); + + it('extracts once, and answers the reads after it from disk', async () => { + const volume = await seed(); + await writeArchive(volume, 'solid.7z'); + + const app = buildApp(); + + await read(app, 'solid.7z', 'first.txt'); + const second = await read(app, 'solid.7z', 'docs/second.txt'); + + expect(second.body.toString()).toBe('second'); + expect(await extractions()).toHaveLength(1); + expect((await cached()).filter((name) => name.endsWith('.tree'))).toHaveLength(1); + + const before = (await calls()).length; + const third = await read(app, 'solid.7z', 'docs/third.txt'); + + expect(third.body.toString()).toBe('third'); + // Nothing more was asked of 7-Zip: the listing is cached in neither + // direction, so the only calls left are the ones that read the archive. + expect((await calls()).filter((line) => line.startsWith('x '))).toHaveLength(2); + expect((await calls()).length).toBeGreaterThan(before - 1); + }); + + /** A zip has a start for every entry: there is nothing to save. */ + it('never does it for an archive that is not solid', async () => { + const volume = await seed(); + await writeArchive(volume, 'plain.zip', { solid: false }); + + const app = buildApp(); + + await read(app, 'plain.zip', 'first.txt'); + await read(app, 'plain.zip', 'docs/second.txt'); + const third = await read(app, 'plain.zip', 'docs/third.txt'); + + expect(third.body.toString()).toBe('third'); + expect(await extractions()).toHaveLength(0); + expect(await cached()).toEqual([]); + }); + + /** + * The objection the measurement had to answer: extracting eight gigabytes + * because somebody clicked one file is a worse trade than the slow read. The + * ceiling is the one that already decides what may be browsed at all. + */ + it('refuses to extract one larger than the ceiling, and reads it anyway', async () => { + const volume = await seed({ MAX_BROWSABLE_ARCHIVE_SIZE: '8' }); + await writeArchive(volume, 'huge.7z'); + + const app = buildApp(); + + await read(app, 'huge.7z', 'first.txt'); + const second = await read(app, 'huge.7z', 'docs/second.txt'); + + expect(second.status).toBe(200); + expect(second.body.toString()).toBe('second'); + expect(await extractions()).toHaveLength(0); + }); + + /** One archive's reads are its own. */ + it('counts each archive separately', async () => { + const volume = await seed(); + await writeArchive(volume, 'one.7z'); + await writeArchive(volume, 'two.7z'); + + const app = buildApp(); + + await read(app, 'one.7z', 'first.txt'); + await read(app, 'two.7z', 'first.txt'); + + expect(await extractions()).toHaveLength(0); + }); +}); + +describe('a tree that is already there', () => { + /** + * The count of reads lives in memory, so a restart forgets it. What does not + * go with it is the tree on disk: the first read after a restart is as free + * as the tenth before it, or the extraction would be done again for nothing. + */ + it('is used from the first read, however many this process has counted', async () => { + const volume = await seed(); + await writeArchive(volume, 'solid.7z'); + + const before = buildApp(); + await read(before, 'solid.7z', 'first.txt'); + await read(before, 'solid.7z', 'docs/second.txt'); + expect(await extractions()).toHaveLength(1); + + // A restart, and a real one: clearing the route alone would leave the + // service — and the count it holds — exactly where it was. + clearModuleCache('src/services/archiveBrowseService'); + const after = buildApp(); + const soFar = (await calls()).filter((line) => line.startsWith('x ')).length; + const response = await read(after, 'solid.7z', 'docs/third.txt'); + + expect(response.body.toString()).toBe('third'); + expect((await calls()).filter((line) => line.startsWith('x '))).toHaveLength(soFar); + }); +}); + +/** + * The second lock on a door the listing already closed. + * + * A name that climbs out of the archive is dropped when the listing is read, + * so nothing should ever reach this — which is exactly why it is worth a test + * of its own: the day something does, joining it onto the tree's path would + * read a file on the disk instead. + */ +describe('reading a name out of the tree', () => { + it('refuses one that points outside it', async () => { + await seed(); + const service = currentEnv.requireFresh('src/services/archiveBrowseService'); + const tree = path.join(currentEnv.tmpRoot, 'tree'); + await fs.mkdir(path.join(tree, 'inside'), { recursive: true }); + await fs.writeFile(path.join(tree, 'inside', 'ok.txt'), 'in'); + await fs.writeFile(path.join(currentEnv.tmpRoot, 'secret.txt'), 'out'); + + expect(await service.openFromTree(tree, '../secret.txt')).toBeNull(); + expect(await service.openFromTree(tree, '/etc/hosts')).toBeNull(); + expect(await service.openFromTree(tree, 'inside/../../secret.txt')).toBeNull(); + + const inside = await service.openFromTree(tree, 'inside/ok.txt'); + expect(inside).not.toBeNull(); + inside.stop(); + }); +}); diff --git a/backend/tests/routes/auth-password.test.js b/backend/tests/routes/auth-password.test.js new file mode 100644 index 000000000..369fac703 --- /dev/null +++ b/backend/tests/routes/auth-password.test.js @@ -0,0 +1,233 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { createRequire } from 'node:module'; +import express from 'express'; +import session from 'express-session'; +import request from 'supertest'; +import { setupTestEnv, modulePath } from '../helpers/env-test-utils.js'; + +const require = createRequire(import.meta.url); + +/** + * Changing, adding and listing the ways an account signs in. + * + * A signed-in session is not proof of knowing the password: a browser left + * open, or a session id that leaked, is enough to reach these routes. So the + * change asks for the current password, and adding one refuses where a + * password already exists — otherwise "add" would be a change that skips the + * question. A refused change has to leave the old password working, which is + * asserted by signing in with it rather than by reading the status code alone. + * + * Every test hashes passwords with bcryptjs at cost 12, hence the timeout. + */ + +const PASSWORD = 'secret123'; + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const build = async () => { + currentEnv = await setupTestEnv({ tag: 'auth-password-', env: { AUTH_ENABLED: 'true' } }); + const authRoutes = currentEnv.requireFresh('src/routes/auth'); + const errorHandlers = currentEnv.requireFresh('src/middleware/errorHandler'); + const users = require(modulePath('src/services/users')); + const db = await require(modulePath('src/services/db')).getDb(); + + /** An app whose requests arrive as `userId`, the way the auth middleware hands them over. */ + const appFor = (userId = null) => { + const app = express(); + app.use(express.json()); + app.use( + session({ secret: process.env.SESSION_SECRET, resave: false, saveUninitialized: false }) + ); + app.use((req, _res, next) => { + req.oidc = { isAuthenticated: () => false }; + if (userId) req.user = { id: userId, roles: ['user'] }; + next(); + }); + app.use('/api/auth', authRoutes); + app.use(errorHandlers.notFoundHandler); + app.use(errorHandlers.errorHandler); + return app; + }; + + return { app: appFor(), appFor, users, db }; +}; + +/** The owner, signed in through the setup route on an agent that keeps the session. */ +const signedInOwner = async (app) => { + const browser = request.agent(app); + const setup = await browser + .post('/api/auth/setup') + .send({ email: 'owner@example.com', username: 'owner', password: PASSWORD }); + expect(setup.status).toBe(201); + return { browser, owner: setup.body.user }; +}; + +/** An account that signs in only through its identity provider: no password row. */ +const federatedAccount = (db) => { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('fed-1', 'federated@example.com', 1, 'federated', 'Federated', '["user"]', ?, ?)` + ).run(now, now); + db.prepare( + `INSERT INTO auth_methods (id, user_id, method_type, provider_issuer, provider_sub, provider_name, created_at) + VALUES ('fed-oidc', 'fed-1', 'oidc', 'https://idp.example', 'sub-fed', 'OIDC', ?)` + ).run(now); + return 'fed-1'; +}; + +const signInStatus = async (app, identifier, password) => + (await request(app).post('/api/auth/login').send({ identifier, password })).status; + +describe('who may touch a password', () => { + it.each([ + ['post', '/api/auth/password', { currentPassword: PASSWORD, newPassword: 'another456' }], + ['post', '/api/auth/password/add', { password: PASSWORD }], + ['get', '/api/auth/methods', undefined], + ])('refuses %s %s to someone who is not signed in', async (method, url, body) => { + const { app } = await build(); + + const pending = request(app)[method](url); + const response = body ? await pending.send(body) : await pending; + + expect(response.status).toBe(401); + expect(response.body.error.message).toBe('Authentication required.'); + }); +}); + +describe('changing a password', { timeout: 30_000 }, () => { + it('asks for the current password, and changes nothing without it', async () => { + const { app } = await build(); + const { browser } = await signedInOwner(app); + + const response = await browser.post('/api/auth/password').send({ newPassword: 'another456' }); + + expect(response.status).toBe(400); + expect(response.body.error.message).toBe('Current password is required.'); + expect(await signInStatus(app, 'owner', PASSWORD)).toBe(200); + expect(await signInStatus(app, 'owner', 'another456')).toBe(401); + }); + + it('refuses a new password under six characters, and keeps the old one', async () => { + const { app } = await build(); + const { browser } = await signedInOwner(app); + + const response = await browser + .post('/api/auth/password') + .send({ currentPassword: PASSWORD, newPassword: '12345' }); + + expect(response.status).toBe(400); + expect(response.body.error.code).toBe('VALIDATION_PASSWORD_TOO_SHORT'); + expect(await signInStatus(app, 'owner', PASSWORD)).toBe(200); + expect(await signInStatus(app, 'owner', '12345')).toBe(401); + }); + + /** A 204 is only worth something if the password it reports changed actually did. */ + it('replaces the password: the old one stops signing in and the new one starts', async () => { + const { app } = await build(); + const { browser } = await signedInOwner(app); + + const response = await browser + .post('/api/auth/password') + .send({ currentPassword: PASSWORD, newPassword: 'another456' }); + + expect(response.status).toBe(204); + expect(await signInStatus(app, 'owner', PASSWORD)).toBe(401); + expect(await signInStatus(app, 'owner', 'another456')).toBe(200); + }); + + it('is refused for an account that signs in only through its identity provider', async () => { + const { appFor, db } = await build(); + const userId = federatedAccount(db); + + const response = await request(appFor(userId)) + .post('/api/auth/password') + .send({ currentPassword: 'anything', newPassword: 'another456' }); + + expect(response.status).toBe(400); + expect(response.body.error.message).toBe( + 'Password change is only allowed for users with password authentication.' + ); + }); +}); + +describe('adding a password to an account', { timeout: 30_000 }, () => { + it('gives an account from the identity provider a password it can then sign in with', async () => { + const { app, appFor, db } = await build(); + const userId = federatedAccount(db); + + const response = await request(appFor(userId)) + .post('/api/auth/password/add') + .send({ password: PASSWORD }); + + expect(response.status).toBe(200); + const signIn = await request(app) + .post('/api/auth/login') + .send({ identifier: 'federated@example.com', password: PASSWORD }); + expect(signIn.status).toBe(200); + expect(signIn.body.user.id).toBe(userId); + }); + + it('refuses a password under six characters, and adds none', async () => { + const { appFor, db } = await build(); + const userId = federatedAccount(db); + + const response = await request(appFor(userId)) + .post('/api/auth/password/add') + .send({ password: '12345' }); + + expect(response.status).toBe(400); + expect(response.body.error.code).toBe('VALIDATION_PASSWORD_TOO_SHORT'); + const passwords = db + .prepare( + "SELECT COUNT(*) AS n FROM auth_methods WHERE user_id = ? AND method_type = 'local_password'" + ) + .get(userId).n; + expect(passwords).toBe(0); + }); + + /** + * Adding over an existing password would be a change that never asked for + * the current one — exactly what a borrowed session must not be able to do. + */ + it('refuses to add one where a password exists, and the existing one keeps working', async () => { + const { app, appFor } = await build(); + const { owner } = await signedInOwner(app); + + const response = await request(appFor(owner.id)) + .post('/api/auth/password/add') + .send({ password: 'taken-over' }); + + expect(response.status).toBe(409); + expect(response.body.error.code).toBe('CONFLICT_PASSWORD_EXISTS'); + expect(await signInStatus(app, 'owner', 'taken-over')).toBe(401); + expect(await signInStatus(app, 'owner', PASSWORD)).toBe(200); + }); +}); + +describe('listing how an account signs in', { timeout: 30_000 }, () => { + it('names each method without handing out the password hash', async () => { + const { app, appFor, db } = await build(); + const { owner } = await signedInOwner(app); + db.prepare( + `INSERT INTO auth_methods (id, user_id, method_type, provider_issuer, provider_sub, provider_name, created_at) + VALUES ('owner-oidc', ?, 'oidc', 'https://idp.example', 'sub-owner', 'OIDC', ?)` + ).run(owner.id, new Date().toISOString()); + + const response = await request(appFor(owner.id)).get('/api/auth/methods'); + + expect(response.status).toBe(200); + expect(response.body.methods.map((m) => [m.type, m.provider]).sort()).toEqual([ + ['local_password', 'Password'], + ['oidc', 'OIDC'], + ]); + expect(JSON.stringify(response.body)).not.toMatch(/password_hash|passwordHash|\$2[aby]\$/); + }); +}); diff --git a/backend/tests/routes/auth-setup-and-sign-in.test.js b/backend/tests/routes/auth-setup-and-sign-in.test.js index a0f2b1629..8e61741a9 100644 --- a/backend/tests/routes/auth-setup-and-sign-in.test.js +++ b/backend/tests/routes/auth-setup-and-sign-in.test.js @@ -247,7 +247,6 @@ describe('signing in', { timeout: 30_000 }, () => { const { app } = await build(); for (let attempt = 0; attempt < 10; attempt += 1) { - // eslint-disable-next-line no-await-in-loop const response = await request(app) .post('/api/auth/login') .send({ email: `guess-${attempt}`, password: PASSWORD }); diff --git a/backend/tests/routes/auth.test.js b/backend/tests/routes/auth.test.js index c6a69780b..8df7732c0 100644 --- a/backend/tests/routes/auth.test.js +++ b/backend/tests/routes/auth.test.js @@ -5,7 +5,7 @@ import express from 'express'; import session from 'express-session'; import bodyParser from 'body-parser'; import request from 'supertest'; -import { setupTestEnv, clearModuleCache } from '../helpers/env-test-utils.js'; +import { setupTestEnv, clearModuleCache, modulePath } from '../helpers/env-test-utils.js'; let envContext; @@ -20,12 +20,26 @@ afterAll(async () => { await envContext.cleanup(); }); -const buildApp = ({ authEnabled } = {}) => { +const buildApp = async ({ authEnabled } = {}) => { if (!envContext) { throw new Error('Test environment not initialized'); } // Ensure each test app starts with a clean database. + // + // Closed before it is removed, not only dropped from the module cache. SQLite + // keeps an unlinked file alive for whoever still holds it open, so the second + // app in a file went on reading the first one's accounts through the old + // handle: `/auth/setup` answered "already configured" and the test that + // needed a fresh install failed on its very first call, for a reason that had + // nothing to do with what it was testing. + try { + // The instance that has it open, not a fresh one: `requireFresh` would hand + // back a module that has never opened anything, and close nothing. + await require(modulePath('src/services/db')).closeDb(); + } catch (_) { + // Nothing had opened it yet. + } try { fs.rmSync(path.join(envContext.configDir, 'app.db'), { force: true }); } catch (_) { @@ -72,7 +86,7 @@ const buildApp = ({ authEnabled } = {}) => { describe('Auth Routes', () => { describe('Authentication Flow', () => { it('should complete setup -> login -> me -> password -> logout flow', async () => { - const app = buildApp({ authEnabled: true }); + const app = await buildApp({ authEnabled: true }); // status before setup const s1 = await request(app).get('/api/auth/status'); @@ -81,13 +95,11 @@ describe('Auth Routes', () => { expect(s1.body.authEnabled).toBe(true); // setup admin - const setup = await request(app) - .post('/api/auth/setup') - .send({ - email: 'admin@example.com', - username: 'admin', - password: 'secret123', - }); + const setup = await request(app).post('/api/auth/setup').send({ + email: 'admin@example.com', + username: 'admin', + password: 'secret123', + }); expect(setup.status).toBe(201); expect(setup.body.user).toBeDefined(); expect(setup.body.user.roles).toContain('admin'); @@ -116,16 +128,14 @@ describe('Auth Routes', () => { }); it('should return JSON 401 when current password is incorrect', async () => { - const app = buildApp({ authEnabled: true }); + const app = await buildApp({ authEnabled: true }); // setup admin - const setup = await request(app) - .post('/api/auth/setup') - .send({ - email: 'admin@example.com', - username: 'admin', - password: 'secret123', - }); + const setup = await request(app).post('/api/auth/setup').send({ + email: 'admin@example.com', + username: 'admin', + password: 'secret123', + }); expect(setup.status).toBe(201); // login @@ -149,7 +159,7 @@ describe('Auth Routes', () => { describe('Auth Status', () => { it('should reflect disabled auth via AUTH_ENABLED', async () => { - const app = buildApp({ authEnabled: false }); + const app = await buildApp({ authEnabled: false }); const status = await request(app).get('/api/auth/status'); expect(status.status).toBe(200); diff --git a/backend/tests/routes/browse-caching.test.js b/backend/tests/routes/browse-caching.test.js new file mode 100644 index 000000000..56d2c75bb --- /dev/null +++ b/backend/tests/routes/browse-caching.test.js @@ -0,0 +1,49 @@ +import express from 'express'; +import request from 'supertest'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A listing is not a document, and must not be cached as one. + * + * `GET /api/browse` carries what is true at that moment: which documents somebody has + * open in an editor, what a folder weighs, whether a write would be refused. None of + * that is worth remembering, and a proxy or a browser that remembers it serves a view + * of a folder as it was — a file that was deleted still listed, a document shown as + * open by somebody who closed it an hour ago. + * + * No header said so, and the answer to a GET with none is cacheable by default. + */ + +let env; + +afterEach(async () => { + if (env) await env.cleanup(); + env = null; +}); + +const app = () => { + const server = express(); + server.use((req, _res, next) => { + req.user = { id: 'admin-1', roles: ['admin'] }; + next(); + }); + server.use('/api', env.requireFresh('src/routes/browse')); + server.use(env.requireFresh('src/middleware/errorHandler').errorHandler); + return server; +}; + +describe('the answer to a listing', () => { + it('is not to be kept by a browser or a proxy', async () => { + env = await setupTestEnv({ tag: 'browse-caching-' }); + + const response = await request(app()).get('/api/browse/'); + + expect(response.status).toBe(200); + // `private` keeps a shared proxy out of it; `no-store` keeps the browser from + // answering the next navigation from what it already has. + expect(response.headers['cache-control']).toContain('no-store'); + expect(response.headers['cache-control']).toContain('private'); + }); +}); diff --git a/backend/tests/routes/browse-hidden-files.test.js b/backend/tests/routes/browse-hidden-files.test.js index 6ef8c7dee..d46d0df41 100644 --- a/backend/tests/routes/browse-hidden-files.test.js +++ b/backend/tests/routes/browse-hidden-files.test.js @@ -20,6 +20,17 @@ const createBrowseContext = async () => { ], }); + // The account the requests below are made as. A preference belongs to an + // account — `user_settings` says so with a foreign key — so writing one for + // an id nothing created fails on the constraint rather than on anything this + // test is about. + const db = await envContext.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('admin', 'admin@example.com', 1, 'admin', 'Admin', '["admin"]', ?, ?)` + ).run(now, now); + const browseRoutes = envContext.requireFresh('src/routes/browse'); const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); const app = createTestApp({ diff --git a/backend/tests/routes/browse-links.test.js b/backend/tests/routes/browse-links.test.js new file mode 100644 index 000000000..f628a0cc2 --- /dev/null +++ b/backend/tests/routes/browse-links.test.js @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import request from 'supertest'; + +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Symbolic links in a listing. + * + * Releases 1.1.8 to 2.0.2 left links in the old cache directory pointing at the + * files moved to /config. Where that directory sits inside a volume, it lists + * them — and the listing followed each one, showing the size and type of a file + * outside the volume, on a row where renaming, deleting and opening were all + * refused with "Resolved path is outside the configured volume root". The + * refusal is right; the row was not. A link that leaves the volume is listed as + * a link, and nothing about what it points at is read to describe it. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const setup = async ({ danglingInside = false } = {}) => { + currentEnv = await setupTestEnv({ tag: 'browse-links-' }); + const browseRoutes = currentEnv.requireFresh('src/routes/browse'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('admin', 'admin@example.com', 1, 'admin', 'Admin', '["admin"]', ?, ?)` + ).run(now, now); + const app = createTestApp({ + router: browseRoutes, + mountPath: '/api', + user: { id: 'admin', roles: ['admin'] }, + errorHandler, + }); + + const volume = currentEnv.volumeDir; + const outside = path.join(currentEnv.tmpRoot, 'config-outside'); + await fs.mkdir(path.join(outside, 'extensions'), { recursive: true }); + // A size nothing inside the volume has, so it can only have been read there. + await fs.writeFile(path.join(outside, 'app-config.json'), 'x'.repeat(4242)); + await fs.writeFile(path.join(outside, 'photo.jpg'), 'not really a photo'); + + await fs.writeFile(path.join(volume, 'real.txt'), 'twelve bytes'); + await fs.symlink(path.join(volume, 'real.txt'), path.join(volume, 'shortcut.txt')); + await fs.symlink(path.join(outside, 'app-config.json'), path.join(volume, 'app-config.json')); + await fs.symlink(path.join(outside, 'extensions'), path.join(volume, 'extensions')); + await fs.symlink(path.join(outside, 'photo.jpg'), path.join(volume, 'photo.jpg')); + await fs.symlink(path.join(outside, 'gone.txt'), path.join(volume, 'dangling.txt')); + if (danglingInside) { + await fs.symlink(path.join(volume, 'never-there.txt'), path.join(volume, 'nowhere.txt')); + } + + const response = await request(app).get('/api/browse/'); + expect(response.status).toBe(200); + const byName = Object.fromEntries(response.body.items.map((item) => [item.name, item])); + return { byName }; +}; + +describe('a symbolic link in a listing', () => { + it('that leaves the volume is listed as a link, with nothing read from what it points at', async () => { + const { byName } = await setup(); + + expect(byName['app-config.json']).toMatchObject({ link: 'outside', size: null, kind: 'json' }); + expect(byName.extensions).toMatchObject({ link: 'outside', size: null }); + expect(byName.extensions.kind).not.toBe('directory'); + }); + + it('that leaves the volume offers no thumbnail of what it points at', async () => { + const { byName } = await setup(); + + expect(byName['photo.jpg']).toMatchObject({ link: 'outside' }); + expect(byName['photo.jpg'].supportsThumbnail).toBeUndefined(); + }); + + it('that stays inside the volume is listed as what it points at, as before', async () => { + const { byName } = await setup(); + + expect(byName['shortcut.txt'].link).toBeUndefined(); + expect(byName['shortcut.txt']).toMatchObject({ kind: 'txt', size: 'twelve bytes'.length }); + expect(byName['real.txt']).toMatchObject({ kind: 'txt', size: 'twelve bytes'.length }); + }); + + it('that leads nowhere outside the volume is still a link out of it, and says so', async () => { + const { byName } = await setup(); + + // An old link to a file /config no longer holds: nothing to open, but the + // row is there on disk, and hiding it left nobody able to see why. + expect(byName['dangling.txt']).toMatchObject({ link: 'outside', size: null }); + }); + + it('that leads nowhere inside the volume is left out, as before', async () => { + const { byName } = await setup({ danglingInside: true }); + + expect(byName['nowhere.txt']).toBeUndefined(); + }); +}); diff --git a/backend/tests/routes/browse-personal.test.js b/backend/tests/routes/browse-personal.test.js new file mode 100644 index 000000000..bfb4231c6 --- /dev/null +++ b/backend/tests/routes/browse-personal.test.js @@ -0,0 +1,173 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * My Files, asked for the way a browser asks for it. + * + * The personal space had no test at this layer at all, and it shipped in a + * release completely broken: `resolveLogicalPath` called the personal resolver + * without awaiting it, so `pathExists` was handed a promise and every folder + * came back "Path not found" — a directory that was right there, reported + * missing. + * + * The containment tests kept passing throughout, because they call the resolver + * directly, one layer below where the application resolves a path and one layer + * below the defect. Nothing asked the question a person asks: open My Files and + * see what is in it. + * + * Thirteen route files resolve paths this way. This covers the space rather + * than the bug. + */ + +let currentEnv; + +const setup = async () => { + const envContext = await setupTestEnv({ + tag: 'browse-personal-', + env: { USER_DIR_ENABLED: 'true' }, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/routes/browse', + 'src/middleware/errorHandler', + 'src/services/accessManager', + 'src/services/settingsService', + 'src/utils/pathUtils', + ], + }); + currentEnv = envContext; + + const browseRoutes = envContext.requireFresh('src/routes/browse'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const { resolvePersonalPath } = envContext.requireFresh('src/utils/pathUtils'); + const { getDb } = envContext.requireFresh('src/services/db'); + + const db = await getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ).run('alice', 'alice@example.com', 1, 'alice', 'Alice', '["user"]', now, now); + + const user = { id: 'alice', username: 'alice', roles: ['user'] }; + const userRoot = await resolvePersonalPath('', user); + await fs.mkdir(userRoot, { recursive: true }); + + const app = createTestApp({ + router: browseRoutes, + mountPath: '/api', + user, + errorHandler, + }); + + return { app, userRoot, envContext }; +}; + +const browse = (app, at = '') => request(app).get(`/api/browse/personal${at ? `/${at}` : ''}`); + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +describe('opening My Files', () => { + it('answers', async () => { + const { app } = await setup(); + + const response = await browse(app); + + expect(response.status).toBe(200); + }); + + it('lists what is in it', async () => { + const { app, userRoot } = await setup(); + await fs.writeFile(path.join(userRoot, 'notes.txt'), 'mine'); + + const response = await browse(app); + + expect((response.body.items || []).map((item) => item.name)).toContain('notes.txt'); + }); + + it('opens a folder inside it', async () => { + const { app, userRoot } = await setup(); + await fs.mkdir(path.join(userRoot, 'docs'), { recursive: true }); + await fs.writeFile(path.join(userRoot, 'docs', 'report.txt'), 'inside'); + + const response = await browse(app, 'docs'); + + expect(response.status).toBe(200); + expect((response.body.items || []).map((item) => item.name)).toContain('report.txt'); + }); + + /** + * The symptom as it was reported: a directory that exists and is readable, + * answered "Path not found". + */ + it('does not call a folder that is there missing', async () => { + const { app, userRoot } = await setup(); + await fs.mkdir(path.join(userRoot, 'docs'), { recursive: true }); + + const response = await browse(app, 'docs'); + + expect(response.status).not.toBe(404); + }); + + it('still says so for a folder that really is not there', async () => { + const { app } = await setup(); + + const response = await browse(app, 'no-such-folder'); + + expect(response.status).toBe(404); + }); +}); + +describe('the edges of My Files', () => { + /** + * One person's space is not another's. The route resolves the folder from the + * signed-in user, never from the path, so there is nothing here to aim + * elsewhere — which is what this pins. + */ + it('refuses a path that climbs out of it', async () => { + const { app } = await setup(); + + const response = await browse(app, '../../etc'); + + expect(response.status).toBeGreaterThanOrEqual(400); + }); + + /** + * And a symbolic link is the other way out. The check that catches it lives + * in the resolver's promise, which is exactly what went unawaited: the + * refusal resolved without complaint and its rejection ended the process + * rather than the request. + */ + it('refuses a symbolic link that leads out of it', async () => { + const { app, userRoot, envContext } = await setup(); + const outside = path.join(envContext.tmpRoot, 'outside-personal-route'); + await fs.mkdir(outside, { recursive: true }); + await fs.writeFile(path.join(outside, 'secret.txt'), 'not yours'); + await fs.symlink(outside, path.join(userRoot, 'escape')); + + const response = await browse(app, 'escape'); + + expect(response.status).toBeGreaterThanOrEqual(400); + expect(JSON.stringify(response.body)).not.toContain('secret.txt'); + }); + + /** Refused as a request, which means the answer arrives at all. */ + it('answers the refusal instead of failing to answer', async () => { + const { app, userRoot, envContext } = await setup(); + const outside = path.join(envContext.tmpRoot, 'outside-answered'); + await fs.mkdir(outside, { recursive: true }); + await fs.symlink(outside, path.join(userRoot, 'escape')); + + const response = await browse(app, 'escape'); + + expect(response.status).toBeLessThan(500); + }); +}); diff --git a/backend/tests/routes/browse-thumbnails-switch.test.js b/backend/tests/routes/browse-thumbnails-switch.test.js new file mode 100644 index 000000000..673dde092 --- /dev/null +++ b/backend/tests/routes/browse-thumbnails-switch.test.js @@ -0,0 +1,78 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The listing must not promise a thumbnail the server will never make. + * + * `supportsThumbnail` on a row is what makes the screen ask for one. Whether + * thumbnails happen at all is settled in two places — THUMBNAILS_ENABLED, for + * the whole installation, and a setting an administrator can turn off — and the + * thumbnail route and the share listing have always read both. The folder + * listing read only the setting, so an installation started with the switch off + * answered rows claiming thumbnails, and every one of those requests came back + * refused. + */ + +let envContext; + +const browseWith = async (env) => { + envContext = await setupTestEnv({ + tag: 'browse-thumbnails-', + env, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/routes/browse', + 'src/middleware/errorHandler', + 'src/services/accessManager', + 'src/services/settingsService', + ], + }); + + await fs.mkdir(path.join(envContext.volumeDir, 'Photos'), { recursive: true }); + await fs.writeFile(path.join(envContext.volumeDir, 'Photos', 'plage.jpg'), 'x'); + + const browseRoutes = envContext.requireFresh('src/routes/browse'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const { getDb } = envContext.requireFresh('src/services/db'); + const db = await getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('admin', 'admin@example.com', 1, 'admin', 'Admin', '["admin"]', ?, ?)` + ).run(now, now); + + const app = createTestApp({ + router: browseRoutes, + mountPath: '/api', + user: { id: 'admin', roles: ['admin'] }, + errorHandler, + }); + + const response = await request(app).get('/api/browse/Photos'); + expect(response.status).toBe(200); + return response.body.items.find((item) => item.name === 'plage.jpg'); +}; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('a picture in a folder listing', () => { + it('is offered a thumbnail when the server makes them', async () => { + expect(await browseWith({ THUMBNAILS_ENABLED: 'true' })).toMatchObject({ + supportsThumbnail: true, + }); + }); + + it('is offered none when the whole installation has them off', async () => { + const picture = await browseWith({ THUMBNAILS_ENABLED: 'false' }); + + expect(picture).toBeDefined(); + expect(picture.supportsThumbnail).toBeUndefined(); + }); +}); diff --git a/backend/tests/routes/browse-version-marks.test.js b/backend/tests/routes/browse-version-marks.test.js new file mode 100644 index 000000000..ae9e5c56f --- /dev/null +++ b/backend/tests/routes/browse-version-marks.test.js @@ -0,0 +1,195 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import express from 'express'; +import request from 'supertest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The mark a listing carries on a file that has earlier versions. + * + * Counted once for the folder, not once per row, and so the interesting cases + * are about which rows the one query answers for: the folder's own files and + * not a descendant's, the files somebody may see the history of and not the + * others, and nothing at all when they asked not to be shown it. + */ + +let envContext; +let users; +let app; + +const load = (relative) => require(modulePath(relative)); + +const volume = (...segments) => path.join(envContext.volumeDir, ...segments); + +const write = async (relative, content) => { + await fs.mkdir(path.dirname(volume(relative)), { recursive: true }); + await fs.writeFile(volume(relative), content); +}; + +const as = (who) => ({ + get: (url) => request(app).get(url).set('x-test-user', who), + put: (url, body) => request(app).put(url).set('x-test-user', who).send(body), + post: (url, body) => request(app).post(url).set('x-test-user', who).send(body), +}); + +/** Save through the text editor: the ordinary way an earlier version appears. */ +const edit = async (who, filePath, content) => { + const response = await as(who).put('/api/editor', { path: filePath, content }); + expect(response.status).toBe(200); +}; + +const browse = async (who, folder) => { + const response = await as(who).get(`/api/browse/${folder}`); + expect(response.status).toBe(200); + return response.body; +}; + +const markOn = (body, name) => body.items.find((item) => item.name === name)?.versions; + +beforeEach(async () => { + envContext = await setupTestEnv({ tag: 'browse-marks-', env: { SHARES_ENABLED: 'true' } }); + users = { + alice: await load('src/services/users').createLocalUser({ + email: 'alice@example.com', + username: 'alice', + displayName: 'Alice', + password: 'secret123', + roles: ['user'], + }), + }; + + app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + const who = req.get('x-test-user'); + if (who?.startsWith('guest:')) { + req.guestSession = { id: 'guest-session', shareId: who.slice('guest:'.length) }; + } else if (who) { + req.user = users[who]; + } + next(); + }); + app.use('/api', load('src/routes/editor')); + app.use('/api', load('src/routes/browse')); + app.use('/api', load('src/routes/versions')); + app.use('/api/shares', load('src/routes/shares')); + app.use('/api/share', load('src/routes/shares')); + app.use(load('src/middleware/errorHandler').errorHandler); + + await write('Projects/notes.md', 'one\n'); + await write('Projects/untouched.md', 'never edited\n'); + await fs.mkdir(volume('Projects', 'deep'), { recursive: true }); +}); + +afterEach(async () => { + load('src/services/trash/maintenance').stop(); + await envContext.cleanup(); +}); + +describe('the mark on a row', () => { + it('says how many versions the file has, and how much they hold', async () => { + await edit('alice', 'Projects/notes.md', 'second\n'); + await edit('alice', 'Projects/notes.md', 'third content\n'); + + const listing = await browse('alice', 'Projects'); + + expect(markOn(listing, 'notes.md')).toMatchObject({ count: 2, bytes: 4 + 7 }); + expect(markOn(listing, 'notes.md').newest).toEqual(expect.any(String)); + }); + + it('is absent, not zero, on a file that has no history', async () => { + await edit('alice', 'Projects/notes.md', 'second\n'); + + const listing = await browse('alice', 'Projects'); + + // Absent rather than `{ count: 0 }`: the client deletes a missing key on + // every refresh, and a zero would be a mark that has to be reasoned about + // at every place that reads one. + expect(markOn(listing, 'untouched.md')).toBeUndefined(); + }); + + it('goes away when the last version does', async () => { + await edit('alice', 'Projects/notes.md', 'second\n'); + expect(markOn(await browse('alice', 'Projects'), 'notes.md')).toMatchObject({ count: 1 }); + + const deleted = await as('alice').post('/api/versions/delete', { + path: 'Projects/notes.md', + all: true, + }); + expect(deleted.status).toBe(200); + + expect(markOn(await browse('alice', 'Projects'), 'notes.md')).toBeUndefined(); + }); + + it('belongs to the file in this folder and not to one of the same name below it', async () => { + // The history of `deep/x.md` is keyed by its own name. Answering for the + // folder above it would put its count on a different file entirely. + await write('Projects/deep/x.md', 'one\n'); + await write('Projects/x.md', 'a different file\n'); + await edit('alice', 'Projects/deep/x.md', 'two\n'); + + const above = await browse('alice', 'Projects'); + const inside = await browse('alice', 'Projects/deep'); + + expect(markOn(above, 'x.md')).toBeUndefined(); + expect(markOn(inside, 'x.md')).toMatchObject({ count: 1 }); + }); + + it('belongs to the file in this folder and not to one of the same name above it', async () => { + // The other way round, which a filter on descendants alone would miss: + // the query has to be bounded to the folder, or every history in the zone + // is a candidate for a row here that happens to share a name. + await write('Projects/deep/notes.md', 'a different file\n'); + await edit('alice', 'Projects/notes.md', 'two\n'); + + const inside = await browse('alice', 'Projects/deep'); + + expect(markOn(inside, 'notes.md')).toBeUndefined(); + expect(markOn(await browse('alice', 'Projects'), 'notes.md')).toMatchObject({ count: 1 }); + }); + + it('is never on a folder, whatever its files hold', async () => { + await write('Projects/deep/x.md', 'one\n'); + await edit('alice', 'Projects/deep/x.md', 'two\n'); + + expect(markOn(await browse('alice', 'Projects'), 'deep')).toBeUndefined(); + }); +}); + +describe('who is shown it', () => { + it('nobody, once they have turned it off', async () => { + await edit('alice', 'Projects/notes.md', 'second\n'); + await load('src/services/settingsService').setUserSetting( + users.alice.id, + 'showVersionMarks', + false + ); + + expect(markOn(await browse('alice', 'Projects'), 'notes.md')).toBeUndefined(); + }); + + it('still them, when they have said nothing: it is on by default', async () => { + await edit('alice', 'Projects/notes.md', 'second\n'); + + expect(markOn(await browse('alice', 'Projects'), 'notes.md')).toMatchObject({ count: 1 }); + }); + + it('through a share, only once its owner turned histories on', async () => { + await edit('alice', 'Projects/notes.md', 'second\n'); + const share = ( + await as('alice').post('/api/shares', { sourcePath: 'Projects', sharingType: 'anyone' }) + ).body; + const listing = () => + request(app) + .get(`/api/share/${share.shareToken}/browse/`) + .set('x-test-user', `guest:${share.id}`); + + expect(markOn((await listing()).body, 'notes.md')).toBeUndefined(); + + await as('alice').put(`/api/shares/${share.id}`, { versionsVisible: true }); + + expect(markOn((await listing()).body, 'notes.md')).toMatchObject({ count: 1 }); + }); +}); diff --git a/backend/tests/routes/collabora-editor-chrome.test.js b/backend/tests/routes/collabora-editor-chrome.test.js new file mode 100644 index 000000000..66415e4db --- /dev/null +++ b/backend/tests/routes/collabora-editor-chrome.test.js @@ -0,0 +1,111 @@ +import fs from 'node:fs/promises'; +import http from 'node:http'; +import path from 'node:path'; +import express from 'express'; +import request from 'supertest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The way out of a Collabora document. + * + * The editor fills the screen, and it draws no close button unless it is asked + * for one. So the page floated a button of its own over the editor's toolbar, + * where it sat looking like something Collabora had not finished drawing + * (nxzai/NextExplorer#303). Asked, the editor draws the button itself, in its + * own toolbar, and posts `UI_Close` rather than closing anything on its own — + * which is the arrangement ONLYOFFICE is already opened with. + * + * The ask is one parameter on the frame's address, read by the editor exactly + * as `revisionhistory` is, so it is checked where that one is: on the answer + * the page is handed, for a document opened either way. + */ + +const COLLABORA_SECRET = 'collabora-chrome-secret'; +const DOCUMENT = 'Projects/report.docx'; + +const DISCOVERY = ` + + +`; + +let env; +let app; +let users; +let discovery; + +const load = (relative) => require(modulePath(relative)); + +beforeEach(async () => { + discovery = http.createServer((_req, res) => { + res.setHeader('Content-Type', 'text/xml'); + res.end(DISCOVERY); + }); + await new Promise((resolve) => discovery.listen(0, '127.0.0.1', resolve)); + const discoveryUrl = `http://127.0.0.1:${discovery.address().port}/hosting/discovery`; + + env = await setupTestEnv({ + tag: 'collabora-chrome-', + env: { + PUBLIC_URL: 'https://files.example.com', + COLLABORA_URL: 'https://collabora.example.com', + COLLABORA_SECRET, + COLLABORA_DISCOVERY_URL: discoveryUrl, + }, + }); + + users = { + alice: await load('src/services/users').createLocalUser({ + email: 'alice@example.com', + username: 'alice', + displayName: 'Alice', + password: 'secret123', + roles: ['user'], + }), + }; + + await fs.mkdir(path.join(env.volumeDir, 'Projects'), { recursive: true }); + await fs.writeFile(path.join(env.volumeDir, 'Projects', 'report.docx'), 'a document'); + + app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = users.alice; + next(); + }); + app.use('/api', load('src/routes/collabora')); + app.use(load('src/middleware/errorHandler').errorHandler); +}); + +afterEach(async () => { + load('src/services/trash/maintenance').stop(); + await new Promise((resolve) => discovery.close(resolve)); + await env.cleanup(); +}); + +const openDocument = () => request(app).post('/api/collabora/config').send({ path: DOCUMENT }); + +describe('the address a document is opened at', () => { + it('asks the editor to draw its own close button', async () => { + const opened = await openDocument(); + + expect(opened.status).toBe(200); + expect(new URL(opened.body.urlSrc).searchParams.get('closebutton')).toBe('1'); + }); + + /** + * A document nobody may write opens through the `view` action, at a different + * address out of discovery. The way out of it is no different. + */ + it('asks for it on a document that opens only to be read', async () => { + await load('src/services/settingsService').setSettings({ + access: { rules: [{ path: 'Projects', recursive: true, permissions: 'ro' }] }, + }); + + const opened = await openDocument(); + + expect(opened.status).toBe(200); + expect(new URL(opened.body.urlSrc).searchParams.get('closebutton')).toBe('1'); + }); +}); diff --git a/backend/tests/routes/collabora-wopi-contract.test.js b/backend/tests/routes/collabora-wopi-contract.test.js new file mode 100644 index 000000000..79873bdfc --- /dev/null +++ b/backend/tests/routes/collabora-wopi-contract.test.js @@ -0,0 +1,314 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What a token opens, and what it does not. + * + * The WOPI endpoints are the one place the application answers without a + * session: the authentication middleware lets `/api/collabora/wopi/` through so + * that Collabora, which is a separate server, can fetch and save the document + * it was handed. The access token is therefore the whole of the authentication, + * and everything it decides — which file, whether it may be written, whether it + * is even a token of this kind — is decided here. + * + * CI measured this route at forty-three per cent, with two tests for five + * endpoints. These cover the contract rather than the happy path: the file + * identifier a token is bound to, the permission it carries, and the locks that + * stop two people saving over each other. + */ + +let currentEnv; + +const buildApp = (routes, { notFoundHandler, errorHandler }) => { + const app = express(); + app.use('/api', routes); + app.use(notFoundHandler); + app.use(errorHandler); + return app; +}; + +const setup = async () => { + currentEnv = await setupTestEnv({ + tag: 'collabora-contract-', + modules: [ + 'src/config/env', + 'src/config/index', + 'src/routes/collabora', + 'src/services/wopiLockService', + 'src/middleware/errorHandler', + ], + env: { + COLLABORA_URL: 'https://collabora.example.com', + COLLABORA_SECRET: 'test-collabora-secret', + PUBLIC_URL: 'https://files.example.com', + }, + }); + + const { collabora } = currentEnv.requireFresh('src/config/index'); + const routes = currentEnv.requireFresh('src/routes/collabora'); + const errorMiddleware = currentEnv.requireFresh('src/middleware/errorHandler'); + + const absolutePath = path.join(currentEnv.tmpRoot, 'quarterly.docx'); + await fs.writeFile(absolutePath, Buffer.from('original')); + + const tokenFor = (claims = {}) => + jwt.sign( + { + typ: 'nextexplorer-wopi', + fileId: 'file-1', + absolutePath, + canWrite: true, + userId: 'user-1', + userName: 'Alice', + ...claims, + }, + collabora.secret, + // `expiresIn` and an explicit `exp` claim contradict each other, and the + // library refuses both together — a test that wants an expired token + // states the moment itself. + 'exp' in claims ? { algorithm: 'HS256' } : { algorithm: 'HS256', expiresIn: 60 } + ); + + return { + app: buildApp(routes, errorMiddleware), + secret: collabora.secret, + absolutePath, + tokenFor, + }; +}; + +const info = (app, token, fileId = 'file-1') => + request(app).get(`/api/collabora/wopi/files/${fileId}`).query({ access_token: token }); + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +describe('the token that stands in for a session', () => { + it('opens the file it names', async () => { + const { app, tokenFor } = await setup(); + + expect((await info(app, tokenFor())).status).toBe(200); + }); + + /** The whole point of binding a token to a file identifier. */ + it('does not open a different file', async () => { + const { app, tokenFor } = await setup(); + + expect((await info(app, tokenFor(), 'file-2')).status).toBe(401); + }); + + it('is refused when signed with another secret', async () => { + const { app, absolutePath } = await setup(); + const forged = jwt.sign({ fileId: 'file-1', absolutePath }, 'not-the-secret', { + algorithm: 'HS256', + }); + + expect((await info(app, forged)).status).toBe(401); + }); + + it('is refused when it has expired', async () => { + const { app, tokenFor } = await setup(); + + expect((await info(app, tokenFor({ exp: Math.floor(Date.now() / 1000) - 60 }))).status).toBe( + 401 + ); + }); + + /** + * A token of another kind must not stand in for this one. Nothing else signs + * with this secret today, which is what makes the older tokens below safe to + * keep accepting — if that changes, this is the test that should stop being + * true on its own. + */ + it('is refused when it declares another kind', async () => { + const { app, tokenFor } = await setup(); + + expect((await info(app, tokenFor({ typ: 'something-else' }))).status).toBe(401); + }); + + it('is accepted when it predates the kind claim', async () => { + const { app, tokenFor } = await setup(); + + expect((await info(app, tokenFor({ typ: undefined }))).status).toBe(200); + }); + + it('is refused when it names no file on disk', async () => { + const { app, tokenFor } = await setup(); + + expect((await info(app, tokenFor({ absolutePath: undefined }))).status).toBe(401); + }); + + it('is accepted from the Authorization header as well as the query', async () => { + const { app, tokenFor } = await setup(); + + const response = await request(app) + .get('/api/collabora/wopi/files/file-1') + .set('Authorization', `Bearer ${tokenFor()}`); + + expect(response.status).toBe(200); + }); +}); + +describe('what the editor is told about the file', () => { + it('names it', async () => { + const { app, tokenFor } = await setup(); + + expect((await info(app, tokenFor())).body.BaseFileName).toBe('quarterly.docx'); + }); + + it('reports its size', async () => { + const { app, tokenFor } = await setup(); + + expect((await info(app, tokenFor())).body.Size).toBe('original'.length); + }); + + /** The permission the token was issued with, not one the editor may assume. */ + it('says it may be written when the token says so', async () => { + const { app, tokenFor } = await setup(); + + expect((await info(app, tokenFor())).body.UserCanWrite).toBe(true); + }); + + it('says it may not when the token says not', async () => { + const { app, tokenFor } = await setup(); + + expect((await info(app, tokenFor({ canWrite: false }))).body.UserCanWrite).toBe(false); + }); + + /** + * Collabora reloads a document whose version changed underneath it, so the + * version has to move when the file does. + */ + it('changes its version when the file changes', async () => { + const { app, tokenFor, absolutePath } = await setup(); + const before = (await info(app, tokenFor())).body.Version; + + await fs.writeFile(absolutePath, Buffer.from('something rather longer')); + const after = (await info(app, tokenFor())).body.Version; + + expect(after).not.toBe(before); + }); + + it('keeps the same version while the file does not change', async () => { + const { app, tokenFor } = await setup(); + + const first = (await info(app, tokenFor())).body.Version; + const second = (await info(app, tokenFor())).body.Version; + + expect(second).toBe(first); + }); + + it('refuses to open a directory', async () => { + const { app, tokenFor } = await setup(); + const directory = path.join(currentEnv.tmpRoot, 'a-folder'); + await fs.mkdir(directory, { recursive: true }); + + const response = await info(app, tokenFor({ absolutePath: directory })); + + expect(response.status).toBe(400); + }); +}); + +describe('the locks that stop two people saving over each other', () => { + const lockRequest = (app, token, override, lockId, extra = {}) => { + const call = request(app) + .post('/api/collabora/wopi/files/file-1') + .query({ access_token: token }) + .set('X-WOPI-Override', override); + if (lockId) call.set('X-WOPI-Lock', lockId); + if (extra.oldLock) call.set('X-WOPI-OldLock', extra.oldLock); + return call; + }; + + it('grants a lock on a file nobody holds', async () => { + const { app, tokenFor } = await setup(); + + expect((await lockRequest(app, tokenFor(), 'LOCK', 'lock-a')).status).toBe(200); + }); + + it('refuses a second lock from somebody else', async () => { + const { app, tokenFor } = await setup(); + await lockRequest(app, tokenFor(), 'LOCK', 'lock-a'); + + expect((await lockRequest(app, tokenFor(), 'LOCK', 'lock-b')).status).toBe(409); + }); + + it('lets the holder take it again', async () => { + const { app, tokenFor } = await setup(); + await lockRequest(app, tokenFor(), 'LOCK', 'lock-a'); + + expect((await lockRequest(app, tokenFor(), 'LOCK', 'lock-a')).status).toBe(200); + }); + + it('says who holds it', async () => { + const { app, tokenFor } = await setup(); + await lockRequest(app, tokenFor(), 'LOCK', 'lock-a'); + + const response = await lockRequest(app, tokenFor(), 'GET_LOCK'); + + expect(response.headers['x-wopi-lock']).toBe('lock-a'); + }); + + it('releases it to its holder', async () => { + const { app, tokenFor } = await setup(); + await lockRequest(app, tokenFor(), 'LOCK', 'lock-a'); + + expect((await lockRequest(app, tokenFor(), 'UNLOCK', 'lock-a')).status).toBe(200); + }); + + it('does not release it to anybody else', async () => { + const { app, tokenFor } = await setup(); + await lockRequest(app, tokenFor(), 'LOCK', 'lock-a'); + + expect((await lockRequest(app, tokenFor(), 'UNLOCK', 'lock-b')).status).toBe(409); + }); + + it('lets its holder refresh it', async () => { + const { app, tokenFor } = await setup(); + await lockRequest(app, tokenFor(), 'LOCK', 'lock-a'); + + expect((await lockRequest(app, tokenFor(), 'REFRESH_LOCK', 'lock-a')).status).toBe(200); + }); + + it('refuses to refresh one that is not held', async () => { + const { app, tokenFor } = await setup(); + + expect((await lockRequest(app, tokenFor(), 'REFRESH_LOCK', 'lock-a')).status).toBe(409); + }); + + it('exchanges one lock for another for its holder', async () => { + const { app, tokenFor } = await setup(); + await lockRequest(app, tokenFor(), 'LOCK', 'lock-a'); + + const response = await lockRequest(app, tokenFor(), 'UNLOCK_AND_RELOCK', 'lock-b', { + oldLock: 'lock-a', + }); + + expect(response.status).toBe(200); + }); + + it('refuses an operation it does not know', async () => { + const { app, tokenFor } = await setup(); + + expect((await lockRequest(app, tokenFor(), 'SOMETHING_ELSE', 'lock-a')).status).toBe(400); + }); + + it('refuses one with no operation named at all', async () => { + const { app, tokenFor } = await setup(); + + const response = await request(app) + .post('/api/collabora/wopi/files/file-1') + .query({ access_token: tokenFor() }); + + expect(response.status).toBe(400); + }); +}); diff --git a/backend/tests/routes/command-argument-safety.test.js b/backend/tests/routes/command-argument-safety.test.js new file mode 100644 index 000000000..81e8654c6 --- /dev/null +++ b/backend/tests/routes/command-argument-safety.test.js @@ -0,0 +1,207 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Regression tests for user input that reaches an external command. + * + * Search terms, account names and file paths are handed to ripgrep, chmod and + * chown. They must always stay operands: a value starting with "-" must never + * be read as an option (ripgrep's --pre runs an arbitrary command per file), + * and nothing may be interpolated into a shell string. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const createSearchApp = async () => { + const envContext = await setupTestEnv({ + tag: 'search-argument-safety-', + env: { SEARCH_RIPGREP: 'true' }, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/routes/search', + 'src/middleware/errorHandler', + 'src/services/accessManager', + 'src/services/settingsService', + ], + }); + + const searchRoutes = envContext.requireFresh('src/routes/search'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const app = createTestApp({ + router: searchRoutes, + mountPath: '/api', + user: { id: 'admin', roles: ['admin'] }, + errorHandler, + }); + + return { envContext, app }; +}; + +describe('Search term argument safety', () => { + it('treats a term starting with a dash as a literal pattern, not a ripgrep option', async () => { + const { envContext, app } = await createSearchApp(); + currentEnv = envContext; + + // The needle is also a valid ripgrep flag. Without the `--` separator the + // process would consume it as an option instead of searching for it. + await fs.writeFile( + path.join(envContext.volumeDir, 'flagged-content.txt'), + 'config: --pre=whoami\n' + ); + await fs.writeFile(path.join(envContext.volumeDir, 'other.txt'), 'nothing to see\n'); + + const response = await request(app).get('/api/search').query({ q: '--pre=whoami' }); + + expect(response.status).toBe(200); + const names = (response.body.items || []).map((item) => item.name); + expect(names).toContain('flagged-content.txt'); + expect(names).not.toContain('other.txt'); + }); +}); + +/** + * The argument list, checked directly. + * + * The end-to-end test below only exercises ripgrep when ripgrep is installed: + * without it the route falls back to a JavaScript scan, passes, and proves + * nothing about the hardening. This one holds wherever it runs. + */ +describe('Content search arguments', () => { + it('puts every search term behind the -- separator', async () => { + const env = await setupTestEnv({ + tag: 'search-args-', + env: { SEARCH_RIPGREP: 'true' }, + modules: ['src/config/env', 'src/config/index', 'src/routes/search'], + }); + currentEnv = env; + + const { buildContentSearchArgs } = env.requireFresh('src/routes/search'); + + for (const term of ['--pre=whoami', '-abc', '-n', 'ordinary']) { + const args = buildContentSearchArgs(term, ['-g', '!.git']); + // The last three arguments are exactly: separator, term, path. Checking + // indexOf would match the wrong slot for a term like "-n", which is also + // a legitimate flag earlier in the list. + expect(args.slice(-3)).toEqual(['--', term, '.']); + } + }); + + it('keeps the separator when a file-size ceiling is prepended', async () => { + const env = await setupTestEnv({ + tag: 'search-args-limit-', + env: { SEARCH_RIPGREP: 'true', SEARCH_MAX_FILESIZE: '5M' }, + modules: ['src/config/env', 'src/config/index', 'src/routes/search'], + }); + currentEnv = env; + + const { buildContentSearchArgs } = env.requireFresh('src/routes/search'); + const args = buildContentSearchArgs('--pre=whoami', [], '5M'); + + expect(args[0]).toBe('--max-filesize'); + expect(args.slice(-3)).toEqual(['--', '--pre=whoami', '.']); + }); +}); + +const createPermissionsApp = async () => { + const envContext = await setupTestEnv({ + tag: 'permissions-argument-safety-', + modules: [ + 'src/config/env', + 'src/config/index', + 'src/utils/pathUtils', + 'src/routes/permissions', + 'src/middleware/errorHandler', + 'src/services/accessManager', + 'src/services/authorizationService', + 'src/services/settingsService', + ], + }); + + const permissionsRoutes = envContext.requireFresh('src/routes/permissions'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const app = createTestApp({ + router: permissionsRoutes, + mountPath: '/api', + user: { id: 'admin', roles: ['admin'] }, + errorHandler, + }); + + return { envContext, app }; +}; + +describe('Ownership change input validation', () => { + it('rejects owner and group names that are not plain account names', async () => { + const { envContext, app } = await createPermissionsApp(); + currentEnv = envContext; + + await fs.writeFile(path.join(envContext.volumeDir, 'target.txt'), 'content'); + + const injected = await request(app) + .post('/api/permissions/chown') + .send({ path: 'target.txt', owner: 'root"; id > /tmp/pwned; echo "' }); + expect(injected.status).toBe(400); + + const optionLike = await request(app) + .post('/api/permissions/chown') + .send({ path: 'target.txt', group: '--reference=/etc/shadow' }); + expect(optionLike.status).toBe(400); + + const substituted = await request(app) + .post('/api/permissions/chown') + .send({ path: 'target.txt', owner: '$(whoami)' }); + expect(substituted.status).toBe(400); + + // The file must be untouched by the rejected attempts. + await expect(fs.readFile(path.join(envContext.volumeDir, 'target.txt'), 'utf-8')).resolves.toBe( + 'content' + ); + }); + + it('accepts a well-formed account name', async () => { + const { envContext, app } = await createPermissionsApp(); + currentEnv = envContext; + + await fs.writeFile(path.join(envContext.volumeDir, 'valid.txt'), 'content'); + + const response = await request(app) + .post('/api/permissions/chown') + .send({ path: 'valid.txt', owner: 'appuser' }); + + // Changing ownership needs privileges we do not have in CI, so the request + // is allowed through validation and fails later (403) instead of 400. + expect(response.status).not.toBe(400); + }); +}); + +describe('Recursive chmod argument safety', () => { + it('applies permissions to a directory whose name contains shell metacharacters', async () => { + const { envContext, app } = await createPermissionsApp(); + currentEnv = envContext; + + // Names like this can exist on a mounted host volume even though the app + // would not create them itself. + const trickyName = 'dir";id;#'; + const trickyDir = path.join(envContext.volumeDir, trickyName); + await fs.mkdir(trickyDir, { recursive: true }); + await fs.writeFile(path.join(trickyDir, 'child.txt'), 'child'); + + const response = await request(app) + .post('/api/permissions/chmod') + .send({ path: trickyName, mode: '755', recursive: true }); + + expect(response.status).toBe(200); + const childStats = await fs.stat(path.join(trickyDir, 'child.txt')); + expect(childStats.mode & 0o777).toBe(0o755); + }); +}); diff --git a/backend/tests/routes/direct-upload.test.js b/backend/tests/routes/direct-upload.test.js new file mode 100644 index 000000000..54534542e --- /dev/null +++ b/backend/tests/routes/direct-upload.test.js @@ -0,0 +1,266 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The default upload path — `UPLOAD_CHUNKED_ENABLED` is false out of the box, + * so this is what nearly every deployment runs, and until now it had no test of + * its own. What it guards: an upload the volume cannot hold, and the remains of + * one that was killed. + */ + +let envContext; + +const startServer = (server) => + new Promise((resolve) => { + server.listen(0, () => resolve(`http://127.0.0.1:${server.address().port}`)); + }); + +const closeServer = (server) => + new Promise((resolve, reject) => { + server.closeAllConnections?.(); + server.close((err) => (err ? reject(err) : resolve())); + }); + +const buildApp = () => { + const express = require('express'); + const http = require('node:http'); + const uploadRoutes = envContext.requireFresh('src/routes/upload'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: 'admin', email: 'admin@example.com', roles: ['admin'] }; + next(); + }); + app.use('/api', uploadRoutes); + app.use(errorHandler); + return http.createServer(app); +}; + +/** An authorised destination, and a server pointing at it. */ +const build = async (env = {}) => { + envContext = await setupTestEnv({ tag: 'direct-upload-test-', env }); + const destination = path.join(envContext.volumeDir, 'Nvm'); + await fs.mkdir(destination, { recursive: true }); + return { destination, server: buildApp() }; +}; + +const upload = (baseUrl, { name = 'hello.txt', content = 'hello' } = {}) => + request(baseUrl) + .post('/api/upload') + .query({ uploadTo: 'Nvm', relativePath: name }) + .attach('filedata', Buffer.from(content), name); + +/** + * The same upload, sent without a Content-Length: node writes the body chunked + * when it is not told how long it is, which is what a client streaming a file + * does and what supertest never does. + */ +const uploadWithoutContentLength = (baseUrl, name) => { + const http = require('node:http'); + const boundary = 'direct-upload-no-length'; + const { port } = new URL(baseUrl); + return new Promise((resolve, reject) => { + const req = http.request( + { + host: '127.0.0.1', + port, + method: 'POST', + path: `/api/upload?uploadTo=Nvm&relativePath=${encodeURIComponent(name)}`, + headers: { 'Content-Type': `multipart/form-data; boundary=${boundary}` }, + }, + (res) => { + res.resume(); + res.on('end', () => resolve(res.statusCode)); + } + ); + req.on('error', reject); + req.write( + `--${boundary}\r\nContent-Disposition: form-data; name="filedata"; filename="${name}"\r\n` + + 'Content-Type: text/plain\r\n\r\n' + ); + req.write('a few bytes'); + req.end(`\r\n--${boundary}--\r\n`); + }); +}; + +const exists = async (target) => + fs + .access(target) + .then(() => true) + .catch(() => false); + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('a direct upload', () => { + it('lands in the authorised folder', async () => { + const { destination, server } = await build(); + const baseUrl = await startServer(server); + + try { + const response = await upload(baseUrl, { name: 'hello.txt', content: 'hello there' }); + + expect(response.status).toBe(200); + expect(await fs.readFile(path.join(destination, 'hello.txt'), 'utf8')).toBe('hello there'); + // The temporary file it was written through is gone, whatever its name. + expect(await fs.readdir(destination)).toEqual(['hello.txt']); + } finally { + await closeServer(server); + } + }); + + // A full volume takes the database down with it where `/config` shares the + // filesystem, so the refusal has to happen before anything is written. + it('is refused with 507 when the volume cannot hold it', async () => { + const { destination, server } = await build({ UPLOAD_STORAGE_RESERVE: '900T' }); + const baseUrl = await startServer(server); + + try { + const response = await upload(baseUrl, { name: 'too-big.bin' }); + + expect(response.status).toBe(507); + expect(await exists(path.join(destination, 'too-big.bin'))).toBe(false); + expect(await exists(path.join(destination, 'too-big.bin.uploading'))).toBe(false); + } finally { + await closeServer(server); + } + }); + + /** + * An upload that announces no size at all. + * + * The only measure of what is coming is Content-Length, and a request sent + * chunked has none — what an API client streaming a file does. The guard + * takes a number and was handed nothing, so it returned without looking: + * this upload landed on a volume the one above it was refused on. Zero is + * what is honestly known about what is coming, and the reserve is still held + * free, which is the part that keeps the database alive. + */ + it('is refused when it announces no size and the reserve is already gone', async () => { + const { destination, server } = await build({ UPLOAD_STORAGE_RESERVE: '900T' }); + const baseUrl = await startServer(server); + + try { + const status = await uploadWithoutContentLength(baseUrl, 'streamed.txt'); + + expect(status).toBe(507); + expect(await fs.readdir(destination)).toEqual([]); + } finally { + await closeServer(server); + } + }); + + /** And one that announces nothing still lands where there is room for it. */ + it('is accepted when it announces no size and there is room', async () => { + const { destination, server } = await build(); + const baseUrl = await startServer(server); + + try { + const status = await uploadWithoutContentLength(baseUrl, 'streamed.txt'); + + expect(status).toBe(200); + expect(await fs.readdir(destination)).toEqual(['streamed.txt']); + } finally { + await closeServer(server); + } + }); + + it('clears the remains of a killed upload from the folder it writes to', async () => { + const { destination, server } = await build(); + const stale = path.join(destination, 'holiday.mp4.uploading'); + await fs.writeFile(stale, 'half a film'); + const twoDaysAgo = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000); + await fs.utimes(stale, twoDaysAgo, twoDaysAgo); + + const recent = path.join(destination, 'still-going.mkv.uploading'); + await fs.writeFile(recent, 'in flight'); + + const baseUrl = await startServer(server); + + try { + const response = await upload(baseUrl, { name: 'notes.txt' }); + + expect(response.status).toBe(200); + expect(await exists(stale)).toBe(false); + // Another upload writing right now is not remains. + expect(await exists(recent)).toBe(true); + } finally { + await closeServer(server); + } + }); +}); + +/** + * What multer refuses is the request's doing, not the server's. Its errors carry + * no status, and every one of them used to answer 500 and log a server error. + * The limit is named along with the setting that raises it, because "File too + * large" leaves whoever reads it nowhere to go. + */ +describe('a direct upload over the limits', () => { + it('is refused with 413 when the file is larger than MAX_DIRECT_UPLOAD_SIZE, and leaves nothing', async () => { + const { destination, server } = await build({ MAX_DIRECT_UPLOAD_SIZE: '1K' }); + const baseUrl = await startServer(server); + + try { + const response = await upload(baseUrl, { name: 'big.bin', content: 'x'.repeat(4096) }); + + expect(response.status).toBe(413); + expect(response.body.error.message).toBe( + 'This file is larger than the 1 KB a direct upload accepts. Use chunked uploads, or raise MAX_DIRECT_UPLOAD_SIZE.' + ); + expect(await fs.readdir(destination)).toEqual([]); + } finally { + await closeServer(server); + } + }); + + it('is refused with 413 when one request carries more than MAX_FILES_PER_UPLOAD files', async () => { + const { destination, server } = await build({ MAX_FILES_PER_UPLOAD: '2' }); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl) + .post('/api/upload') + .query({ uploadTo: 'Nvm' }) + .attach('filedata', Buffer.from('one'), 'one.txt') + .attach('filedata', Buffer.from('two'), 'two.txt') + .attach('filedata', Buffer.from('three'), 'three.txt'); + + expect(response.status).toBe(413); + expect(response.body.error.message).toBe( + 'One upload request takes at most 2 files. Send the others in another, or raise MAX_FILES_PER_UPLOAD.' + ); + // The files that arrived before the refusal are taken back with it. + expect(await fs.readdir(destination)).toEqual([]); + } finally { + await closeServer(server); + } + }); + + it('is refused with 400 when the file comes in a field the route does not read', async () => { + const { destination, server } = await build(); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl) + .post('/api/upload') + .query({ uploadTo: 'Nvm', relativePath: 'stray.txt' }) + .attach('attachment', Buffer.from('stray'), 'stray.txt'); + + expect(response.status).toBe(400); + expect(response.body.error.message).toBe( + 'A file was sent in a field this request does not take.' + ); + expect(await fs.readdir(destination)).toEqual([]); + } finally { + await closeServer(server); + } + }); +}); diff --git a/backend/tests/routes/download.test.js b/backend/tests/routes/download.test.js new file mode 100644 index 000000000..3e79768e0 --- /dev/null +++ b/backend/tests/routes/download.test.js @@ -0,0 +1,218 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Getting bytes back out. + * + * The route had 19.8% coverage while deciding two things that matter: whether + * the caller is allowed the file at all, and what the browser will call what it + * receives. The second sounds cosmetic and is not — the name is built from a + * logical path, a base path and a share prefix, and getting it wrong hands + * somebody a file called `share` or a zip named after the wrong folder. + * + * Downloading is also its own permission. A share can be readable and still + * refuse downloads, which is the difference between "look at this" and "take a + * copy", and nothing else in the suite covered that branch. + */ + +const setup = async ({ user = { id: 'admin', roles: ['admin'] }, env = {} } = {}) => { + const envContext = await setupTestEnv({ + tag: 'download-test-', + env, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/routes/files/download', + 'src/middleware/errorHandler', + 'src/services/accessManager', + 'src/services/settingsService', + ], + }); + + const volume = envContext.volumeDir; + await fs.mkdir(path.join(volume, 'Docs/2026'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Docs/report.txt'), 'annual report'); + await fs.writeFile(path.join(volume, 'Docs/notes.md'), '# notes'); + await fs.writeFile(path.join(volume, 'Docs/.env'), 'SECRET=1'); + await fs.writeFile(path.join(volume, 'Docs/2026/q1.txt'), 'first quarter'); + + const routes = envContext.requireFresh('src/routes/files/download'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const { getDb } = envContext.requireFresh('src/services/db'); + const db = await getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ).run('admin', 'admin@example.com', 1, 'admin', 'Admin', '["admin"]', now, now); + + const app = createTestApp({ router: routes, mountPath: '/api', user, errorHandler }); + return { envContext, app }; +}; + +let ctx; +afterEach(async () => { + if (ctx) { + await ctx.envContext.cleanup(); + ctx = null; + } +}); + +const post = async (body, options) => { + ctx = await setup(options); + return request(ctx.app).post('/api/download').send(body); +}; + +/** supertest leaves an unknown content type as a string; a zip needs its bytes. */ +const binary = (res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(Buffer.from(chunk))); + res.on('end', () => callback(null, Buffer.concat(chunks))); +}; + +const postBinary = async (body, options) => { + ctx = await setup(options); + return request(ctx.app).post('/api/download').send(body).buffer().parse(binary); +}; + +describe('what it refuses before reading anything', () => { + it('asks for a path when the body carries none', async () => { + const response = await post({}); + + expect(response.status).toBe(400); + }); + + it('asks again when every path normalises away to nothing', async () => { + const response = await post({ paths: ['', ' ', null] }); + + expect(response.status).toBe(400); + }); + + it('refuses a path that climbs out of the volume', async () => { + const response = await post({ paths: ['Docs/../../etc/passwd'] }); + + expect(response.status).toBeGreaterThanOrEqual(400); + }); +}); + +describe('one file', () => { + it('sends it under its own name', async () => { + const response = await post({ paths: ['Docs/report.txt'] }); + + expect(response.status).toBe(200); + expect(response.headers['content-disposition']).toContain('report.txt'); + expect(response.text).toBe('annual report'); + }); + + /** + * Express refuses dotfiles by default, which would make `.env`, `.gitignore` + * and every dotfile in a repository undownloadable with no message saying so. + */ + it('sends a dotfile, which Express would otherwise refuse', async () => { + const response = await postBinary({ paths: ['Docs/.env'] }); + + expect(response.status).toBe(200); + expect(response.body.toString('utf8')).toBe('SECRET=1'); + }); + + it('names it from the base path when one is given', async () => { + const response = await post({ paths: ['Docs/2026/q1.txt'], basePath: 'Docs' }); + + expect(response.status).toBe(200); + expect(response.headers['content-disposition']).toContain('q1.txt'); + }); + + it('accepts the singular `path` field as well as `paths`', async () => { + const response = await post({ path: 'Docs/report.txt' }); + + expect(response.status).toBe(200); + expect(response.text).toBe('annual report'); + }); +}); + +describe('when it has to build a zip', () => { + const isZip = (response) => { + expect(response.headers['content-type']).toContain('zip'); + // Local file header — proves an archive came back rather than a file. + expect(response.body.subarray(0, 2).toString('latin1')).toBe('PK'); + }; + + it('archives two files rather than sending one', async () => { + const response = await postBinary({ + paths: ['Docs/report.txt', 'Docs/notes.md'], + basePath: 'Docs', + }); + + expect(response.status).toBe(200); + isZip(response); + }); + + it('archives a single directory', async () => { + const response = await postBinary({ paths: ['Docs/2026'], basePath: 'Docs' }); + + expect(response.status).toBe(200); + isZip(response); + expect(response.headers['content-disposition']).toContain('2026.zip'); + }); + + it('names a multi-item archive after the folder they came from', async () => { + const response = await post({ paths: ['Docs/report.txt', 'Docs/notes.md'], basePath: 'Docs' }); + + expect(response.headers['content-disposition']).toContain('Docs.zip'); + }); + + it('falls back to download.zip when there is no base path to name it after', async () => { + const response = await post({ paths: ['Docs/report.txt', 'Docs/notes.md'] }); + + expect(response.headers['content-disposition']).toContain('download.zip'); + }); + + /** + * The same file twice is one file. Without the dedupe the archive carries two + * entries of the same name, which some extractors silently collapse and + * others refuse. + */ + it('treats the same path listed twice as one', async () => { + const response = await post({ paths: ['Docs/report.txt', 'Docs/report.txt'] }); + + expect(response.status).toBe(200); + // One target left after the dedupe, so this is a plain file, not a zip. + expect(response.headers['content-type']).not.toContain('zip'); + expect(response.text).toBe('annual report'); + }); +}); + +describe('a caller who may not reach the file', () => { + /** + * `USER_VOLUMES=true` with an account assigned no volume: the access check is + * the only thing that can refuse here. + * + * Note what this does NOT cover. The route also tests `accessInfo.canDownload`, + * and no test can reach that branch, because `canDownload` is set to `true` + * everywhere it is set except in the denied-access object — where `canAccess` + * is already false and answers first. Removing the check from the route breaks + * nothing, which is how it was found. It is recorded in TODO.md rather than + * asserted here: a test that claims to cover it would be claiming something + * untrue. + */ + it('is refused, and told it is a permission problem', async () => { + const response = await post( + { paths: ['Docs/report.txt'] }, + { user: { id: 'nobody', roles: ['user'] }, env: { USER_VOLUMES: 'true' } } + ); + + expect(response.status).toBe(403); + }); + + it('is refused even when one path in the list is allowed', async () => { + const response = await post( + { paths: ['Docs/report.txt', 'Docs/notes.md'] }, + { user: { id: 'nobody', roles: ['user'] }, env: { USER_VOLUMES: 'true' } } + ); + + expect(response.status).toBe(403); + }); +}); diff --git a/backend/tests/routes/editor-caching.test.js b/backend/tests/routes/editor-caching.test.js new file mode 100644 index 000000000..d60b2b921 --- /dev/null +++ b/backend/tests/routes/editor-caching.test.js @@ -0,0 +1,258 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import express from 'express'; +import request from 'supertest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Opening the editor from the Markdown preview downloaded the whole file a + * second time: both asked for it with a POST, and a POST is never kept. A GET + * is kept by the browser and asked again with the ETag it came with, and a file + * that has not changed is answered 304 from its metadata alone. + * + * What must never happen is the other way round: a 304 for a file that did + * change, or for someone who may no longer read it. + */ + +let envContext; +let users; +let app; + +const load = (relative) => require(modulePath(relative)); + +const volume = (...segments) => path.join(envContext.volumeDir, ...segments); + +const buildApp = () => { + const application = express(); + application.use(express.json()); + application.use((req, _res, next) => { + const who = req.get('x-test-user'); + if (who) req.user = users[who]; + next(); + }); + application.use('/api', load('src/routes/editor')); + application.use(load('src/middleware/errorHandler').errorHandler); + return application; +}; + +const FILE = 'Projects/notes.md'; +const ORIGINAL = '# Notes\n\nThe first version of these notes.\n'; + +const open = (who, filePath = FILE, headers = {}) => + request(app) + .get(`/api/editor?path=${encodeURIComponent(filePath)}`) + .set('x-test-user', who) + .set(headers); + +const revalidate = (who, etag, filePath = FILE) => open(who, filePath, { 'If-None-Match': etag }); + +const save = (who, content, filePath = FILE) => + request(app).put('/api/editor').set('x-test-user', who).send({ path: filePath, content }); + +const setRules = (rules) => + load('src/services/settingsService').setSystemSetting('system', 'access', { rules }); + +/** + * Filesystems keep times at the granularity of their clock tick — a few + * milliseconds on Linux — so two changes inside one tick can carry the same + * time. The changes below are spaced past that. + */ +const tick = () => new Promise((resolve) => setTimeout(resolve, 30)); + +/** Overwrite bytes of the file where it is: same inode, same size. */ +const rewriteInPlace = async (absolutePath, content) => { + const handle = await fs.open(absolutePath, 'r+'); + try { + await handle.write(content, 0, 'utf8'); + } finally { + await handle.close(); + } +}; + +beforeEach(async () => { + envContext = await setupTestEnv({ tag: 'editor-caching-' }); + const usersService = load('src/services/users'); + users = { + alice: await usersService.createLocalUser({ + email: 'alice@example.com', + username: 'alice', + displayName: 'Alice', + password: 'secret123', + roles: ['user'], + }), + }; + app = buildApp(); + await fs.mkdir(volume('Projects'), { recursive: true }); + await fs.writeFile(volume(FILE), ORIGINAL); +}); + +afterEach(async () => { + vi.restoreAllMocks(); + load('src/services/trash/maintenance').stop?.(); + await envContext.cleanup(); +}); + +describe('reading a file with a GET', () => { + it('answers exactly what the POST answers', async () => { + const byGet = await open('alice'); + const byPost = await request(app) + .post('/api/editor') + .set('x-test-user', 'alice') + .send({ path: FILE }); + + expect(byGet.status).toBe(200); + expect(byGet.body).toEqual({ content: ORIGINAL }); + expect(byGet.text).toBe(byPost.text); + expect(byGet.headers['content-type']).toBe(byPost.headers['content-type']); + }); + + it('is kept by the browser but checked every time, under an identity that holds still', async () => { + const first = await open('alice'); + const second = await open('alice'); + + expect(first.headers['cache-control']).toBe('private, no-cache'); + expect(first.headers.etag).toMatch(/^W\/".+"$/); + expect(second.headers.etag).toBe(first.headers.etag); + }); + + it('answers 304 to that identity without reading the file', async () => { + const { etag } = (await open('alice')).headers; + const readFile = vi.spyOn(fs, 'readFile'); + + const response = await revalidate('alice', etag); + const inAList = await revalidate('alice', `"something-else", ${etag}`); + + expect(response.status).toBe(304); + expect(response.text).toBeFalsy(); + expect(response.headers.etag).toBe(etag); + expect(response.headers.vary).toMatch(/accept-encoding/i); + expect(inAList.status).toBe(304); + const readsOfTheFile = readFile.mock.calls.filter( + ([target]) => String(target) === volume(FILE) + ); + expect(readsOfTheFile).toEqual([]); + }); + + it('answers the raw text 304 the same way', async () => { + const target = `/api/raw?path=${encodeURIComponent(FILE)}`; + const first = await request(app).get(target).set('x-test-user', 'alice'); + + const again = await request(app) + .get(target) + .set('x-test-user', 'alice') + .set('If-None-Match', first.headers.etag); + + expect(first.status).toBe(200); + expect(first.text).toBe(ORIGINAL); + expect(again.status).toBe(304); + }); +}); + +describe('a file that changed is read again', () => { + it('after a save through the editor, whose answer carries the new identity', async () => { + const { etag } = (await open('alice')).headers; + // The same length, so neither the size nor the content length tells them apart. + const rewritten = ORIGINAL.replace('first', 'final'); + await tick(); + + const saved = await save('alice', rewritten); + const response = await revalidate('alice', etag); + + expect(saved.status).toBe(200); + expect(response.status).toBe(200); + expect(response.body).toEqual({ content: rewritten }); + expect(response.headers.etag).not.toBe(etag); + expect(saved.headers.etag).toBe(response.headers.etag); + expect((await revalidate('alice', saved.headers.etag)).status).toBe(304); + }); + + it('after another file is renamed over it, with the same size and times', async () => { + const when = new Date('2026-01-02T03:04:05Z'); + await fs.utimes(volume(FILE), when, when); + const { etag } = (await open('alice')).headers; + const replacement = ORIGINAL.replace('first', 'other'); + await fs.writeFile(volume('Projects/incoming.md'), replacement); + await fs.utimes(volume('Projects/incoming.md'), when, when); + await tick(); + + await fs.rename(volume('Projects/incoming.md'), volume(FILE)); + const response = await revalidate('alice', etag); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ content: replacement }); + }); + + it('after a write in place of the same length, at a different time', async () => { + const { etag } = (await open('alice')).headers; + const rewritten = ORIGINAL.replace('first', 'fixed'); + await tick(); + + await rewriteInPlace(volume(FILE), rewritten); + const response = await revalidate('alice', etag); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ content: rewritten }); + }); + + it('after an append', async () => { + const { etag } = (await open('alice')).headers; + await tick(); + + await fs.appendFile(volume(FILE), 'One more line.\n'); + const response = await revalidate('alice', etag); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ content: `${ORIGINAL}One more line.\n` }); + }); + + /** + * `cp -p`, `rsync --inplace -t` and an archive extracted over the file all + * write in place and put the modification time back. Nothing can put back the + * change time. + */ + it('after a write in place that put the modification time back', async () => { + const when = new Date('2026-01-02T03:04:05Z'); + await fs.utimes(volume(FILE), when, when); + const { etag } = (await open('alice')).headers; + const rewritten = ORIGINAL.replace('first', 'fixed'); + await tick(); + + await rewriteInPlace(volume(FILE), rewritten); + await fs.utimes(volume(FILE), when, when); + const response = await revalidate('alice', etag); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ content: rewritten }); + }); +}); + +describe('what a 304 must never stand in for', () => { + it('a refusal: someone who may no longer read the file is refused, whatever they hold', async () => { + const { etag } = (await open('alice')).headers; + await setRules([{ path: 'Projects', recursive: true, permissions: 'hidden' }]); + + const response = await revalidate('alice', etag); + + expect(response.status).toBe(403); + expect(response.headers.etag).not.toBe(etag); + expect(response.headers['cache-control']).not.toBe('private, no-cache'); + }); + + /** + * An answer carrying an ETag and `private` may be kept by the browser, error + * or not, and revalidated like any other — a 304 would then keep the error. + */ + it('an error: a file that cannot be opened is answered without an identity', async () => { + await fs.writeFile(volume('Projects/photo.md'), Buffer.from([0, 159, 146, 150, 0, 0, 1, 2])); + + const response = await open('alice', 'Projects/photo.md'); + + // Express still tags the JSON of the error with a hash of its own bytes; + // what must be absent is the file's identity, and the permission to keep. + expect(response.status).toBe(415); + expect(response.headers.etag ?? '').not.toMatch(/-t\d+"$/); + expect(response.headers['cache-control']).toBeUndefined(); + }); +}); diff --git a/backend/tests/routes/editor-encoding.test.js b/backend/tests/routes/editor-encoding.test.js new file mode 100644 index 000000000..df5e16903 --- /dev/null +++ b/backend/tests/routes/editor-encoding.test.js @@ -0,0 +1,251 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A text file the editor called binary. + * + * A 3.5 MB `.txt` was refused with "this file appears to be binary and cannot + * be opened in the text editor". It was UTF-16 — every ASCII character stored + * with a zero byte beside it, and a zero byte is what the binary test looks + * for. PowerShell wrote UTF-16LE from `Out-File` until PowerShell 6 and Notepad + * still offers it as "Unicode", so this is what a Windows log or export + * ordinarily is. + * + * Driven through the route rather than the detector, because the round trip is + * the thing: open it, save it, and find the file still written the way whatever + * produced it will read it back. + */ + +let envContext; + +const startServer = (server) => + new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => resolve(`http://127.0.0.1:${server.address().port}`)); + }); + +const closeServer = (server) => + new Promise((resolve, reject) => { + server.closeAllConnections?.(); + server.close((err) => (err ? reject(err) : resolve())); + }); + +const build = async (env = {}) => { + envContext = await setupTestEnv({ tag: 'editor-encoding-test-', env }); + const destination = path.join(envContext.volumeDir, 'Nvm'); + await fs.mkdir(destination, { recursive: true }); + + const express = require('express'); + const http = require('node:http'); + const { uploads } = envContext.requireFresh('src/config/index'); + const editorRoutes = envContext.requireFresh('src/routes/editor'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + + const app = express(); + app.use(express.json({ limit: uploads.maxJsonBodyBytes })); + app.use((req, _res, next) => { + req.user = { id: 'admin', email: 'admin@example.com', roles: ['admin'] }; + next(); + }); + app.use('/api', editorRoutes); + app.use(errorHandler); + + return { destination, server: http.createServer(app) }; +}; + +const LOG = 'Nom de la machine : POSTE-042\r\nStatut : à jour\r\n'.repeat(30); + +const write = async (destination, name, buffer) => + fs.writeFile(path.join(destination, name), buffer); + +const utf16le = (text, { bom = true } = {}) => { + const body = Buffer.from(text, 'utf16le'); + return bom ? Buffer.concat([Buffer.from([0xff, 0xfe]), body]) : body; +}; + +const utf16be = (text, { bom = true } = {}) => { + const body = Buffer.from(text, 'utf16le').swap16(); + return bom ? Buffer.concat([Buffer.from([0xfe, 0xff]), body]) : body; +}; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('opening a text file that is not UTF-8', () => { + it('opens a UTF-16 file rather than calling it binary', async () => { + const { destination, server } = await build(); + await write(destination, 'rapport.txt', utf16le(LOG)); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl).post('/api/editor').send({ path: 'Nvm/rapport.txt' }); + + expect(response.status).toBe(200); + expect(response.body.content).toBe(LOG); + } finally { + await closeServer(server); + } + }); + + it('opens one written the other way round', async () => { + const { destination, server } = await build(); + await write(destination, 'rapport.txt', utf16be(LOG)); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl).post('/api/editor').send({ path: 'Nvm/rapport.txt' }); + + expect(response.status).toBe(200); + expect(response.body.content).toBe(LOG); + } finally { + await closeServer(server); + } + }); + + it('opens one with no mark to announce it', async () => { + const { destination, server } = await build(); + await write(destination, 'rapport.txt', utf16le(LOG, { bom: false })); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl).post('/api/editor').send({ path: 'Nvm/rapport.txt' }); + + expect(response.status).toBe(200); + expect(response.body.content).toBe(LOG); + } finally { + await closeServer(server); + } + }); + + it('serves it as text at the raw endpoint too', async () => { + const { destination, server } = await build(); + await write(destination, 'rapport.txt', utf16le(LOG)); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl).get('/api/raw').query({ path: 'Nvm/rapport.txt' }); + + expect(response.status).toBe(200); + expect(response.text).toBe(LOG); + } finally { + await closeServer(server); + } + }); + + /** Something genuinely binary is still refused, and still says so. */ + it('still refuses a file that really is binary', async () => { + const { destination, server } = await build(); + await write(destination, 'image.txt', Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0x1a, 0x0a])); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl).post('/api/editor').send({ path: 'Nvm/image.txt' }); + + expect(response.status).toBe(415); + expect(response.body.error.message).toMatch(/binary/i); + } finally { + await closeServer(server); + } + }); +}); + +describe('saving a text file that is not UTF-8', () => { + /** + * The file keeps the encoding it had. Saving it back as UTF-8 would read + * perfectly well here and break whatever wrote it. + */ + it('writes a UTF-16 file back as UTF-16', async () => { + const { destination, server } = await build(); + const target = path.join(destination, 'rapport.txt'); + await write(destination, 'rapport.txt', utf16le(LOG)); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl) + .put('/api/editor') + .send({ path: 'Nvm/rapport.txt', content: `${LOG}Statut : terminé\r\n` }); + + expect(response.status).toBe(200); + const written = await fs.readFile(target); + expect(written.subarray(0, 2)).toEqual(Buffer.from([0xff, 0xfe])); + expect(written.subarray(2).toString('utf16le')).toBe(`${LOG}Statut : terminé\r\n`); + } finally { + await closeServer(server); + } + }); + + it('keeps a big-endian file big-endian', async () => { + const { destination, server } = await build(); + const target = path.join(destination, 'rapport.txt'); + await write(destination, 'rapport.txt', utf16be(LOG)); + const baseUrl = await startServer(server); + + try { + await request(baseUrl).put('/api/editor').send({ path: 'Nvm/rapport.txt', content: 'Fini' }); + + const written = await fs.readFile(target); + expect(written).toEqual(utf16be('Fini')); + } finally { + await closeServer(server); + } + }); + + it('keeps a UTF-8 file without a mark exactly that', async () => { + const { destination, server } = await build(); + const target = path.join(destination, 'notes.md'); + await write(destination, 'notes.md', Buffer.from('# Notes\n', 'utf8')); + const baseUrl = await startServer(server); + + try { + await request(baseUrl) + .put('/api/editor') + .send({ path: 'Nvm/notes.md', content: '# Autres\n' }); + + expect(await fs.readFile(target)).toEqual(Buffer.from('# Autres\n', 'utf8')); + } finally { + await closeServer(server); + } + }); + + it('writes a file that did not exist in UTF-8', async () => { + const { destination, server } = await build(); + const baseUrl = await startServer(server); + + try { + await request(baseUrl) + .put('/api/editor') + .send({ path: 'Nvm/nouveau.md', content: 'Bonjour' }); + + expect(await fs.readFile(path.join(destination, 'nouveau.md'))).toEqual( + Buffer.from('Bonjour', 'utf8') + ); + } finally { + await closeServer(server); + } + }); + + /** + * The limit is about what lands on disk, and a UTF-16 file takes two bytes + * per character — so the same text is twice the file. + */ + it('measures the size against the bytes it is about to write', async () => { + const { destination, server } = await build({ EDITOR_MAX_FILESIZE: '4K' }); + await write(destination, 'rapport.txt', utf16le('court')); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl) + .put('/api/editor') + .send({ path: 'Nvm/rapport.txt', content: 'x'.repeat(3 * 1024) }); + + expect(response.status).toBe(400); + expect(response.body.error.message).toMatch(/too large to save/i); + } finally { + await closeServer(server); + } + }); +}); diff --git a/backend/tests/routes/editor-save.test.js b/backend/tests/routes/editor-save.test.js new file mode 100644 index 000000000..dd68b79ef --- /dev/null +++ b/backend/tests/routes/editor-save.test.js @@ -0,0 +1,131 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The editor refuses to open a file past `EDITOR_MAX_FILESIZE`, and used to + * save anything it was handed. Paste two megabytes into a small file, save, + * and the next attempt to open it answered "This file is too large to open in + * the text editor" — a file written by the editor that the editor would not + * take back. + */ + +let envContext; + +const startServer = (server) => + new Promise((resolve) => { + server.listen(0, () => resolve(`http://127.0.0.1:${server.address().port}`)); + }); + +const closeServer = (server) => + new Promise((resolve, reject) => { + server.closeAllConnections?.(); + server.close((err) => (err ? reject(err) : resolve())); + }); + +const build = async (env = {}) => { + envContext = await setupTestEnv({ tag: 'editor-save-test-', env }); + const destination = path.join(envContext.volumeDir, 'Nvm'); + await fs.mkdir(destination, { recursive: true }); + + const express = require('express'); + const http = require('node:http'); + const { uploads } = envContext.requireFresh('src/config/index'); + const editorRoutes = envContext.requireFresh('src/routes/editor'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + + const app = express(); + app.use(express.json({ limit: uploads.maxJsonBodyBytes })); + app.use((req, _res, next) => { + req.user = { id: 'admin', email: 'admin@example.com', roles: ['admin'] }; + next(); + }); + app.use('/api', editorRoutes); + app.use(errorHandler); + + return { destination, server: http.createServer(app) }; +}; + +const exists = async (target) => + fs + .access(target) + .then(() => true) + .catch(() => false); + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('saving from the text editor', () => { + it('writes what fits', async () => { + const { destination, server } = await build({ EDITOR_MAX_FILESIZE: '64K' }); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl) + .put('/api/editor') + .send({ path: 'Nvm/notes.md', content: '# Notes\n\nA short document.\n' }); + + expect(response.status).toBe(200); + expect(await fs.readFile(path.join(destination, 'notes.md'), 'utf8')).toContain('A short'); + } finally { + await closeServer(server); + } + }); + + it('refuses what it would not be able to open again', async () => { + const { destination, server } = await build({ EDITOR_MAX_FILESIZE: '4K' }); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl) + .put('/api/editor') + .send({ path: 'Nvm/big.md', content: 'x'.repeat(5 * 1024) }); + + expect(response.status).toBe(400); + expect(response.body.error.message).toMatch(/too large to save/i); + // Nothing written: the refusal comes before the file is touched. + expect(await exists(path.join(destination, 'big.md'))).toBe(false); + } finally { + await closeServer(server); + } + }); + + it('says nothing about size for a document at the limit', async () => { + const { server } = await build({ EDITOR_MAX_FILESIZE: '4K' }); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl) + .put('/api/editor') + .send({ path: 'Nvm/exact.md', content: 'x'.repeat(4 * 1024) }); + + expect(response.status).toBe(200); + } finally { + await closeServer(server); + } + }); + + // What the reporter of nxzai#368 saw, and could do nothing with. The body + // limit is the smallest the pair allows, so the request is refused before it + // ever reaches the route. + it('tells a request that is too big which setting governs it', async () => { + const { server } = await build({ EDITOR_MAX_FILESIZE: '64K', MAX_JSON_BODY_SIZE: '1M' }); + const baseUrl = await startServer(server); + + try { + const response = await request(baseUrl) + .put('/api/editor') + .send({ path: 'Nvm/huge.md', content: 'x'.repeat(3 * 1024 * 1024) }); + + expect(response.status).toBe(413); + expect(response.body.error.message).toMatch(/MAX_JSON_BODY_SIZE/); + expect(response.body.error.message).not.toMatch(/request entity too large/i); + } finally { + await closeServer(server); + } + }); +}); diff --git a/backend/tests/routes/favorites.test.js b/backend/tests/routes/favorites.test.js new file mode 100644 index 000000000..cce6d6c20 --- /dev/null +++ b/backend/tests/routes/favorites.test.js @@ -0,0 +1,203 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import nodeFs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Favorites belong to one account. Every operation takes the caller's id from + * the session and never from the request, which is the only thing standing + * between two accounts that both hold an id they did not create. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'favorites-' }); + const dbService = currentEnv.requireFresh('src/services/db'); + const db = await dbService.getDb(); + const now = new Date().toISOString(); + for (const [id, username] of [ + ['alice', 'alice'], + ['bob', 'bob'], + ]) { + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, '["user"]', ?, ?)` + ).run(id, `${username}@example.com`, username, username, now, now); + } + // A favorite points at somewhere that exists; the service checks. + for (const folder of ['Docs/alice', 'Docs/bob', 'Docs/one', 'Docs/two', 'Docs/notes']) { + await nodeFs.mkdir(path.join(currentEnv.volumeDir, folder), { recursive: true }); + } +}; + +const buildApp = (user) => { + const routes = currentEnv.requireFresh('src/routes/favorites'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + if (user) req.user = user; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +const ALICE = { id: 'alice', roles: ['user'] }; +const BOB = { id: 'bob', roles: ['user'] }; + +const addFor = async (user, path, label) => { + const response = await request(buildApp(user)).post('/api/favorites').send({ path, label }); + expect(response.status).toBe(200); + return response.body; +}; + +describe('who may have favorites at all', () => { + it.each([ + ['get', '/api/favorites', undefined], + ['post', '/api/favorites', { path: 'Docs', label: 'Docs' }], + ['patch', '/api/favorites/reorder', { order: [] }], + ['patch', '/api/favorites/some-id', { label: 'x' }], + ['delete', '/api/favorites', { path: 'Docs' }], + ])('refuses a caller with no account on %s %s', async (method, url, body) => { + await seed(); + + const pending = request(buildApp(null))[method](url); + const response = body ? await pending.send(body) : await pending; + + expect(response.status).toBe(401); + }); +}); + +describe('keeping one account’s favorites out of another’s', () => { + it('shows each account only its own', async () => { + await seed(); + await addFor(ALICE, 'Docs/alice', 'Alice'); + await addFor(BOB, 'Docs/bob', 'Bob'); + + const alice = await request(buildApp(ALICE)).get('/api/favorites'); + const bob = await request(buildApp(BOB)).get('/api/favorites'); + + expect(alice.body.map((f) => f.label)).toEqual(['Alice']); + expect(bob.body.map((f) => f.label)).toEqual(['Bob']); + }); + + /** + * The id is guessable and travels in the URL. Holding one belonging to + * somebody else must not be enough to rename it. + */ + it('does not let one account rename another’s favorite', async () => { + await seed(); + const aliceFavorite = await addFor(ALICE, 'Docs/alice', 'Alice'); + + await request(buildApp(BOB)) + .patch(`/api/favorites/${aliceFavorite.id}`) + .send({ label: 'taken over' }); + + const alice = await request(buildApp(ALICE)).get('/api/favorites'); + expect(alice.body.map((f) => f.label)).toEqual(['Alice']); + }); + + it('does not let one account delete another’s favorite', async () => { + await seed(); + await addFor(ALICE, 'Docs/alice', 'Alice'); + + await request(buildApp(BOB)).delete('/api/favorites').send({ path: 'Docs/alice' }); + + const alice = await request(buildApp(ALICE)).get('/api/favorites'); + expect(alice.body).toHaveLength(1); + }); + + it('does not let one account reorder another’s', async () => { + await seed(); + const first = await addFor(ALICE, 'Docs/one', 'One'); + const second = await addFor(ALICE, 'Docs/two', 'Two'); + + await request(buildApp(BOB)) + .patch('/api/favorites/reorder') + .send({ order: [second.id, first.id] }); + + const alice = await request(buildApp(ALICE)).get('/api/favorites'); + expect(alice.body.map((f) => f.label)).toEqual(['One', 'Two']); + }); +}); + +describe('managing one’s own favorites', () => { + it('adds one and gives it back', async () => { + await seed(); + + const favorite = await addFor(ALICE, 'Docs/notes', 'Notes'); + + expect(favorite).toMatchObject({ path: 'Docs/notes', label: 'Notes' }); + }); + + it('renames one', async () => { + await seed(); + const favorite = await addFor(ALICE, 'Docs/notes', 'Notes'); + + const response = await request(buildApp(ALICE)) + .patch(`/api/favorites/${favorite.id}`) + .send({ label: 'Renamed' }); + + expect(response.status).toBe(200); + const listed = await request(buildApp(ALICE)).get('/api/favorites'); + expect(listed.body.map((f) => f.label)).toEqual(['Renamed']); + }); + + it('removes one by its path', async () => { + await seed(); + await addFor(ALICE, 'Docs/notes', 'Notes'); + + await request(buildApp(ALICE)).delete('/api/favorites').send({ path: 'Docs/notes' }); + + const listed = await request(buildApp(ALICE)).get('/api/favorites'); + expect(listed.body).toEqual([]); + }); + + it('reorders them', async () => { + await seed(); + const first = await addFor(ALICE, 'Docs/one', 'One'); + const second = await addFor(ALICE, 'Docs/two', 'Two'); + + const response = await request(buildApp(ALICE)) + .patch('/api/favorites/reorder') + .send({ order: [second.id, first.id] }); + + expect(response.status).toBe(200); + expect(response.body.map((f) => f.label)).toEqual(['Two', 'One']); + }); +}); + +describe('a favorite that points nowhere', () => { + it('says not found rather than failing, for a folder that is gone', async () => { + await seed(); + + const response = await request(buildApp(ALICE)) + .post('/api/favorites') + .send({ path: 'Docs/deleted-yesterday', label: 'Gone' }); + + expect(response.status).toBe(404); + }); + + it('refuses a file, since a favorite is a place to go', async () => { + await seed(); + await nodeFs.writeFile(path.join(currentEnv.volumeDir, 'Docs', 'note.txt'), 'x'); + + const response = await request(buildApp(ALICE)) + .post('/api/favorites') + .send({ path: 'Docs/note.txt', label: 'Note' }); + + expect(response.status).toBe(400); + }); +}); diff --git a/backend/tests/routes/features.test.js b/backend/tests/routes/features.test.js index 9c2e2eefc..1ceb6cdbc 100644 --- a/backend/tests/routes/features.test.js +++ b/backend/tests/routes/features.test.js @@ -53,7 +53,12 @@ describe('Features Routes', () => { expect(response.body.collabora.enabled).toBe(false); expect(response.body.collabora.extensions).toEqual([]); expect(response.body.editor.extensions).toEqual([]); - expect(response.body.hiddenFiles.patterns).toEqual(['.']); + // The dot, and the suffixes the application's own in-flight files carry. + expect(response.body.hiddenFiles.patterns).toEqual([ + '.', + 'regex:\\.download$', + 'regex:\\.uploading$', + ]); expect(response.body.terminal.extensions).toEqual(['sh']); expect(response.body.volumeUsage.enabled).toBe(false); expect(response.body.navigation.skipHome).toBe(false); diff --git a/backend/tests/routes/file-create.test.js b/backend/tests/routes/file-create.test.js new file mode 100644 index 000000000..cd5de19fd --- /dev/null +++ b/backend/tests/routes/file-create.test.js @@ -0,0 +1,73 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import request from 'supertest'; +import { setupTestEnv, createTestApp } from '../helpers/env-test-utils.js'; + +let envContext; + +beforeAll(async () => { + envContext = await setupTestEnv({ + tag: 'file-create-routes-test-', + modules: [ + 'src/utils/pathUtils', + 'src/services/accessManager', + 'src/services/authorizationService', + 'src/routes/files/file', + 'src/middleware/errorHandler', + ], + }); +}); + +afterAll(async () => { + await envContext.cleanup(); +}); + +describe('File creation route', () => { + it('never truncates an existing file when the browser listing is stale', async () => { + const directory = path.join(envContext.volumeDir, 'Documents'); + await fs.mkdir(directory, { recursive: true }); + await fs.writeFile(path.join(directory, 'Untitled.txt'), 'keep this content'); + + const router = envContext.requireFresh('src/routes/files/file'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const app = createTestApp({ + router, + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + errorHandler, + }); + + const created = await request(app).post('/api/files/file').send({ path: 'Documents' }); + expect(created.status).toBe(201); + expect(created.body.item.name).toBe('Untitled 2.txt'); + expect(await fs.readFile(path.join(directory, 'Untitled.txt'), 'utf-8')).toBe( + 'keep this content' + ); + expect(await fs.readFile(path.join(directory, 'Untitled 2.txt'), 'utf-8')).toBe(''); + }); + + it('allocates distinct names for concurrent creation requests', async () => { + const directory = path.join(envContext.volumeDir, 'Concurrent'); + await fs.mkdir(directory, { recursive: true }); + + const router = envContext.requireFresh('src/routes/files/file'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const app = createTestApp({ + router, + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + errorHandler, + }); + + const responses = await Promise.all( + Array.from({ length: 3 }, () => + request(app).post('/api/files/file').send({ path: 'Concurrent' }) + ) + ); + expect(responses.map((response) => response.status)).toEqual([201, 201, 201]); + expect(new Set(responses.map((response) => response.body.item.name))).toEqual( + new Set(['Untitled.txt', 'Untitled 2.txt', 'Untitled 3.txt']) + ); + }); +}); diff --git a/backend/tests/routes/files-download-share-count.test.js b/backend/tests/routes/files-download-share-count.test.js new file mode 100644 index 000000000..66a6cf7d2 --- /dev/null +++ b/backend/tests/routes/files-download-share-count.test.js @@ -0,0 +1,98 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A public link's own download count, for a fetch that came through the files + * route rather than the share one. + * + * A visitor inside a shared folder who picks several files gets them as one zip + * from `/api/files/download`, and one who opens a single file gets it from the + * same place. Both are that link being used, and only the share route was + * counting them — so a link whose owner watched its last-downloaded date saw + * nothing for the way their visitors actually download. + */ + +let envContext; + +const seed = async () => { + envContext = await setupTestEnv({ + tag: 'files-download-share-', + modules: [ + 'src/routes/files/download', + 'src/services/sharesService', + 'src/middleware/errorHandler', + 'src/services/accessManager', + ], + }); + const { getDb } = envContext.requireFresh('src/services/db'); + const db = await getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1', 'u1@example.com', 1, 'u1', 'U1', '["admin"]', ?, ?)` + ).run(now, now); + + const folder = path.join(envContext.volumeDir, 'Partage'); + await fs.mkdir(folder, { recursive: true }); + await fs.writeFile(path.join(folder, 'rapport.txt'), 'du texte'); + + const shares = envContext.requireFresh('src/services/sharesService'); + const share = await shares.createShare({ + ownerId: 'u1', + sourceSpace: 'volume', + sourcePath: 'Partage', + isDirectory: true, + accessMode: 'readonly', + sharingType: 'anyone', + }); + return { share, db }; +}; + +const asVisitor = (share) => { + const downloadRoutes = envContext.requireFresh('src/routes/files/download'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.guestSession = { shareId: share.id }; + next(); + }); + app.use('/api/files', downloadRoutes); + app.use(errorHandler); + return app; +}; + +const counters = (db, shareId) => + db + .prepare('SELECT download_count, last_downloaded_at, last_download_ip FROM shares WHERE id = ?') + .get(shareId); + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('a file fetched from inside a public share', () => { + it('counts against the link it was reached through', async () => { + const { share, db } = await seed(); + expect(counters(db, share.id)).toMatchObject({ + download_count: 0, + last_downloaded_at: null, + }); + + const response = await request(asVisitor(share)) + .post('/api/files/download') + .send({ paths: [`share/${share.shareToken}/rapport.txt`] }); + expect(response.status).toBe(200); + + const after = counters(db, share.id); + expect(after.download_count).toBe(1); + expect(after.last_downloaded_at).toBeTruthy(); + expect(typeof after.last_download_ip).toBe('string'); + expect(after.last_download_ip.length).toBeGreaterThan(0); + }); +}); diff --git a/backend/tests/routes/guest-exposure.test.js b/backend/tests/routes/guest-exposure.test.js new file mode 100644 index 000000000..6d7e9672c --- /dev/null +++ b/backend/tests/routes/guest-exposure.test.js @@ -0,0 +1,120 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The auth middleware lets any guest session through on every /api route, so + * endpoints that are not share-scoped have to say no themselves. These pin the + * ones that were answering to share visitors — and to anonymous callers. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const buildApp = ({ routes, mountPath, user, guestSession }) => { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + if (user) req.user = user; + if (guestSession) req.guestSession = guestSession; + next(); + }); + app.use(mountPath, routes); + return app; +}; + +describe('Volume listing', () => { + it('returns nothing to a share visitor and to an anonymous caller', async () => { + const env = await setupTestEnv({ + tag: 'guest-volumes-', + modules: [ + 'src/config/env', + 'src/config/index', + 'src/utils/pathUtils', + 'src/routes/volumes', + 'src/services/accessManager', + ], + }); + currentEnv = env; + + await fs.mkdir(path.join(env.volumeDir, 'Private'), { recursive: true }); + const volumesRoutes = env.requireFresh('src/routes/volumes'); + + const guest = await request( + buildApp({ + routes: volumesRoutes, + mountPath: '/api', + guestSession: { id: 'guest-1', shareId: 'share-1' }, + }) + ).get('/api/volumes'); + expect(guest.status).toBe(200); + expect(guest.body).toEqual([]); + + const anonymous = await request(buildApp({ routes: volumesRoutes, mountPath: '/api' })).get( + '/api/volumes' + ); + expect(anonymous.body).toEqual([]); + + // A real user still sees the volumes. + const member = await request( + buildApp({ + routes: volumesRoutes, + mountPath: '/api', + user: { id: 'user-1', roles: ['user'] }, + }) + ).get('/api/volumes'); + expect(member.status).toBe(200); + expect(Array.isArray(member.body)).toBe(true); + expect(member.body.length).toBeGreaterThan(0); + }); +}); + +describe('Folder size lookup', () => { + it('does not resolve volume paths for a share visitor', async () => { + const env = await setupTestEnv({ + tag: 'guest-folder-size-', + env: { FOLDER_SIZE_MODE: 'index' }, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/utils/pathUtils', + 'src/services/db', + 'src/routes/folderSize', + 'src/services/accessManager', + 'src/services/folderSizeIndex', + ], + }); + currentEnv = env; + + const secret = path.join(env.volumeDir, 'Private'); + await fs.mkdir(secret, { recursive: true }); + await fs.writeFile(path.join(secret, 'a.bin'), Buffer.alloc(4096, 1)); + + const folderSizeRoutes = env.requireFresh('src/routes/folderSize'); + const app = buildApp({ + routes: folderSizeRoutes, + mountPath: '/api', + guestSession: { id: 'guest-1', shareId: 'share-1' }, + }); + + const response = await request(app).get('/api/folder-size').query({ path: 'Private' }); + + // Whatever the transport says, no size may come back for a guest. + if (response.status === 200) { + const payload = response.body?.result || response.body; + expect(payload?.sizeBytes ?? null).toBeNull(); + expect(payload?.indexed ?? false).toBe(false); + } else { + expect(response.status).toBeGreaterThanOrEqual(400); + } + }); +}); diff --git a/backend/tests/routes/in-flight-saves.test.js b/backend/tests/routes/in-flight-saves.test.js new file mode 100644 index 000000000..5e862940a --- /dev/null +++ b/backend/tests/routes/in-flight-saves.test.js @@ -0,0 +1,197 @@ +import fs from 'node:fs'; +import http from 'node:http'; +import path from 'node:path'; + +import request from 'supertest'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { createTestApp, modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Saving a file and pulling a document from ONLYOFFICE write a hidden temporary + * beside the file first. A stop half-way left it there for good, out of sight. + * Each records the temporary before writing it and releases the record once + * done; this watches the journal while the content is being written — the + * moment a stop would leave the record behind — and after. + */ + +let env; +let documentServer; + +afterEach(async () => { + if (documentServer) { + await new Promise((resolve) => documentServer.close(resolve)); + documentServer = null; + } + if (env) await env.cleanup(); + env = null; +}); + +const journalOf = () => path.join(env.cacheDir, 'in-flight'); + +const onDisk = () => { + const journal = journalOf(); + if (!fs.existsSync(journal)) return []; + return fs + .readdirSync(journal) + .filter((name) => name.endsWith('.json')) + .map((name) => JSON.parse(fs.readFileSync(path.join(journal, name), 'utf8'))); +}; + +describe('saving a file', () => { + it('records its temporary while writing it, and releases it once the file is saved', async () => { + env = await setupTestEnv({ tag: 'in-flight-save-', env: { UPLOAD_STORAGE_RESERVE: '0' } }); + await require(modulePath('src/services/db')).getDb(); + const target = path.join(env.volumeDir, 'report.txt'); + fs.writeFileSync(target, 'first'); + + let whileWriting = null; + await require(modulePath('src/services/versions/operations')).saveFile( + target, + async (temporary) => { + whileWriting = { temporary, records: onDisk() }; + fs.writeFileSync(temporary, 'second'); + }, + { source: 'editor' } + ); + + expect(whileWriting.records).toMatchObject([ + { path: whileWriting.temporary, kind: 'temporary-file' }, + ]); + expect(path.dirname(whileWriting.temporary)).toBe(env.volumeDir); + expect(fs.readFileSync(target, 'utf8')).toBe('second'); + expect(onDisk()).toEqual([]); + }); + + it('releases its record when the write fails', async () => { + env = await setupTestEnv({ tag: 'in-flight-save-fail-', env: { UPLOAD_STORAGE_RESERVE: '0' } }); + await require(modulePath('src/services/db')).getDb(); + const target = path.join(env.volumeDir, 'report.txt'); + fs.writeFileSync(target, 'first'); + + await expect( + require(modulePath('src/services/versions/operations')).saveFile( + target, + async () => { + throw new Error('the editor went away'); + }, + { source: 'editor' } + ) + ).rejects.toThrow('the editor went away'); + + expect(fs.readFileSync(target, 'utf8')).toBe('first'); + expect(onDisk()).toEqual([]); + }); +}); + +describe('saving a new file', () => { + it('records its temporary while writing it, releases it, and never takes a name already held', async () => { + env = await setupTestEnv({ tag: 'in-flight-save-new-', env: { UPLOAD_STORAGE_RESERVE: '0' } }); + await require(modulePath('src/services/db')).getDb(); + fs.writeFileSync(path.join(env.volumeDir, 'notes.md'), 'theirs'); + + let whileWriting = null; + const placed = await require(modulePath('src/services/versions/operations')).saveNewFile( + env.volumeDir, + 'notes.md', + async (temporary) => { + whileWriting = { temporary, records: onDisk() }; + fs.writeFileSync(temporary, 'mine'); + }, + { purpose: 'restore' } + ); + + expect(whileWriting.records).toMatchObject([ + { path: whileWriting.temporary, kind: 'temporary-file' }, + ]); + expect(path.dirname(whileWriting.temporary)).toBe(env.volumeDir); + expect(placed).toEqual({ + name: 'notes (1).md', + path: path.join(env.volumeDir, 'notes (1).md'), + }); + expect(fs.readFileSync(path.join(env.volumeDir, 'notes.md'), 'utf8')).toBe('theirs'); + expect(fs.readFileSync(placed.path, 'utf8')).toBe('mine'); + expect(fs.existsSync(whileWriting.temporary)).toBe(false); + expect(onDisk()).toEqual([]); + }); + + it('releases its record and leaves no temporary when the write fails', async () => { + env = await setupTestEnv({ + tag: 'in-flight-save-new-fail-', + env: { UPLOAD_STORAGE_RESERVE: '0' }, + }); + await require(modulePath('src/services/db')).getDb(); + + let temporaryPath = null; + await expect( + require(modulePath('src/services/versions/operations')).saveNewFile( + env.volumeDir, + 'notes.md', + async (temporary) => { + temporaryPath = temporary; + fs.writeFileSync(temporary, 'half'); + throw new Error('the version could not be read'); + } + ) + ).rejects.toThrow('the version could not be read'); + + expect(fs.existsSync(temporaryPath)).toBe(false); + expect(fs.existsSync(path.join(env.volumeDir, 'notes.md'))).toBe(false); + expect(onDisk()).toEqual([]); + }); +}); + +describe('a document pulled from ONLYOFFICE', () => { + it('records its temporary while it downloads, and releases it once the copy is in place', async () => { + let whileDownloading = null; + documentServer = http.createServer((req, res) => { + res.setHeader('Content-Type', 'application/octet-stream'); + res.write('converted '); + // Headers and a first chunk are out: the temporary is being written. + setTimeout(() => { + whileDownloading = onDisk(); + res.end('document'); + }, 50); + }); + await new Promise((resolve, reject) => { + documentServer.once('error', reject); + documentServer.listen(0, '127.0.0.1', resolve); + }); + const { port } = documentServer.address(); + + env = await setupTestEnv({ + tag: 'in-flight-onlyoffice-', + modules: ['src/routes/onlyoffice', 'src/middleware/errorHandler'], + env: { + PUBLIC_URL: 'https://files.example.com', + ONLYOFFICE_URL: `http://127.0.0.1:${port}`, + ONLYOFFICE_SECRET: 'onlyoffice-test-secret', + }, + }); + fs.writeFileSync(path.join(env.volumeDir, 'report.docx'), 'original'); + const app = createTestApp({ + router: env.requireFresh('src/routes/onlyoffice'), + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + errorHandler: env.requireFresh('src/middleware/errorHandler').errorHandler, + }); + + const response = await request(app) + .post('/api/onlyoffice/save-as') + .send({ + path: 'report.docx', + url: `http://127.0.0.1:${port}/converted.pdf`, + title: 'report.pdf', + }); + + expect(response.status).toBe(200); + expect(whileDownloading).toHaveLength(1); + expect(whileDownloading[0].kind).toBe('temporary-file'); + expect(path.dirname(whileDownloading[0].path)).toBe(env.volumeDir); + expect(path.basename(whileDownloading[0].path)).toMatch(/^\.report\.pdf\.onlyoffice-.+\.tmp$/); + expect(fs.readFileSync(path.join(env.volumeDir, 'report.pdf'), 'utf8')).toBe( + 'converted document' + ); + expect(onDisk()).toEqual([]); + }); +}); diff --git a/backend/tests/routes/large-selection.test.js b/backend/tests/routes/large-selection.test.js new file mode 100644 index 000000000..ebbd7068c --- /dev/null +++ b/backend/tests/routes/large-selection.test.js @@ -0,0 +1,74 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Deleting a large selection sends one path per file, and Express caps a JSON + * body at 100 kB by default. Two thousand files is around 150 kB of paths — an + * ordinary selection in a file manager — and the request came back as "request + * entity too large" before anything looked at it. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const selectionOf = (count) => + Array.from({ length: count }, (_, i) => ({ + path: 'Photos/2024/Vacances ete/Albums', + name: `IMG_20240715_${String(i).padStart(5, '0')}.jpeg`, + })); + +const buildApp = (env) => { + const { uploads } = env.requireFresh('src/config/index'); + const app = express(); + app.use(express.json({ limit: uploads.maxJsonBodyBytes })); + app.post('/echo', (req, res) => res.json({ received: req.body.items.length })); + // Express answers 413 through the error pipeline, not the route. + // eslint-disable-next-line no-unused-vars + app.use((err, _req, res, _next) => res.status(err.status || 500).json({ error: err.type })); + return app; +}; + +describe('Large selections', () => { + it('accepts the body a 2000-file selection produces', async () => { + const env = await setupTestEnv({ + tag: 'large-selection-', + modules: ['src/config/env', 'src/config/index'], + }); + currentEnv = env; + + const items = selectionOf(2000); + const bytes = Buffer.byteLength(JSON.stringify({ items })); + // Well past Express's 100 kB default — that is the whole point. + expect(bytes).toBeGreaterThan(120 * 1024); + + const response = await request(buildApp(env)).post('/echo').send({ items }); + expect(response.status).toBe(200); + expect(response.body.received).toBe(2000); + }); + + it('is configurable, and still refuses a body past the ceiling', async () => { + const env = await setupTestEnv({ + tag: 'large-selection-limit-', + env: { MAX_JSON_BODY_SIZE: '100kb' }, + modules: ['src/config/env', 'src/config/index'], + }); + currentEnv = env; + + const { uploads } = env.requireFresh('src/config/index'); + expect(uploads.maxJsonBodyBytes).toBe(100 * 1024); + + // The ceiling still exists: it is a guard, not an open door. + const response = await request(buildApp(env)) + .post('/echo') + .send({ items: selectionOf(2000) }); + expect(response.status).toBe(413); + }); +}); diff --git a/backend/tests/routes/malformed-file-requests.test.js b/backend/tests/routes/malformed-file-requests.test.js new file mode 100644 index 000000000..5690d004e --- /dev/null +++ b/backend/tests/routes/malformed-file-requests.test.js @@ -0,0 +1,229 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A request that is the caller's fault, answered as such. + * + * Sending no items to delete raised a plain `Error`, which carries no status. + * The handler read that as an unexpected failure: 500, `isOperational: false`, + * and a full stack in the log saying the server had broken. Nothing had — the + * request was malformed, and a malformed request is a thing the caller can fix + * and the operator should not be paged about. + */ + +let currentEnv; + +const setup = async () => { + const env = await setupTestEnv({ + tag: 'malformed-file-requests-', + modules: [ + 'src/config/env', + 'src/config/index', + 'src/routes/files/index', + 'src/middleware/errorHandler', + 'src/services/accessManager', + 'src/services/settingsService', + 'src/utils/pathUtils', + ], + }); + currentEnv = env; + + const fileRoutes = env.requireFresh('src/routes/files/index'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + const db = await env.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('admin', 'admin@example.com', 1, 'admin', 'Admin', '["admin"]', ?, ?)` + ).run(now, now); + + return createTestApp({ + router: fileRoutes, + mountPath: '/api', + user: { id: 'admin', roles: ['admin'] }, + errorHandler, + }); +}; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +describe('a delete with nothing to delete', () => { + it('is a client error, not a server one', async () => { + const app = await setup(); + + const response = await request(app).delete('/api/files').send({ items: [] }); + + expect(response.status).toBe(400); + }); + + it('is the same when the field is missing entirely', async () => { + const app = await setup(); + + const response = await request(app).delete('/api/files').send({}); + + expect(response.status).toBe(400); + }); + + /** The wrong shape is as much the caller's fault as the empty one. */ + it('is the same when it is not a list', async () => { + const app = await setup(); + + const response = await request(app).delete('/api/files').send({ items: 'everything' }); + + expect(response.status).toBe(400); + }); + + it('says what was wrong with it', async () => { + const app = await setup(); + + const response = await request(app).delete('/api/files').send({ items: [] }); + + expect(response.body?.error?.message).toMatch(/at least one item/i); + }); +}); + +describe('asking what a delete would affect, with nothing to affect', () => { + it('is a client error too', async () => { + const app = await setup(); + + const response = await request(app).post('/api/files/delete-impact').send({ items: [] }); + + expect(response.status).toBe(400); + }); +}); + +describe('a transfer with nothing to transfer', () => { + it('is a client error', async () => { + const app = await setup(); + + const response = await request(app) + .post('/api/files/copy') + .send({ items: [], destination: 'Docs' }); + + expect(response.status).toBe(400); + }); + + /** + * The root is not a destination — there is no volume in it to write to. That + * is the caller choosing wrongly, not the server failing. + */ + it('refuses the root as a destination without calling it a server failure', async () => { + const app = await setup(); + + const response = await request(app) + .post('/api/files/copy') + .send({ items: [{ path: '', name: 'notes.txt' }], destination: '' }); + + expect(response.status).toBe(400); + }); +}); + +/** + * A refusal, answered as a refusal. + * + * Every access decision in the transfer service raised a plain `Error` too, so + * being told "you may not" arrived as 500 with `isOperational: false` and a + * stack in the log — an outage, by the shape of it, for a permission working + * exactly as designed. It also cost the caller: the uploader retries a 500 and + * does not retry a 403, so a refusal was retried until it ran out of attempts. + */ +describe('a path the caller may not use', () => { + const setupWithAcl = async (rules) => { + const env = await setupTestEnv({ + tag: 'transfer-denials-', + modules: [ + 'src/config/env', + 'src/config/index', + 'src/routes/files/index', + 'src/middleware/errorHandler', + 'src/services/accessManager', + 'src/services/accessControlService', + 'src/services/settingsService', + 'src/utils/pathUtils', + ], + }); + currentEnv = env; + + const accessControl = env.requireFresh('src/services/accessControlService'); + await accessControl.setRules(rules); + + const fileRoutes = env.requireFresh('src/routes/files/index'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + const db = await env.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('user', 'user@example.com', 1, 'user', 'User', '["user"]', ?, ?)` + ).run(now, now); + + const app = createTestApp({ + router: fileRoutes, + mountPath: '/api', + user: { id: 'user', roles: ['user'] }, + errorHandler, + }); + return { app, volume: env.volumeDir }; + }; + + it('refuses a delete from a read-only folder as forbidden, not as a failure', async () => { + const { app, volume } = await setupWithAcl([ + { path: '/Locked', permissions: 'ro', recursive: true }, + ]); + await fs.mkdir(path.join(volume, 'Locked'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Locked', 'note.txt'), 'read only'); + + const response = await request(app) + .delete('/api/files') + .send({ items: [{ path: 'Locked', name: 'note.txt' }] }); + + expect(response.status).toBe(403); + }); + + it('leaves the file where it is', async () => { + const { app, volume } = await setupWithAcl([ + { path: '/Locked', permissions: 'ro', recursive: true }, + ]); + await fs.mkdir(path.join(volume, 'Locked'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Locked', 'note.txt'), 'read only'); + + await request(app) + .delete('/api/files') + .send({ items: [{ path: 'Locked', name: 'note.txt' }] }); + + await expect(fs.access(path.join(volume, 'Locked', 'note.txt'))).resolves.toBeUndefined(); + }); + + it('says why rather than only that it failed', async () => { + const { app, volume } = await setupWithAcl([ + { path: '/Locked', permissions: 'ro', recursive: true }, + ]); + await fs.mkdir(path.join(volume, 'Locked'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Locked', 'note.txt'), 'read only'); + + const response = await request(app) + .delete('/api/files') + .send({ items: [{ path: 'Locked', name: 'note.txt' }] }); + + expect(response.body?.error?.message).toBeTruthy(); + }); +}); + +describe('a source that is not there', () => { + it('is a not-found, not a server failure', async () => { + const app = await setup(); + + const response = await request(app) + .post('/api/files/copy') + .send({ items: [{ path: '', name: 'never-existed.txt' }], destination: 'Docs' }); + + expect(response.status).toBeLessThan(500); + }); +}); diff --git a/backend/tests/routes/media-tracks-route.test.js b/backend/tests/routes/media-tracks-route.test.js index 095423e28..cb7a0c34e 100644 --- a/backend/tests/routes/media-tracks-route.test.js +++ b/backend/tests/routes/media-tracks-route.test.js @@ -35,13 +35,35 @@ const buildFilm = async (dir) => { const srt = path.join(dir, 'subs.srt'); await fs.writeFile(srt, '1\n00:00:00,500 --> 00:00:02,000\nBonjour le monde\n'); await execFileAsync('ffmpeg', [ - '-v', 'error', '-y', - '-f', 'lavfi', '-i', 'testsrc=size=160x120:rate=25:duration=2', - '-f', 'lavfi', '-i', 'sine=frequency=440:duration=2', - '-i', srt, - '-map', '0:v', '-map', '1:a', '-map', '2:s', - '-c:v', 'libx264', '-preset', 'ultrafast', '-c:a', 'ac3', '-c:s', 'srt', - '-metadata:s:a:0', 'language=fre', + '-v', + 'error', + '-y', + '-f', + 'lavfi', + '-i', + 'testsrc=size=160x120:rate=25:duration=2', + '-f', + 'lavfi', + '-i', + 'sine=frequency=440:duration=2', + '-i', + srt, + '-map', + '0:v', + '-map', + '1:a', + '-map', + '2:s', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-c:a', + 'ac3', + '-c:s', + 'srt', + '-metadata:s:a:0', + 'language=fre', path.join(dir, 'film.mkv'), ]); await fs.rm(srt); @@ -97,7 +119,9 @@ describe.skipIf(!(await hasFfmpeg()))('asking what is in a video', () => { it('names the soundtrack it found', async () => { const response = await tracksOf(await setup()); - expect(response.body.audio).toEqual([expect.objectContaining({ codec: 'ac3', language: 'fr' })]); + expect(response.body.audio).toEqual([ + expect.objectContaining({ codec: 'ac3', language: 'fr' }), + ]); }); /** The reported symptom: sound present, browser silent. */ @@ -202,7 +226,10 @@ describe.skipIf(!(await hasFfmpeg()))('asking for a subtitle', () => { */ it('refuses a subtitle file belonging to another video', async () => { const app = await setup(); - await fs.writeFile(path.join(ctx.volumeDir, 'other.fr.srt'), '1\n00:00:01,000 --> 00:00:02,000\nx\n'); + await fs.writeFile( + path.join(ctx.volumeDir, 'other.fr.srt'), + '1\n00:00:01,000 --> 00:00:02,000\nx\n' + ); const response = await subtitle(app, { path: 'film.mkv', file: 'other.fr.srt' }); diff --git a/backend/tests/routes/no-shell.test.js b/backend/tests/routes/no-shell.test.js new file mode 100644 index 000000000..cc8092240 --- /dev/null +++ b/backend/tests/routes/no-shell.test.js @@ -0,0 +1,117 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import request from 'supertest'; +import { setupTestEnv, createTestApp } from '../helpers/env-test-utils.js'; + +/** + * A name from a request is not a shell string. + * + * Two routes built command lines with values from the request pasted into them and + * handed the line to `/bin/sh`. A folder name, an owner, a group: anything that + * closed the quoting left the rest for the shell to run, as the user this server + * runs as. The folder one needs no privilege at all — any account that can make a + * folder can name one, and with AUTH_ENABLED=false that is anybody who can reach + * the server. + * + * The payloads below only create a file inside the test's own temporary directory, + * and what each case asserts is that the file is not there. + */ + +let env; + +// Where a payload would land: a folder name cannot hold a slash, so it writes into +// the working directory of the process running this. +const PROOF = path.join(process.cwd(), 'a-shell-ran-here'); +const shellRan = async () => + fs + .access(PROOF) + .then(() => true) + .catch(() => false); + +afterEach(async () => { + // Whatever an assertion did, this happens: a run that does execute must not leave + // the file behind for the next one to find. + await fs.rm(PROOF, { force: true }); + if (env) { + await env.cleanup(); + env = null; + } +}); + +describe('asking how full a volume is', () => { + it('does not run what a folder is called', async () => { + env = await setupTestEnv({ + tag: 'usage-no-shell-', + modules: ['src/routes/usage', 'src/services/accessManager', 'src/utils/pathUtils'], + }); + + // A name that closes the quoting the route used to open around it. + const folder = 'Vol";touch a-shell-ran-here;echo "'; + await fs.mkdir(path.join(env.volumeDir, folder)); + + const app = createTestApp({ + router: env.requireFresh('src/routes/usage'), + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + }); + + const response = await request(app).get(`/api/usage/${encodeURIComponent(folder)}`); + + expect(response.status).toBe(200); + expect(await shellRan()).toBe(false); + }); +}); + +describe('changing an owner', () => { + // With the error handler, so a refusal arrives as the sentence it is meant to be + // rather than an empty body with a status on it. + const appFor = () => + createTestApp({ + router: env.requireFresh('src/routes/permissions'), + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + errorHandler: env.requireFresh('src/middleware/errorHandler').errorHandler, + }); + + const setup = async (tag) => { + env = await setupTestEnv({ + tag, + modules: [ + 'src/routes/permissions', + 'src/middleware/errorHandler', + 'src/services/accessManager', + 'src/utils/pathUtils', + ], + }); + await fs.mkdir(path.join(env.volumeDir, 'Vol'), { recursive: true }); + await fs.writeFile(path.join(env.volumeDir, 'Vol', 'file.txt'), 'x'); + }; + + it('does not run what an owner is called', async () => { + await setup('chown-no-shell-'); + const response = await request(appFor()) + .post('/api/permissions/chown') + .send({ path: 'Vol/file.txt', owner: 'root";touch a-shell-ran-here;echo "' }); + + // Refused as a name, or attempted as one argument and failed — either way the + // command inside it is not a command. + expect(await shellRan()).toBe(false); + expect(response.status).toBeGreaterThanOrEqual(400); + }); + + it('refuses a name that would be read as an option', async () => { + await setup('chown-option-'); + + const response = await request(appFor()) + .post('/api/permissions/chown') + .send({ path: 'Vol/file.txt', owner: '--reference=/etc/shadow' }); + + // Refused as a name rather than attempted as one: a 400 from the route, and a + // sentence that says which field and that it is a name. Not the exact wording — + // a test that pins a sentence breaks when somebody improves it. + expect(response.status).toBe(400); + expect(JSON.stringify(response.body)).toMatch(/owner/i); + expect(JSON.stringify(response.body)).toMatch(/(invalid|not a valid).{0,20}name/i); + }); +}); diff --git a/backend/tests/routes/office-versions.test.js b/backend/tests/routes/office-versions.test.js new file mode 100644 index 000000000..54be02fad --- /dev/null +++ b/backend/tests/routes/office-versions.test.js @@ -0,0 +1,371 @@ +import fs from 'node:fs/promises'; +import http from 'node:http'; +import path from 'node:path'; +import express from 'express'; +import jwt from 'jsonwebtoken'; +import request from 'supertest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A document's history, inside the office editors. + * + * ONLYOFFICE is handed the history and then, version by version, a URL to fetch + * each one from; Collabora is handed an earlier version to open on its own. + * Either way the Document Server or the WOPI client fetches content on its own, + * with no session of ours: the token in the URL is all that decides what goes + * out. So these check that it names the version and nothing else, that nothing + * can be written through it, and that a share whose owner keeps the history + * hidden hands none of it out. + */ + +const ONLYOFFICE_SECRET = 'office-versions-secret'; +const COLLABORA_SECRET = 'office-versions-collabora-secret'; +const DOCUMENT = 'Projects/report.docx'; + +const DISCOVERY = ` + + +`; + +let env; +let users; +let app; +let discovery; +let documentServer; + +const load = (relative) => require(modulePath(relative)); + +beforeEach(async () => { + // Stands in for both servers: Collabora's discovery, and a Document Server + // with a saved document ready to be fetched — so that a save refused is + // refused, not merely unable to download. + discovery = http.createServer((req, res) => { + if (req.url === '/saved.docx') { + res.setHeader('Content-Type', 'application/octet-stream'); + res.end('overwritten'); + return; + } + res.setHeader('Content-Type', 'text/xml'); + res.end(DISCOVERY); + }); + await new Promise((resolve) => discovery.listen(0, '127.0.0.1', resolve)); + documentServer = `http://127.0.0.1:${discovery.address().port}`; + + env = await setupTestEnv({ + tag: 'office-versions-', + env: { + PUBLIC_URL: 'https://files.example.com', + SHARES_ENABLED: 'true', + ONLYOFFICE_URL: documentServer, + ONLYOFFICE_SECRET, + COLLABORA_URL: 'https://collabora.example.com', + COLLABORA_SECRET, + COLLABORA_DISCOVERY_URL: `${documentServer}/hosting/discovery`, + }, + }); + users = { + alice: await load('src/services/users').createLocalUser({ + email: 'alice@example.com', + username: 'alice', + displayName: 'Alice', + password: 'secret123', + roles: ['user'], + }), + }; + await fs.mkdir(path.join(env.volumeDir, 'Projects'), { recursive: true }); + + app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + const who = req.get('x-test-user'); + if (who?.startsWith('guest:')) { + req.guestSession = { id: 'guest-session', shareId: who.slice('guest:'.length) }; + } else if (who) { + req.user = users[who]; + } + next(); + }); + app.use('/api', load('src/routes/onlyoffice')); + app.use('/api', load('src/routes/collabora')); + app.use('/api/shares', load('src/routes/shares')); + app.use(load('src/middleware/errorHandler').errorHandler); +}); + +afterEach(async () => { + load('src/services/trash/maintenance').stop(); + await new Promise((resolve) => discovery.close(resolve)); + await env.cleanup(); +}); + +const absolute = (relative) => path.join(env.volumeDir, ...relative.split('/')); + +/** A document saved over twice: two versions, "first" then "second", and "third" now. */ +const withHistory = async (relative = DOCUMENT) => { + const target = absolute(relative); + await fs.writeFile(target, 'first'); + for (const content of ['second', 'third']) { + await load('src/services/versions/operations').saveFile( + target, + (temporaryPath) => fs.writeFile(temporaryPath, content), + { + purpose: 'test', + author: { id: users.alice.id, label: 'Alice' }, + source: 'editor', + explicit: true, + } + ); + } +}; + +const post = (url, body, who = 'alice') => { + const call = request(app).post(url).send(body); + return who ? call.set('x-test-user', who) : call; +}; + +/** + * A document goes out with its own content type, which supertest does not + * buffer: the body is collected by hand, or the test compares nothing. + */ +const collectBody = (res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks).toString('utf8'))); +}; + +/** What the Document Server gets when it fetches a URL it was handed: no session, its own token. */ +const fetchAsDocumentServer = (absoluteUrl) => { + const url = new URL(absoluteUrl); + return request(app) + .get(`${url.pathname}${url.search}`) + .set('Authorization', `Bearer ${jwt.sign({ any: true }, ONLYOFFICE_SECRET)}`) + .buffer(true) + .parse(collectBody); +}; + +describe('the history ONLYOFFICE shows', () => { + it('lists the versions oldest first, and the document as it is last, under its open key', async () => { + await withHistory(); + const opened = await post('/api/onlyoffice/config', { path: DOCUMENT }); + expect(opened.status).toBe(200); + + const response = await post('/api/onlyoffice/history', { path: DOCUMENT }); + + expect(response.status).toBe(200); + const { history, currentVersion, canRestore } = response.body; + expect(currentVersion).toBe(3); + expect(history.map((entry) => entry.version)).toEqual([1, 2, 3]); + expect(history[2]).toMatchObject({ versionId: null, key: opened.body.config.document.key }); + expect(history[0].key).toBe(`version-${history[0].versionId}`); + expect(new Set(history.map((entry) => entry.key)).size).toBe(3); + // "first" was written outside the app, so nobody is known to have written it; + // "second" was Alice's save. + expect(history[0].user).toEqual({ id: '', name: '' }); + expect(history[1].user).toEqual({ id: users.alice.id, name: 'Alice' }); + expect(canRestore).toBe(true); + }); + + it('hands each version over signed, at a URL that serves that version', async () => { + await withHistory(); + const { history } = (await post('/api/onlyoffice/history', { path: DOCUMENT })).body; + + const oldest = await post('/api/onlyoffice/history-data', { + path: DOCUMENT, + version: 1, + versionId: history[0].versionId, + }); + const middle = await post('/api/onlyoffice/history-data', { + path: DOCUMENT, + version: 2, + versionId: history[1].versionId, + }); + const current = await post('/api/onlyoffice/history-data', { path: DOCUMENT, version: 3 }); + + expect(oldest.status).toBe(200); + expect(oldest.body).toMatchObject({ version: 1, fileType: 'docx', key: history[0].key }); + expect(jwt.verify(oldest.body.token, ONLYOFFICE_SECRET)).toMatchObject({ + version: 1, + fileType: 'docx', + key: history[0].key, + url: oldest.body.url, + }); + expect((await fetchAsDocumentServer(oldest.body.url)).body).toBe('first'); + expect((await fetchAsDocumentServer(middle.body.url)).body).toBe('second'); + expect(current.body.key).toBe(history[2].key); + expect((await fetchAsDocumentServer(current.body.url)).body).toBe('third'); + }); + + it("refuses a version number that is not one, and another file's version", async () => { + await withHistory(); + await withHistory('Projects/other.docx'); + const others = (await post('/api/onlyoffice/history', { path: 'Projects/other.docx' })).body; + + const noNumber = await post('/api/onlyoffice/history-data', { path: DOCUMENT, version: 0 }); + const borrowed = await post('/api/onlyoffice/history-data', { + path: DOCUMENT, + version: 1, + versionId: others.history[0].versionId, + }); + + expect(noNumber.status).toBe(400); + expect(borrowed.status).toBe(404); + }); +}); + +describe('an earlier version opened in ONLYOFFICE', () => { + it('is a viewer on that version, with no callback and a key of its own', async () => { + await withHistory(); + const { history } = (await post('/api/onlyoffice/history', { path: DOCUMENT })).body; + + const viewed = await post('/api/onlyoffice/config', { + path: DOCUMENT, + versionId: history[0].versionId, + }); + + expect(viewed.status).toBe(200); + const { config } = viewed.body; + expect(config.editorConfig.mode).toBe('view'); + expect(config.editorConfig.callbackUrl).toBeUndefined(); + expect(config.document.permissions).toMatchObject({ + edit: false, + comment: false, + review: false, + }); + expect(config.document.key).toBe(history[0].key); + expect(viewed.body.forceSaveSessionId).toBeNull(); + expect(jwt.verify(config.token, ONLYOFFICE_SECRET).document.url).toBe(config.document.url); + expect((await fetchAsDocumentServer(config.document.url)).body).toBe('first'); + }); + + it('writes nothing through its token, even when a save is sent with it', async () => { + await withHistory(); + const { history } = (await post('/api/onlyoffice/history', { path: DOCUMENT })).body; + const { config } = ( + await post('/api/onlyoffice/config', { path: DOCUMENT, versionId: history[0].versionId }) + ).body; + const backend = new URL(config.document.url).searchParams.get('backend'); + + // The Document Server does have a document ready: were the save allowed, it + // would be written. + const response = await request(app) + .post(`/api/onlyoffice/callback?path=${encodeURIComponent(DOCUMENT)}&backend=${backend}`) + .set('Authorization', `Bearer ${jwt.sign({ callback: true }, ONLYOFFICE_SECRET)}`) + .send({ status: 6, key: history[0].key, url: `${documentServer}/saved.docx` }); + + expect(response.body).toEqual({ error: 1 }); + expect(await fs.readFile(absolute(DOCUMENT), 'utf8')).toBe('third'); + }); +}); + +describe('an earlier version opened in Collabora', () => { + const versionIds = async () => + ( + await load('src/services/versions').listVersions({ user: users.alice }, DOCUMENT) + ).versions.map((version) => version.id); + + it('is read under the file name, apart from the document, and never written', async () => { + await withHistory(); + const [, oldestId] = await versionIds(); + + const opened = await post('/api/collabora/config', { path: DOCUMENT, versionId: oldestId }); + const live = await post('/api/collabora/config', { path: DOCUMENT }); + + expect(opened.status).toBe(200); + const { accessToken, fileId } = opened.body; + expect(fileId).not.toBe(live.body.fileId); + expect(new URL(opened.body.urlSrc).searchParams.get('revisionhistory')).toBeNull(); + + const info = await request(app) + .get(`/api/collabora/wopi/files/${fileId}`) + .query({ access_token: accessToken }); + expect(info.body).toMatchObject({ + BaseFileName: 'report.docx', + UserCanWrite: false, + UserCanNotWriteRelative: true, + }); + + const content = await request(app) + .get(`/api/collabora/wopi/files/${fileId}/contents`) + .query({ access_token: accessToken }) + .buffer(true) + .parse(collectBody); + expect(content.body).toBe('first'); + + const put = await request(app) + .post(`/api/collabora/wopi/files/${fileId}/contents`) + .query({ access_token: accessToken }) + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from('overwritten')); + expect(put.status).toBe(403); + expect(await fs.readFile(absolute(DOCUMENT), 'utf8')).toBe('third'); + }); + + it('shows the Revision history entry on the document itself, where there is one', async () => { + await withHistory(); + + const live = await post('/api/collabora/config', { path: DOCUMENT }); + + expect(live.status).toBe(200); + expect(new URL(live.body.urlSrc).searchParams.get('revisionhistory')).toBe('1'); + const info = await request(app) + .get(`/api/collabora/wopi/files/${live.body.fileId}`) + .query({ access_token: live.body.accessToken }); + expect(info.body.BaseFileName).toBe('report.docx'); + expect(info.body.UserCanNotWriteRelative).toBeUndefined(); + }); + + it('shows no Revision history entry once versions are switched off', async () => { + await withHistory(); + await load('src/services/settingsService').setSettings({ versions: { enabled: false } }); + + const live = await post('/api/collabora/config', { path: DOCUMENT }); + + expect(live.status).toBe(200); + expect(new URL(live.body.urlSrc).searchParams.get('revisionhistory')).toBeNull(); + }); +}); + +describe('through a share', () => { + it('hands out no history until its owner shows it', async () => { + await withHistory(); + const { history } = (await post('/api/onlyoffice/history', { path: DOCUMENT })).body; + const share = (await post('/api/shares', { sourcePath: 'Projects', sharingType: 'anyone' })) + .body; + const shared = `share/${share.shareToken}/report.docx`; + const guest = `guest:${share.id}`; + const versionId = history[0].versionId; + + expect((await post('/api/onlyoffice/history', { path: shared }, guest)).status).toBe(403); + expect( + (await post('/api/onlyoffice/history-data', { path: shared, version: 1, versionId }, guest)) + .status + ).toBe(403); + // Nor the document as it is, from inside a history it may not show. + expect( + (await post('/api/onlyoffice/history-data', { path: shared, version: 3 }, guest)).status + ).toBe(403); + expect((await post('/api/onlyoffice/config', { path: shared, versionId }, guest)).status).toBe( + 403 + ); + expect((await post('/api/collabora/config', { path: shared, versionId }, guest)).status).toBe( + 403 + ); + const live = await post('/api/collabora/config', { path: shared }, guest); + expect(live.status).toBe(200); + expect(new URL(live.body.urlSrc).searchParams.get('revisionhistory')).toBeNull(); + + await request(app) + .put(`/api/shares/${share.id}`) + .set('x-test-user', 'alice') + .send({ versionsVisible: true }); + + const shown = await post('/api/onlyoffice/history', { path: shared }, guest); + expect(shown.status).toBe(200); + expect(shown.body.history).toHaveLength(3); + // A link for anyone is read-only: seeing the history is not restoring it. + expect(shown.body.canRestore).toBe(false); + const reopened = await post('/api/collabora/config', { path: shared }, guest); + expect(new URL(reopened.body.urlSrc).searchParams.get('revisionhistory')).toBe('1'); + }); +}); diff --git a/backend/tests/routes/onlyoffice-callback-security.test.js b/backend/tests/routes/onlyoffice-callback-security.test.js new file mode 100644 index 000000000..30aa354ab --- /dev/null +++ b/backend/tests/routes/onlyoffice-callback-security.test.js @@ -0,0 +1,181 @@ +import { describe, it, expect } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import express from 'express'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The ONLYOFFICE callback is exempt from authentication: the Document Server + * calls it directly. It trusts a backend token we signed instead of resolving + * permissions again, so that token has to carry the write decision, and the + * URL it is asked to download from has to belong to the Document Server. + */ + +const SECRET = 'test-onlyoffice-secret'; +const DOCUMENT_SERVER = 'https://documentserver.example.com'; + +const createContext = async (tag, extraEnv = {}) => { + const env = await setupTestEnv({ + tag, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/utils/pathUtils', + 'src/routes/onlyoffice', + 'src/middleware/errorHandler', + 'src/services/accessManager', + 'src/services/settingsService', + ], + env: { + ONLYOFFICE_URL: DOCUMENT_SERVER, + ONLYOFFICE_SECRET: SECRET, + PUBLIC_URL: 'https://files.example.com', + ...extraEnv, + }, + }); + + const onlyofficeRoutes = env.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + + const app = express(); + app.use(express.json()); + app.use('/api', onlyofficeRoutes); + app.use(errorHandler); + + return { env, app }; +}; + +const signBackendToken = (payload) => + jwt.sign({ typ: 'nextexplorer-backend', ...payload }, SECRET, { algorithm: 'HS256' }); + +/** Stand-in Document Server that serves the "saved" document. */ +const startDocumentServer = async (body) => { + const server = http.createServer((_req, res) => { + res.writeHead(200, { 'Content-Type': 'application/octet-stream' }); + res.end(body); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address(); + return { + origin: `http://127.0.0.1:${port}`, + url: `http://127.0.0.1:${port}/cache/files/edited.docx`, + close: () => new Promise((resolve) => server.close(resolve)), + }; +}; + +describe('ONLYOFFICE callback security', () => { + it('saves the document when the session was allowed to write, and only then', async () => { + const documentServer = await startDocumentServer('edited content'); + const { env, app } = await createContext('onlyoffice-write-', { + ONLYOFFICE_DOWNLOAD_ORIGINS: documentServer.origin, + }); + try { + const writable = path.join(env.volumeDir, 'writable.docx'); + const readonly = path.join(env.volumeDir, 'readonly.docx'); + await fs.writeFile(writable, 'original'); + await fs.writeFile(readonly, 'original'); + + const dsToken = jwt.sign({ status: 2 }, SECRET, { algorithm: 'HS256' }); + + // Baseline: an editing session that was allowed to write saves. + const allowed = await request(app) + .post('/api/onlyoffice/callback') + .query({ + path: 'writable.docx', + backend: signBackendToken({ absolutePath: writable, canWrite: true }), + }) + .set('Authorization', `Bearer ${dsToken}`) + .send({ status: 2, url: documentServer.url }); + + expect(allowed.body.error).toBe(0); + await expect(fs.readFile(writable, 'utf-8')).resolves.toBe('edited content'); + + // Same request, same reachable URL: only the write flag differs. + const denied = await request(app) + .post('/api/onlyoffice/callback') + .query({ + path: 'readonly.docx', + backend: signBackendToken({ absolutePath: readonly, canWrite: false }), + }) + .set('Authorization', `Bearer ${dsToken}`) + .send({ status: 2, url: documentServer.url }); + + // The callback contract answers 200 with a non-zero error code. + expect(denied.body.error).toBe(1); + await expect(fs.readFile(readonly, 'utf-8')).resolves.toBe('original'); + } finally { + await env.cleanup(); + await documentServer.close(); + } + }); + + it('refuses a document URL that does not come from the Document Server', async () => { + const { env, app } = await createContext('onlyoffice-ssrf-'); + try { + const target = path.join(env.volumeDir, 'writable.docx'); + await fs.writeFile(target, 'original'); + + const backend = signBackendToken({ absolutePath: target, canWrite: true }); + const dsToken = jwt.sign({ status: 2 }, SECRET, { algorithm: 'HS256' }); + + const response = await request(app) + .post('/api/onlyoffice/callback') + .query({ path: 'writable.docx', backend }) + .set('Authorization', `Bearer ${dsToken}`) + .send({ status: 2, url: 'http://169.254.169.254/latest/meta-data/' }); + + expect(response.body.error).toBe(1); + await expect(fs.readFile(target, 'utf-8')).resolves.toBe('original'); + } finally { + await env.cleanup(); + } + }); + + it('ignores a Document Server token replayed as a backend token', async () => { + const { env, app } = await createContext('onlyoffice-token-type-'); + try { + const target = path.join(env.volumeDir, 'typed.docx'); + await fs.writeFile(target, 'original'); + + // Same secret, no type claim: this must not be accepted as a backend + // context, otherwise any signed payload could name a path to overwrite. + const forged = jwt.sign({ absolutePath: target, canWrite: true }, SECRET, { + algorithm: 'HS256', + }); + const dsToken = jwt.sign({ status: 2 }, SECRET, { algorithm: 'HS256' }); + + const response = await request(app) + .post('/api/onlyoffice/callback') + .query({ path: 'typed.docx', backend: forged }) + .set('Authorization', `Bearer ${dsToken}`) + .send({ status: 2, url: `${DOCUMENT_SERVER}/cache/files/edited.docx` }); + + // Falls back to a real permission check, which fails without a user. + expect(response.body.error).toBe(1); + await expect(fs.readFile(target, 'utf-8')).resolves.toBe('original'); + } finally { + await env.cleanup(); + } + }); + + it('accepts an extra download origin when it is configured', async () => { + const env = await setupTestEnv({ + tag: 'onlyoffice-origins-', + modules: ['src/config/env', 'src/config/index'], + env: { + ONLYOFFICE_URL: DOCUMENT_SERVER, + ONLYOFFICE_SECRET: SECRET, + ONLYOFFICE_DOWNLOAD_ORIGINS: 'http://onlyoffice-internal:80, https://ds.lan', + }, + }); + try { + const { onlyoffice } = env.requireFresh('src/config/index'); + expect(onlyoffice.downloadOrigins).toEqual(['http://onlyoffice-internal', 'https://ds.lan']); + } finally { + await env.cleanup(); + } + }); +}); diff --git a/backend/tests/routes/onlyoffice-coediting.test.js b/backend/tests/routes/onlyoffice-coediting.test.js new file mode 100644 index 000000000..838ec7867 --- /dev/null +++ b/backend/tests/routes/onlyoffice-coediting.test.js @@ -0,0 +1,149 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Two people in the same document have to be given the same key. + * + * The Document Server files an open document under the key it was handed, and + * treats a different key as a different document — a second editing session on + * the same file, invisible to the first, where whoever saves last overwrites the + * other with nothing to warn either of them. + * + * The key used to be computed from the file's mtime and size, so the first save + * changed it: from that moment, everyone arriving got their own session. That is + * the failure this pins. The other half matters just as much — once the document + * is closed the key must change, or the Document Server serves the copy it still + * has cached instead of the file that was saved. + */ + +describe('ONLYOFFICE co-editing', () => { + let env; + let app; + const filename = 'report.docx'; + + const setup = async () => { + env = await setupTestEnv({ + tag: 'onlyoffice-coediting-', + modules: [ + 'src/services/onlyofficeActivityService', + 'src/services/onlyofficeDocumentKeyService', + 'src/routes/onlyoffice', + 'src/services/accessManager', + 'src/middleware/errorHandler', + ], + env: { + PUBLIC_URL: 'https://files.example.com', + ONLYOFFICE_URL: 'http://127.0.0.1:1', + ONLYOFFICE_SECRET: 'onlyoffice-test-secret', + }, + }); + + await fs.writeFile(path.join(env.volumeDir, filename), Buffer.from('original')); + + const routes = env.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + app = createTestApp({ + router: routes, + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + errorHandler, + }); + }; + + /** Open the document, as a client asking for its editor configuration. */ + const open = async () => { + const response = await request(app).post('/api/onlyoffice/config').send({ path: filename }); + expect(response.status).toBe(200); + return { + key: response.body.config.document.key, + sessionId: response.body.forceSaveSessionId, + callbackPath: `/api/onlyoffice/callback${new URL(response.body.config.editorConfig.callbackUrl).search}`, + token: response.body.config.token, + }; + }; + + /** What the client sends once ONLYOFFICE reports the document ready. */ + const declareOpen = (sessionId) => + request(app).post('/api/onlyoffice/session-heartbeat').send({ path: filename, sessionId }); + + /** A save landing on disk while the document is open. */ + const documentSaved = async () => { + await new Promise((resolve) => setTimeout(resolve, 5)); + await fs.writeFile(path.join(env.volumeDir, filename), Buffer.from('edited by the first user')); + }; + + afterEach(async () => { + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('gives the second person the key the first is already using', async () => { + await setup(); + + const first = await open(); + await declareOpen(first.sessionId); + await documentSaved(); + + const second = await open(); + + expect(second.key).toBe(first.key); + }); + + it('still hands out a fresh key once the document has been released', async () => { + await setup(); + + const first = await open(); + await declareOpen(first.sessionId); + await documentSaved(); + + // Status 4: Document Server has closed the document without changes left to + // write. Its cached copy is now the stale one. + const released = await request(app) + .post(first.callbackPath) + .set('Authorization', `Bearer ${first.token}`) + .send({ status: 4, key: first.key }); + expect(released.body).toEqual({ error: 0 }); + + const reopened = await open(); + + expect(reopened.key).not.toBe(first.key); + }); + + it('does not reuse a key for a file that changed while nobody had it open', async () => { + // No one is in the document, so there is no session to protect — and the + // file is not what the Document Server cached. + await setup(); + + const first = await open(); + await documentSaved(); + + const second = await open(); + + expect(second.key).not.toBe(first.key); + }); + + it('keeps everyone together when the document is renamed from the editor', async () => { + await setup(); + + const first = await open(); + await declareOpen(first.sessionId); + + const renamed = await request(app) + .post('/api/onlyoffice/rename') + .send({ path: filename, sessionId: first.sessionId, newName: 'quarterly.docx' }); + expect(renamed.status).toBe(200); + + // Someone opening the document under its new name joins the session that is + // already running, rather than starting a rival one. + const second = await request(app) + .post('/api/onlyoffice/config') + .send({ path: 'quarterly.docx' }); + expect(second.status).toBe(200); + expect(second.body.config.document.key).toBe(first.key); + }); +}); diff --git a/backend/tests/routes/onlyoffice-comment-permission.test.js b/backend/tests/routes/onlyoffice-comment-permission.test.js new file mode 100644 index 000000000..6b8db89de --- /dev/null +++ b/backend/tests/routes/onlyoffice-comment-permission.test.js @@ -0,0 +1,112 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Whether the reader may annotate, and whether the editor will let them. + * + * Two things decide it, and only one of them is obvious. `permissions.comment` + * is the grant — ONLYOFFICE infers it from `edit` when it is missing, which is + * why it is now written down rather than inherited. `editorConfig.mode` is the + * one that silently wins: 'view' loads a viewer, and a viewer has no comment UI + * however the permissions read. A comment-only reader needs `mode: 'edit'` with + * `edit: false`, so the two must not be computed from the same boolean. + * + * Nothing grants comment-only yet. These tests pin the shape that will make it + * a one-line change, and the behaviour that must not drift in the meantime. + */ + +describe('what the editor is told about commenting', () => { + let env; + + const configFor = async ({ readOnly = false, body = {} } = {}) => { + env = await setupTestEnv({ + tag: 'onlyoffice-comment-', + modules: [ + 'src/routes/onlyoffice', + 'src/services/accessManager', + 'src/middleware/errorHandler', + ], + env: { + PUBLIC_URL: 'https://files.example.com', + ONLYOFFICE_URL: 'http://127.0.0.1:1', + ONLYOFFICE_SECRET: 'onlyoffice-test-secret', + ...(readOnly ? { READ_ONLY: 'true' } : {}), + }, + }); + + await fs.writeFile(path.join(env.volumeDir, 'report.docx'), Buffer.from('original')); + + const routes = env.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + const app = createTestApp({ + router: routes, + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + errorHandler, + }); + + const response = await request(app) + .post('/api/onlyoffice/config') + .send({ path: 'report.docx', ...body }); + expect(response.status).toBe(200); + return response.body.config; + }; + + afterEach(async () => { + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('states the comment permission rather than leaving it to be inferred', async () => { + const config = await configFor(); + + expect(config.document.permissions).toHaveProperty('comment'); + }); + + it('lets somebody who can edit comment, in a mode that shows the UI', async () => { + const config = await configFor(); + + expect(config.document.permissions.edit).toBe(true); + expect(config.document.permissions.comment).toBe(true); + expect(config.editorConfig.mode).toBe('edit'); + }); + + it('keeps track changes available to an editor', async () => { + const config = await configFor(); + + expect(config.document.permissions.review).toBe(true); + }); + + /** + * `mode: 'view'` explicitly asks for a viewer, so nothing is offered — this is + * the caller saying "just show it", not an access decision. + */ + it('offers nothing when the caller asked for a viewer', async () => { + const config = await configFor({ body: { mode: 'view' } }); + + expect(config.document.permissions.edit).toBe(false); + expect(config.document.permissions.comment).toBe(false); + expect(config.editorConfig.mode).toBe('view'); + }); + + /** + * The invariant that outlives the current wiring: whatever grants commenting, + * granting it while the mode stays 'view' ships a document nobody can comment + * on and no error to say so. + */ + it('never grants commenting in a mode that cannot show it', async () => { + for (const body of [{}, { mode: 'view' }, { mode: 'edit' }]) { + const config = await configFor({ body }); + if (config.document.permissions.comment) { + expect(config.editorConfig.mode, JSON.stringify(body)).toBe('edit'); + } + await env.cleanup(); + env = null; + } + }); +}); diff --git a/backend/tests/routes/onlyoffice-customization.test.js b/backend/tests/routes/onlyoffice-customization.test.js new file mode 100644 index 000000000..c92bc16af --- /dev/null +++ b/backend/tests/routes/onlyoffice-customization.test.js @@ -0,0 +1,98 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What the editor is told to draw for itself. + * + * These settings are the difference between an editor that behaves like part of + * NextExplorer and one that behaves like an iframe someone dropped in. They are + * easy to lose in a refactor of the config object and produce no error when they + * go missing — the editor simply stops offering the control. + */ + +describe('ONLYOFFICE editor customization', () => { + let env; + let app; + + const setup = async (filename = 'report.docx', body = {}) => { + env = await setupTestEnv({ + tag: 'onlyoffice-customization-', + modules: [ + 'src/routes/onlyoffice', + 'src/services/accessManager', + 'src/middleware/errorHandler', + ], + env: { + PUBLIC_URL: 'https://files.example.com', + ONLYOFFICE_URL: 'http://127.0.0.1:1', + ONLYOFFICE_SECRET: 'onlyoffice-test-secret', + }, + }); + + await fs.writeFile(path.join(env.volumeDir, filename), Buffer.from('original')); + + const routes = env.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + app = createTestApp({ + router: routes, + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + errorHandler, + }); + + const response = await request(app) + .post('/api/onlyoffice/config') + .send({ path: filename, ...body }); + expect(response.status).toBe(200); + return response.body.config; + }; + + afterEach(async () => { + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('asks the editor to draw its own close button', async () => { + const config = await setup(); + + // Without this the editor draws no close control at all, and the overlay + // has no header — the only way out was a button laid over the toolbar. + expect(config.editorConfig.customization.close).toEqual({ visible: true }); + }); + + it('dresses the editor in the theme the client is showing', async () => { + const dark = await setup('dark.docx', { theme: 'dark' }); + expect(dark.editorConfig.customization.uiTheme).toBe('theme-dark'); + + await env.cleanup(); + env = null; + + const light = await setup('light.docx', { theme: 'light' }); + expect(light.editorConfig.customization.uiTheme).toBe('theme-light'); + }); + + it('leaves the theme to the editor when the client sends nothing usable', async () => { + // A client that predates this, or one sending something unexpected, must + // not end up forcing a theme — the editor has a sensible default of its own. + const config = await setup('report.docx', { theme: 'sepia' }); + + expect(config.editorConfig.customization).not.toHaveProperty('uiTheme'); + }); + + it('keeps the customization inside the signed token', async () => { + const config = await setup(); + + // The Document Server reads the config from the token when one is present. + // Settings added to the object after signing are silently ignored, which + // looks exactly like a Document Server that does not support them. + const [, payload] = config.token.split('.'); + const signed = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')); + + expect(signed.editorConfig.customization.close).toEqual({ visible: true }); + }); +}); diff --git a/backend/tests/routes/onlyoffice-document-key.test.js b/backend/tests/routes/onlyoffice-document-key.test.js new file mode 100644 index 000000000..0415213aa --- /dev/null +++ b/backend/tests/routes/onlyoffice-document-key.test.js @@ -0,0 +1,78 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What makes a document the document it is, as far as the Document Server's + * cache is concerned. + * + * The signature is not the key — the key is what everyone in a document shares, + * and it deliberately outlives their saves (see onlyoffice-coediting.test.js). + * The signature is what tells a *new* document from the one that was cached, and + * these are the properties it has to keep. + * + * Previously reproduced inline here, which meant the test could keep passing + * against a copy of the code it was supposed to be pinning. + */ + +describe('ONLYOFFICE document signature', () => { + let env; + let buildSignature; + + const setup = async () => { + env = await setupTestEnv({ + tag: 'onlyoffice-signature-', + modules: ['src/services/onlyofficeDocumentKeyService'], + }); + ({ buildSignature } = env.requireFresh('src/services/onlyofficeDocumentKeyService')); + }; + + const STAT = { mtimeMs: 1_700_000_000_000, ctimeMs: 1_700_000_000_000, size: 4096 }; + + afterEach(async () => { + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('changes when the same file is opened with a different editor', async () => { + // The cache holds the file as one editor prepared it. A drawing once opened + // as a text document kept answering with that failed attempt, from cache, + // long after the mapping was corrected — nothing was reconverted, so nothing + // appeared in the converter logs either. + await setup(); + + expect(buildSignature('drawing.odg', STAT, 'slide')).not.toBe( + buildSignature('drawing.odg', STAT, 'word') + ); + }); + + it('stays the same across opens while nothing changes', async () => { + await setup(); + + expect(buildSignature('report.docx', STAT, 'word')).toBe( + buildSignature('report.docx', STAT, 'word') + ); + }); + + it('changes when the file itself changes', async () => { + await setup(); + + const before = buildSignature('report.docx', STAT, 'word'); + + expect(buildSignature('report.docx', { ...STAT, mtimeMs: STAT.mtimeMs + 1 }, 'word')).not.toBe( + before + ); + expect(buildSignature('report.docx', { ...STAT, size: STAT.size + 1 }, 'word')).not.toBe( + before + ); + }); + + it('separates two files that differ only by path', async () => { + await setup(); + + expect(buildSignature('a/report.docx', STAT, 'word')).not.toBe( + buildSignature('b/report.docx', STAT, 'word') + ); + }); +}); diff --git a/backend/tests/routes/onlyoffice-file-contract.test.js b/backend/tests/routes/onlyoffice-file-contract.test.js new file mode 100644 index 000000000..4ce3a51d9 --- /dev/null +++ b/backend/tests/routes/onlyoffice-file-contract.test.js @@ -0,0 +1,260 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The endpoint Document Server fetches a document from. + * + * It is exempt from authentication — the authentication middleware lets + * `/api/onlyoffice/file` through, because the caller is a separate server with + * no session of ours — so what stands in its place is all that decides which + * file goes out. + * + * That deserved checking rather than assuming, because the token the editor + * config hands to the browser is signed with the same secret and says nothing + * about which file it is for. It turns out not to be enough on its own: with no + * backend context, resolution runs with no user and is refused. The first test + * below is that fact, pinned, because it is the one holding the door shut. + */ + +let currentEnv; + +const setup = async () => { + currentEnv = await setupTestEnv({ + tag: 'onlyoffice-file-', + env: { + ONLYOFFICE_URL: 'https://ds.example.com', + ONLYOFFICE_SECRET: 'shared-ds-secret', + }, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/routes/onlyoffice', + 'src/middleware/errorHandler', + 'src/services/accessManager', + 'src/utils/pathUtils', + ], + }); + + const routes = currentEnv.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const { onlyoffice } = currentEnv.requireFresh('src/config/index'); + + await fs.mkdir(path.join(currentEnv.volumeDir, 'Private'), { recursive: true }); + const secretFile = path.join(currentEnv.volumeDir, 'Private', 'salaries.xlsx'); + const ordinaryFile = path.join(currentEnv.volumeDir, 'report.docx'); + await fs.writeFile(secretFile, 'CONFIDENTIAL'); + await fs.writeFile(ordinaryFile, 'the report'); + + // No user: the route is reached without a session, as Document Server does. + const app = express(); + app.use('/api', routes); + app.use(errorHandler); + + /** The shape of token the editor config hands to the browser. */ + const dsToken = (secret = onlyoffice.secret) => + jwt.sign({ document: { key: 'anything' } }, secret, { algorithm: 'HS256' }); + + /** The one that actually names a file, and is what authorises the fetch. */ + const backendToken = (claims = {}, secret = onlyoffice.secret) => + jwt.sign({ typ: 'nextexplorer-backend', absolutePath: ordinaryFile, ...claims }, secret, { + algorithm: 'HS256', + }); + + return { app, secretFile, ordinaryFile, dsToken, backendToken }; +}; + +/** + * A document is served with its own content type, and supertest will not buffer + * a body it has no parser for — the response arrives with an empty `text` and a + * green status, which is a test agreeing with nothing. + */ +const collectBody = (res, callback) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => callback(null, Buffer.concat(chunks).toString('utf8'))); +}; + +const fetchFile = (app, { query = {}, token } = {}) => { + const call = request(app) + .get('/api/onlyoffice/file') + .query(query) + .buffer(true) + .parse(collectBody); + if (token) call.set('Authorization', `Bearer ${token}`); + return call; +}; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +describe('a request with nothing but a signed token', () => { + it('is refused without one at all', async () => { + const { app } = await setup(); + + expect((await fetchFile(app, { query: { path: 'report.docx' } })).status).toBe(401); + }); + + /** + * And says which of the two it was. + * + * Both refusals answer 401, so a test that reads only the status cannot tell + * the absent token from the wrong one — removing the check for the first + * changes nothing it can see. The distinction is worth keeping: somebody + * wiring up a Document Server needs to know whether their token never + * arrived or arrived wrong. + */ + it('says the token was missing rather than wrong', async () => { + const { app } = await setup(); + + const response = await fetchFile(app, { query: { path: 'report.docx' } }); + + // The parser above returns every body as text, errors included. + expect(String(response.body)).toMatch(/missing token/i); + }); + + it('says the token was wrong when it is', async () => { + const { app, dsToken } = await setup(); + + const response = await fetchFile(app, { + query: { path: 'report.docx' }, + token: dsToken('not-the-secret'), + }); + + expect(String(response.body)).toMatch(/invalid token/i); + }); + + it('is refused when the token was signed with another secret', async () => { + const { app, dsToken } = await setup(); + + const response = await fetchFile(app, { + query: { path: 'report.docx' }, + token: dsToken('not-the-secret'), + }); + + expect(response.status).toBe(401); + }); + + /** + * The door this holds shut. A signed token proves the caller shares the + * secret with Document Server — and the editor config hands one to every + * browser that opens a document. It says nothing about which file, so on its + * own it resolves with no user behind it and is refused. + */ + it('is refused when the token names no file', async () => { + const { app, dsToken } = await setup(); + + const response = await fetchFile(app, { + query: { path: 'Private/salaries.xlsx' }, + token: dsToken(), + }); + + expect(response.status).toBe(403); + }); + + it('sends nothing of the file it refused', async () => { + const { app, dsToken } = await setup(); + + const response = await fetchFile(app, { + query: { path: 'Private/salaries.xlsx' }, + token: dsToken(), + }); + + expect(String(response.body || '')).not.toContain('CONFIDENTIAL'); + }); + + it('is refused with no path to go on', async () => { + const { app, dsToken } = await setup(); + + expect((await fetchFile(app, { token: dsToken() })).status).toBe(400); + }); +}); + +describe('a request carrying the token that names a file', () => { + it('is answered with that file', async () => { + const { app, dsToken, backendToken } = await setup(); + + const response = await fetchFile(app, { + query: { path: 'report.docx', backend: backendToken() }, + token: dsToken(), + }); + + expect(response.status).toBe(200); + expect(String(response.body || '')).toBe('the report'); + }); + + /** + * The token decides, not the query. Otherwise the path beside it would be a + * way to ask for something else while carrying an authorisation for this. + */ + it('ignores a different file named in the query', async () => { + const { app, dsToken, backendToken } = await setup(); + + const response = await fetchFile(app, { + query: { path: 'Private/salaries.xlsx', backend: backendToken() }, + token: dsToken(), + }); + + expect(String(response.body || '')).toBe('the report'); + expect(String(response.body || '')).not.toContain('CONFIDENTIAL'); + }); + + it('falls back to being refused when the token declares another kind', async () => { + const { app, secretFile, dsToken, backendToken } = await setup(); + + const response = await fetchFile(app, { + query: { + path: 'Private/salaries.xlsx', + backend: backendToken({ typ: 'something-else', absolutePath: secretFile }), + }, + token: dsToken(), + }); + + expect(response.status).toBe(403); + }); + + it('falls back to being refused when it was signed with another secret', async () => { + const { app, secretFile, dsToken, backendToken } = await setup(); + + const response = await fetchFile(app, { + query: { + path: 'Private/salaries.xlsx', + backend: backendToken({ absolutePath: secretFile }, 'not-the-secret'), + }, + token: dsToken(), + }); + + expect(response.status).toBe(403); + }); + + it('falls back to being refused when it names no file', async () => { + const { app, dsToken, backendToken } = await setup(); + + const response = await fetchFile(app, { + query: { path: 'report.docx', backend: backendToken({ absolutePath: '' }) }, + token: dsToken(), + }); + + expect(response.status).toBe(403); + }); + + it('refuses a directory', async () => { + const { app, dsToken, backendToken } = await setup(); + const directory = path.join(currentEnv.volumeDir, 'Private'); + + const response = await fetchFile(app, { + query: { path: 'Private', backend: backendToken({ absolutePath: directory }) }, + token: dsToken(), + }); + + expect(response.status).toBe(400); + }); +}); diff --git a/backend/tests/routes/onlyoffice-mentions.test.js b/backend/tests/routes/onlyoffice-mentions.test.js new file mode 100644 index 000000000..cb60c1604 --- /dev/null +++ b/backend/tests/routes/onlyoffice-mentions.test.js @@ -0,0 +1,117 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Mentions in comments. + * + * ONLYOFFICE asks for everyone who can be mentioned and filters the list in the + * editor, so this route answers with names and addresses rather than running a + * search. That makes it a directory listing, and the only interesting question + * about a directory listing is who is allowed to read it. + */ + +describe('ONLYOFFICE mentions', () => { + let env; + let app; + const filename = 'report.docx'; + + const setup = async ({ user } = {}) => { + env = await setupTestEnv({ + tag: 'onlyoffice-mentions-', + modules: [ + 'src/routes/onlyoffice', + 'src/services/accessManager', + 'src/services/userSearchService', + 'src/services/db', + 'src/middleware/errorHandler', + ], + env: { + PUBLIC_URL: 'https://files.example.com', + ONLYOFFICE_URL: 'http://127.0.0.1:1', + ONLYOFFICE_SECRET: 'onlyoffice-test-secret', + }, + }); + + await fs.writeFile(path.join(env.volumeDir, filename), Buffer.from('original')); + + const routes = env.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + app = createTestApp({ + router: routes, + mountPath: '/api', + user: user === undefined ? { id: 'admin-user', roles: ['admin'] } : user, + errorHandler, + }); + }; + + afterEach(async () => { + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('answers with the list of people who can be mentioned', async () => { + await setup(); + + const response = await request(app).get('/api/onlyoffice/users'); + + expect(response.status).toBe(200); + expect(Array.isArray(response.body.users)).toBe(true); + // The seeded administrator is enough to show the shape reaches the editor: + // it reads id and name, and matches on email when notifying. + for (const user of response.body.users) { + expect(user).toHaveProperty('id'); + expect(user).toHaveProperty('name'); + expect(user).toHaveProperty('email'); + } + }); + + it('keeps the user directory away from guests', async () => { + // A guest editing through a share link has no reason to receive every + // account name and address on the server. + await setup({ user: null }); + + const response = await request(app).get('/api/onlyoffice/users'); + + expect(response.status).toBe(403); + }); + + it('records a mention and says plainly that nothing was delivered', async () => { + await setup(); + + const response = await request(app) + .post('/api/onlyoffice/notify') + .send({ path: filename, emails: ['someone@example.com'], comment: 'have a look' }); + + // Answering `{delivered: false}` rather than an error: the comment itself + // was saved by the editor, only the notification has nowhere to go. + expect(response.status).toBe(200); + expect(response.body).toEqual({ delivered: false }); + }); + + it('refuses a mention that names no document', async () => { + // The path is what the access check runs on, so a request without one has + // to be turned away rather than recorded against nothing. + await setup(); + + const response = await request(app) + .post('/api/onlyoffice/notify') + .send({ emails: ['someone@example.com'] }); + + expect(response.status).toBe(400); + }); + + it('keeps the notify route away from guests', async () => { + await setup({ user: null }); + + const response = await request(app) + .post('/api/onlyoffice/notify') + .send({ path: filename, emails: [] }); + + expect(response.status).toBe(403); + }); +}); diff --git a/backend/tests/routes/onlyoffice-rename.test.js b/backend/tests/routes/onlyoffice-rename.test.js new file mode 100644 index 000000000..61cc46fb2 --- /dev/null +++ b/backend/tests/routes/onlyoffice-rename.test.js @@ -0,0 +1,189 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import path from 'node:path'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Renaming from the editor's title bar is an ordinary rename with one thing + * that is not ordinary: an editor is still open on the file. + * + * The Document Server holds a token minted when the editor opened, naming the + * path as it was then, and returns it unchanged with every save for as long as + * the session lasts. If nothing follows the file, the next autosave recreates + * the old name beside the new one — two documents, neither of them wrong from + * where it was written. + */ + +describe('ONLYOFFICE rename', () => { + let env; + let documentServer; + let app; + let port; + let sessionId; + let callbackPath; + let callbackToken; + let documentKey; + + const setup = async (filename = 'report.docx') => { + documentServer = http.createServer((req, res) => { + if (req.method === 'GET' && req.url.startsWith('/saved')) { + res.setHeader('Content-Type', 'application/octet-stream'); + res.end('edited contents'); + return; + } + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 0 })); + }); + await new Promise((resolve, reject) => { + documentServer.once('error', reject); + documentServer.listen(0, '127.0.0.1', resolve); + }); + ({ port } = documentServer.address()); + + env = await setupTestEnv({ + tag: 'onlyoffice-rename-', + modules: [ + 'src/routes/onlyoffice', + 'src/services/accessManager', + 'src/services/folderSizeHooks', + 'src/middleware/errorHandler', + ], + env: { + PUBLIC_URL: 'https://files.example.com', + ONLYOFFICE_URL: `http://127.0.0.1:${port}`, + ONLYOFFICE_SECRET: 'onlyoffice-test-secret', + }, + }); + + await fs.writeFile(path.join(env.volumeDir, filename), Buffer.from('original')); + + const routes = env.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + app = createTestApp({ + router: routes, + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + errorHandler, + }); + + const config = await request(app).post('/api/onlyoffice/config').send({ path: filename }); + expect(config.status).toBe(200); + sessionId = config.body.forceSaveSessionId; + documentKey = config.body.config.document.key; + const callbackUrl = new URL(config.body.config.editorConfig.callbackUrl); + callbackPath = `/api/onlyoffice/callback${callbackUrl.search}`; + callbackToken = config.body.config.token; + }; + + /** What the Document Server sends when it has a saved document waiting. */ + const documentServerSave = () => + request(app) + .post(callbackPath) + .set('Authorization', `Bearer ${callbackToken}`) + .send({ + status: 2, + key: documentKey, + url: `http://127.0.0.1:${port}/saved.docx`, + }); + + afterEach(async () => { + if (documentServer) { + await new Promise((resolve) => documentServer.close(resolve)); + documentServer = null; + } + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('renames the open document', async () => { + await setup(); + + const response = await request(app) + .post('/api/onlyoffice/rename') + .send({ path: 'report.docx', sessionId, newName: 'quarterly.docx' }); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ path: 'quarterly.docx', name: 'quarterly.docx' }); + expect(await fs.readFile(path.join(env.volumeDir, 'quarterly.docx'), 'utf8')).toBe('original'); + await expect(fs.access(path.join(env.volumeDir, 'report.docx'))).rejects.toThrow(); + }); + + it('writes a later save to the new name, not the old one', async () => { + await setup(); + await request(app) + .post('/api/onlyoffice/rename') + .send({ path: 'report.docx', sessionId, newName: 'quarterly.docx' }); + + // The Document Server still holds the token naming report.docx. + const saved = await documentServerSave(); + expect(saved.status).toBe(200); + + expect(await fs.readFile(path.join(env.volumeDir, 'quarterly.docx'), 'utf8')).toBe( + 'edited contents' + ); + // The whole point: the old name must not come back. + await expect(fs.access(path.join(env.volumeDir, 'report.docx'))).rejects.toThrow(); + }); + + it('falls back to the token when the session is gone', async () => { + await setup(); + + // No rename, and a token from a session this process never recorded — what + // a restart mid-edit looks like. The save must still land. + const orphanToken = jwt.sign( + { + typ: 'nextexplorer-backend', + absolutePath: path.join(env.volumeDir, 'report.docx'), + logicalPath: 'report.docx', + space: 'volume', + canWrite: true, + sessionId: 'session-that-no-longer-exists', + userId: 'admin-user', + }, + 'onlyoffice-test-secret', + { algorithm: 'HS256', expiresIn: 3600 } + ); + + const saved = await request(app) + .post(`/api/onlyoffice/callback?path=report.docx&backend=${orphanToken}`) + .set('Authorization', `Bearer ${callbackToken}`) + .send({ status: 2, key: documentKey, url: `http://127.0.0.1:${port}/saved.docx` }); + + expect(saved.status).toBe(200); + expect(await fs.readFile(path.join(env.volumeDir, 'report.docx'), 'utf8')).toBe( + 'edited contents' + ); + }); + + it('refuses a rename without a valid editing session', async () => { + await setup(); + + const response = await request(app) + .post('/api/onlyoffice/rename') + .send({ path: 'report.docx', sessionId: 'not-a-session', newName: 'stolen.docx' }); + + expect(response.status).toBe(403); + await expect(fs.access(path.join(env.volumeDir, 'stolen.docx'))).rejects.toThrow(); + }); + + it('refuses a name that would leave the folder, and one already taken', async () => { + await setup(); + await fs.writeFile(path.join(env.volumeDir, 'taken.docx'), Buffer.from('someone else')); + + const escaping = await request(app) + .post('/api/onlyoffice/rename') + .send({ path: 'report.docx', sessionId, newName: '../escaped.docx' }); + expect(escaping.status).toBe(400); + + const conflicting = await request(app) + .post('/api/onlyoffice/rename') + .send({ path: 'report.docx', sessionId, newName: 'taken.docx' }); + expect(conflicting.status).toBe(409); + expect(await fs.readFile(path.join(env.volumeDir, 'taken.docx'), 'utf8')).toBe('someone else'); + }); +}); diff --git a/backend/tests/routes/onlyoffice-save-as.test.js b/backend/tests/routes/onlyoffice-save-as.test.js new file mode 100644 index 000000000..3c4ac8962 --- /dev/null +++ b/backend/tests/routes/onlyoffice-save-as.test.js @@ -0,0 +1,199 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import path from 'node:path'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * "Save as" hands the backend a URL and a name, both chosen inside an editor + * running on a different origin. Neither can be taken at face value: + * + * - the URL is fetched by the server, so it is only ever followed when it + * points at the configured Document Server. Otherwise this route would fetch + * any address an editor asked for; + * - the name lands on the filesystem, so it goes through the same validation + * as a name typed into the app, and never overwrites an existing file. + */ + +describe('ONLYOFFICE save as', () => { + let env; + let documentServer; + let app; + let filename; + let port; + // Runs while the Document Server is half way through its answer. + let whileConverting = null; + + const setup = async () => { + documentServer = http.createServer(async (req, res) => { + if (req.method === 'GET' && req.url.startsWith('/converted')) { + res.setHeader('Content-Type', 'application/octet-stream'); + res.write('converted '); + if (whileConverting) await whileConverting(); + res.end('document'); + return; + } + res.statusCode = 404; + res.end(); + }); + await new Promise((resolve, reject) => { + documentServer.once('error', reject); + documentServer.listen(0, '127.0.0.1', resolve); + }); + ({ port } = documentServer.address()); + + env = await setupTestEnv({ + tag: 'onlyoffice-save-as-', + modules: [ + 'src/routes/onlyoffice', + 'src/services/accessManager', + 'src/services/folderSizeHooks', + 'src/middleware/errorHandler', + ], + env: { + PUBLIC_URL: 'https://files.example.com', + ONLYOFFICE_URL: `http://127.0.0.1:${port}`, + ONLYOFFICE_SECRET: 'onlyoffice-test-secret', + }, + }); + + filename = 'report.docx'; + await fs.writeFile(path.join(env.volumeDir, filename), Buffer.from('original')); + + const routes = env.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + app = createTestApp({ + router: routes, + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + errorHandler, + }); + }; + + const saveAs = (body) => request(app).post('/api/onlyoffice/save-as').send(body); + + afterEach(async () => { + whileConverting = null; + if (documentServer) { + await new Promise((resolve) => documentServer.close(resolve)); + documentServer = null; + } + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('writes the converted document beside the original', async () => { + await setup(); + + const response = await saveAs({ + path: filename, + url: `http://127.0.0.1:${port}/converted.pdf`, + title: 'report.pdf', + }); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ name: 'report.pdf', path: 'report.pdf' }); + expect(await fs.readFile(path.join(env.volumeDir, 'report.pdf'), 'utf8')).toBe( + 'converted document' + ); + // The original is untouched: this saves a copy, it does not move anything. + expect(await fs.readFile(path.join(env.volumeDir, filename), 'utf8')).toBe('original'); + }); + + it('refuses a URL that does not come from the Document Server', async () => { + await setup(); + + const response = await saveAs({ + path: filename, + url: 'http://169.254.169.254/latest/meta-data/', + title: 'stolen.pdf', + }); + + expect(response.status).toBe(403); + await expect(fs.access(path.join(env.volumeDir, 'stolen.pdf'))).rejects.toThrow(); + }); + + it('never overwrites an existing file', async () => { + await setup(); + await fs.writeFile(path.join(env.volumeDir, 'report.pdf'), Buffer.from('do not lose me')); + + const response = await saveAs({ + path: filename, + url: `http://127.0.0.1:${port}/converted.pdf`, + title: 'report.pdf', + }); + + expect(response.status).toBe(200); + expect(response.body.name).toBe('report (1).pdf'); + expect(await fs.readFile(path.join(env.volumeDir, 'report.pdf'), 'utf8')).toBe( + 'do not lose me' + ); + }); + + it('never replaces a file that arrives under the name while the document downloads', async () => { + await setup(); + const target = path.join(env.volumeDir, 'report.pdf'); + const theirs = Buffer.from('saved over SMB while the Document Server answered\n'); + let arrived = false; + whileConverting = async () => { + arrived = true; + await fs.writeFile(target, theirs); + }; + + const response = await saveAs({ + path: filename, + url: `http://127.0.0.1:${port}/converted.pdf`, + title: 'report.pdf', + }); + + expect(arrived).toBe(true); + expect(response.status).toBe(200); + // The answer names the file actually written, not the one asked for. + expect(response.body).toMatchObject({ name: 'report (1).pdf', path: 'report (1).pdf' }); + expect(await fs.readFile(target)).toEqual(theirs); + expect(await fs.readFile(path.join(env.volumeDir, 'report (1).pdf'), 'utf8')).toBe( + 'converted document' + ); + expect( + (await fs.readdir(env.volumeDir)).filter((name) => name.includes('.onlyoffice-')) + ).toEqual([]); + }); + + it('refuses a title that tries to leave the folder', async () => { + await setup(); + + for (const title of ['../escaped.pdf', 'nested/escaped.pdf', '']) { + const response = await saveAs({ + path: filename, + url: `http://127.0.0.1:${port}/converted.pdf`, + title, + }); + // Named, so a failure says which title got through rather than leaving + // three candidates and a line number. + expect(`${title || ''}: ${response.status}`).toBe(`${title || ''}: 400`); + } + + await expect(fs.access(path.join(env.volumeDir, '..', 'escaped.pdf'))).rejects.toThrow(); + }); + + it('saves into the folder the document lives in, not the volume root', async () => { + await setup(); + await fs.mkdir(path.join(env.volumeDir, 'reports'), { recursive: true }); + await fs.writeFile(path.join(env.volumeDir, 'reports', 'q4.docx'), Buffer.from('original')); + + const response = await saveAs({ + path: 'reports/q4.docx', + url: `http://127.0.0.1:${port}/converted.pdf`, + title: 'q4.pdf', + }); + + expect(response.status).toBe(200); + expect(response.body.path).toBe('reports/q4.pdf'); + expect(await fs.readFile(path.join(env.volumeDir, 'reports', 'q4.pdf'), 'utf8')).toBe( + 'converted document' + ); + }); +}); diff --git a/backend/tests/routes/onlyoffice-session-end.test.js b/backend/tests/routes/onlyoffice-session-end.test.js new file mode 100644 index 000000000..8e9e203bb --- /dev/null +++ b/backend/tests/routes/onlyoffice-session-end.test.js @@ -0,0 +1,179 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import path from 'node:path'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Ending an editing session in one request. + * + * Closing the panel over a folder could afford two calls — ask for a last + * save, wait until this server has accepted it, then end the session. A + * browser tab being closed can wait for nothing: whatever is sent at that + * moment is sent in one breath, and a second request that depended on the + * first would arrive in whichever order the network felt like, or not at all. + * + * So the order lives here, and both ways of closing use this one route. What + * is pinned below is that one call does both things, and that it still ends + * the session in every case where there is nothing left to save — because a + * session that does not end is a document reported as being edited by somebody + * who left. + */ + +describe('ending an ONLYOFFICE editing session', () => { + let env; + let app; + let documentServer = null; + const folder = 'Docs'; + const filename = 'report.docx'; + const documentPath = `${folder}/${filename}`; + + const buildApp = ({ user } = {}) => { + const routes = env.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + return createTestApp({ + router: routes, + mountPath: '/api', + user: user || { id: 'admin-user', roles: ['admin'] }, + errorHandler, + }); + }; + + /** + * A Document Server that accepts a force-save command and says no more. + * + * A save is pending from the moment the command is accepted until the + * document comes back on the callback — which is the window the coalescing + * lives in. A refused connection closes that window before it opens, which + * is why the test below needs something that answers. + */ + const acceptingDocumentServer = async () => { + documentServer = http.createServer((_req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json', Connection: 'close' }); + res.end(JSON.stringify({ error: 0 })); + }); + await new Promise((resolve) => documentServer.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${documentServer.address().port}`; + }; + + const setup = async (options = {}) => { + env = await setupTestEnv({ + tag: 'onlyoffice-session-end-', + env: { + PUBLIC_URL: 'https://files.example.com', + ONLYOFFICE_URL: options.documentServerUrl || 'http://127.0.0.1:1', + ONLYOFFICE_SECRET: 'onlyoffice-test-secret', + }, + }); + + await fs.mkdir(path.join(env.volumeDir, folder), { recursive: true }); + await fs.writeFile(path.join(env.volumeDir, documentPath), Buffer.from('original')); + + app = buildApp(options); + + const config = await request(app).post('/api/onlyoffice/config').send({ path: documentPath }); + expect(config.status).toBe(200); + return config.body.forceSaveSessionId; + }; + + const end = (sessionId, body = {}) => + request(app) + .post('/api/onlyoffice/session-end') + .send({ path: documentPath, sessionId, ...body }); + + const heartbeat = (sessionId) => + request(app).post('/api/onlyoffice/session-heartbeat').send({ path: documentPath, sessionId }); + + afterEach(async () => { + if (documentServer) { + documentServer.closeAllConnections?.(); + await new Promise((resolve) => documentServer.close(resolve)); + documentServer = null; + } + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('does both in one request: the last save queued, the session ended', async () => { + const sessionId = await setup(); + + const ended = await end(sessionId); + + expect(ended.status).toBe(200); + expect(ended.body.ended).toBe(true); + // Both, from one call. What the order between them buys is only visible + // from the client — a tab on its way out cannot make two calls, and this + // is the whole reason the route exists. + expect(ended.body.flushed).toBe(true); + expect(ended.body.requestId).toEqual(expect.stringContaining('force-save')); + + // And the session is gone: anything still holding it is refused. + expect((await heartbeat(sessionId)).status).toBe(403); + }); + + it('coalesces with a save already on its way rather than asking twice', async () => { + const sessionId = await setup({ documentServerUrl: await acceptingDocumentServer() }); + + const first = await request(app) + .post('/api/onlyoffice/force-save') + .send({ path: documentPath, sessionId, reason: 'auto' }); + expect(first.status).toBe(202); + + const ended = await end(sessionId); + + expect(ended.status).toBe(200); + // The same request, not a second one: two saves of the same document + // racing each other is how the older one wins. + expect(ended.body.requestId).toBe(first.body.requestId); + }); + + it('still ends the session when write access went away under the editor', async () => { + const sessionId = await setup({ user: { id: 'writer', roles: ['user'] } }); + + // The rules change while the document is open: what was writable a minute + // ago is not. There is nothing left to save — and the close still has to + // happen, or the document stays marked as being edited by somebody who + // closed their browser long ago. + const accessControl = env.requireFresh('src/services/accessControlService'); + await accessControl.setRules([{ path: `/${folder}`, permissions: 'ro', recursive: true }]); + + const ended = await end(sessionId); + + expect(ended.status).toBe(200); + expect(ended.body).toMatchObject({ ended: true, flushed: false }); + expect((await heartbeat(sessionId)).status).toBe(403); + }); + + it('refuses a session that is not this document’s, and ends nothing', async () => { + const sessionId = await setup(); + await fs.writeFile(path.join(env.volumeDir, folder, 'other.docx'), Buffer.from('other')); + + const wrongDocument = await request(app) + .post('/api/onlyoffice/session-end') + .send({ path: `${folder}/other.docx`, sessionId }); + expect(wrongDocument.status).toBe(403); + + expect((await end('not-a-session')).status).toBe(403); + + for (const body of [{ sessionId: '' }, { sessionId: 42 }]) { + const refused = await request(app) + .post('/api/onlyoffice/session-end') + .send({ path: documentPath, ...body }); + expect(refused.status).toBe(400); + } + + // None of that touched the real session, which still answers. + expect((await heartbeat(sessionId)).status).toBe(200); + }); + + it('is the same close twice: ending it again changes nothing', async () => { + const sessionId = await setup(); + + expect((await end(sessionId)).status).toBe(200); + // A beacon can arrive twice — a tab closed while its panel was closing. + expect((await end(sessionId)).status).toBe(403); + }); +}); diff --git a/backend/tests/routes/onlyoffice-session-persistence.test.js b/backend/tests/routes/onlyoffice-session-persistence.test.js new file mode 100644 index 000000000..7177c34e5 --- /dev/null +++ b/backend/tests/routes/onlyoffice-session-persistence.test.js @@ -0,0 +1,142 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * An editing session has to outlive the process. + * + * The Document Server is handed a token when the editor opens and returns it + * unchanged with every save, so the token says where the document *was*. The + * session is what knows where it is now — and while it was held in memory, a + * restart in the middle of an edit lost that: the next save landed under the old + * name, recreating a file the user had renamed minutes earlier, with no error + * anywhere to explain it. + * + * The restart is simulated by rebuilding the router from a cleared module cache, + * which is what a fresh process would do, against the same database. + */ + +describe('ONLYOFFICE session persistence', () => { + let env; + let app; + const filename = 'report.docx'; + + const buildApp = () => { + const routes = env.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + return createTestApp({ + router: routes, + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + errorHandler, + }); + }; + + const setup = async () => { + env = await setupTestEnv({ + tag: 'onlyoffice-session-persistence-', + modules: [ + 'src/services/onlyofficeEditorSessionService', + 'src/routes/onlyoffice', + 'src/services/accessManager', + 'src/middleware/errorHandler', + ], + env: { + PUBLIC_URL: 'https://files.example.com', + ONLYOFFICE_URL: 'http://127.0.0.1:1', + ONLYOFFICE_SECRET: 'onlyoffice-test-secret', + }, + }); + + await fs.writeFile(path.join(env.volumeDir, filename), Buffer.from('original')); + app = buildApp(); + + const config = await request(app).post('/api/onlyoffice/config').send({ path: filename }); + expect(config.status).toBe(200); + return config.body.forceSaveSessionId; + }; + + afterEach(async () => { + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('still knows the session after a restart', async () => { + const sessionId = await setup(); + + app = buildApp(); + + const heartbeat = await request(app) + .post('/api/onlyoffice/session-heartbeat') + .send({ path: filename, sessionId }); + + expect(heartbeat.status).toBe(200); + expect(heartbeat.body).toEqual({ active: true }); + }); + + it('remembers a rename made before the restart', async () => { + const sessionId = await setup(); + + const renamed = await request(app) + .post('/api/onlyoffice/rename') + .send({ path: filename, sessionId, newName: 'quarterly.docx' }); + expect(renamed.status).toBe(200); + expect(renamed.body.path).toBe('quarterly.docx'); + + app = buildApp(); + + // The session is what a save consults to find the document. Asking under + // the new name is what the client does after a rename, and it has to be the + // session that answers — the token still names the old file. + const heartbeat = await request(app) + .post('/api/onlyoffice/session-heartbeat') + .send({ path: 'quarterly.docx', sessionId }); + + expect(heartbeat.status).toBe(200); + }); + + it('refuses a session that belongs to someone else', async () => { + // Stored sessions are handed out by id, so ownership has to be checked on + // every use rather than trusted from whoever holds the identifier. + const sessionId = await setup(); + + const routes = env.requireFresh('src/routes/onlyoffice'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + const otherUsersApp = createTestApp({ + router: routes, + mountPath: '/api', + user: { id: 'someone-else', roles: ['admin'] }, + errorHandler, + }); + + const heartbeat = await request(otherUsersApp) + .post('/api/onlyoffice/session-heartbeat') + .send({ path: filename, sessionId }); + + expect(heartbeat.status).toBe(403); + }); + + it('refuses a session that was closed', async () => { + const sessionId = await setup(); + + const closed = await request(app) + .post('/api/onlyoffice/session-end') + .send({ path: filename, sessionId }); + // Asserted, not assumed. Everything below only means anything if the close + // actually happened, and a close that quietly failed would leave the + // session answering — reported as "the heartbeat was not refused", which + // sends whoever reads it looking at the wrong route. + expect(closed.status).toBe(200); + app = buildApp(); + + const heartbeat = await request(app) + .post('/api/onlyoffice/session-heartbeat') + .send({ path: filename, sessionId }); + + expect(heartbeat.status).toBe(403); + }); +}); diff --git a/backend/tests/routes/passkeys.test.js b/backend/tests/routes/passkeys.test.js new file mode 100644 index 000000000..5d8cdcb91 --- /dev/null +++ b/backend/tests/routes/passkeys.test.js @@ -0,0 +1,749 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import express from 'express'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +const { ALGORITHMS, createCredential, signAssertion } = require('../helpers/soft-authenticator'); + +/** + * Signing in with a passkey, through the routes that do it. + * + * The authenticator is software (tests/helpers/soft-authenticator.js) and the + * server is wired the way the application wires it — session store, auth + * middleware, routes — because what is pinned here is what a passkey opens. + * `/api/users/shareable` is the plainest thing that answers whether anything + * is open at all. + * + * PUBLIC_URL is set, so the site a passkey is bound to is settled and the + * ceremonies can be written down rather than discovered. One test below takes + * it away again, to hold the other half: an installation that configured + * nothing answers for the name the request arrived on. + * + * Passwords are hashed with bcrypt at cost 12, hence the timeout. + */ + +const PASSWORD = 'secret123'; +const PUBLIC_URL = 'https://files.example.test'; + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const build = async (env = {}) => { + currentEnv = await setupTestEnv({ + tag: 'passkeys-', + env: { AUTH_ENABLED: 'true', AUTH_MODE: 'local', PUBLIC_URL, ...env }, + }); + const { configureSession } = currentEnv.requireFresh('src/middleware/session'); + const authMiddleware = currentEnv.requireFresh('src/middleware/authMiddleware'); + const authRoutes = currentEnv.requireFresh('src/routes/auth'); + const userRoutes = currentEnv.requireFresh('src/routes/users'); + const { errorHandler, notFoundHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const { totpCode } = currentEnv.requireFresh('src/utils/totp'); + + const app = express(); + app.use(express.json()); + app.use(cookieParser()); + configureSession(app); + app.use(authMiddleware); + app.use('/api/auth', authRoutes); + app.use('/api', userRoutes); + app.use(notFoundHandler); + app.use(errorHandler); + + /** + * The same routes, reached by somebody the identity provider signed in. + * + * `req.user` without `session.localUserId` is what that looks like from + * inside a route, and it is the case the passkey routes have to tell apart + * from a password sign-in. + */ + const appForProviderSession = (userId) => { + const provider = express(); + provider.use(express.json()); + provider.use(cookieParser()); + configureSession(provider); + provider.use((req, _res, next) => { + req.oidc = { isAuthenticated: () => true }; + req.user = { id: userId, roles: ['user'] }; + next(); + }); + provider.use('/api/auth', authRoutes); + provider.use(notFoundHandler); + provider.use(errorHandler); + return provider; + }; + + return { app, totpCode, appForProviderSession }; +}; + +/** The first administrator, signed in through the setup. */ +const setUpOwner = async (app) => { + const browser = request.agent(app); + const response = await browser + .post('/api/auth/setup') + .send({ email: 'owner@example.com', username: 'owner', password: PASSWORD }); + expect(response.status).toBe(201); + return browser; +}; + +const signedIn = async (browser) => (await browser.get('/api/users/shareable')).status === 200; + +/** Make a passkey on the signed-in account, the way a browser would. */ +const addPasskey = async (browser, { name, ...options } = {}) => { + const started = await browser.post('/api/auth/passkeys/register/start').send({}); + expect(started.status).toBe(200); + + const credential = createCredential({ + challenge: started.body.options.challenge, + rpId: started.body.options.rp.id, + origin: started.body.origins[0], + ...options, + }); + const finished = await browser.post('/api/auth/passkeys/register/finish').send({ + name, + response: { + attestationObject: credential.attestationObject.toString('base64url'), + clientDataJSON: credential.clientDataJSON.toString('base64url'), + transports: ['internal', 'hybrid'], + }, + }); + + return { credential, started, finished }; +}; + +/** Sign in with one, from a browser that is signed out. */ +const signInWith = async (browser, credential, options = {}) => { + const started = await browser.post('/api/auth/login/passkey/start').send({}); + expect(started.status).toBe(200); + + const assertion = signAssertion({ + credential, + challenge: started.body.options.challenge, + rpId: started.body.options.rpId, + origin: started.body.origins[0], + signCount: 1, + ...options, + }); + + return browser.post('/api/auth/login/passkey/finish').send({ + response: { + id: credential.credentialId.toString('base64url'), + authenticatorData: assertion.authenticatorData.toString('base64url'), + clientDataJSON: assertion.clientDataJSON.toString('base64url'), + signature: assertion.signature.toString('base64url'), + }, + }); +}; + +describe('adding a passkey', { timeout: 30_000 }, () => { + it('keeps it, and lists it under the name it was given', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + + const { finished } = await addPasskey(browser, { name: 'The yellow key' }); + + expect(finished.status).toBe(201); + expect(finished.body.passkey).toMatchObject({ + name: 'The yellow key', + transports: ['internal', 'hybrid'], + }); + + const listed = await browser.get('/api/auth/passkeys'); + expect(listed.body.passkeys).toHaveLength(1); + expect(listed.body.passkeys[0].lastUsedAt).toBeNull(); + }); + + it('names it for them when they do not', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + + await addPasskey(browser); + const second = await addPasskey(browser); + + expect(second.finished.body.passkey.name).toBe('Passkey 2'); + }); + + it('asks the browser to skip the ones already on the account', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { credential } = await addPasskey(browser); + + const started = await browser.post('/api/auth/passkeys/register/start').send({}); + + expect(started.body.options.excludeCredentials).toEqual([ + expect.objectContaining({ + id: credential.credentialId.toString('base64url'), + type: 'public-key', + }), + ]); + }); + + it('offers the algorithms it can actually verify', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + + const started = await browser.post('/api/auth/passkeys/register/start').send({}); + + expect(started.body.options.pubKeyCredParams.map((p) => p.alg)).toEqual([-7, -8, -257]); + expect(started.body.options.attestation).toBe('none'); + expect(started.body.options.rp).toEqual({ id: 'files.example.test', name: 'NextExplorer' }); + }); + + it('refuses one from somebody who is not signed in', async () => { + const { app } = await build(); + await setUpOwner(app); + const stranger = request.agent(app); + + const started = await stranger.post('/api/auth/passkeys/register/start').send({}); + + expect(started.status).toBe(401); + }); + + it('refuses an answer to a question this browser was never asked', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const credential = createCredential({ + challenge: 'c29tZXRoaW5nLWVsc2U', + rpId: 'files.example.test', + origin: PUBLIC_URL, + }); + + const finished = await browser.post('/api/auth/passkeys/register/finish').send({ + response: { + attestationObject: credential.attestationObject.toString('base64url'), + clientDataJSON: credential.clientDataJSON.toString('base64url'), + }, + }); + + expect(finished.status).toBe(400); + }); + + it('refuses one made for another site', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const started = await browser.post('/api/auth/passkeys/register/start').send({}); + const credential = createCredential({ + challenge: started.body.options.challenge, + rpId: 'evil.test', + origin: 'https://evil.test', + }); + + const finished = await browser.post('/api/auth/passkeys/register/finish').send({ + response: { + attestationObject: credential.attestationObject.toString('base64url'), + clientDataJSON: credential.clientDataJSON.toString('base64url'), + }, + }); + + expect(finished.status).toBe(401); + expect(finished.body.error.code).toBe('AUTH_PASSKEY_REJECTED'); + }); + + it('refuses the same credential twice', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { credential } = await addPasskey(browser); + + const started = await browser.post('/api/auth/passkeys/register/start').send({}); + const again = createCredential({ + challenge: started.body.options.challenge, + rpId: started.body.options.rp.id, + origin: started.body.origins[0], + credentialId: credential.credentialId, + }); + const finished = await browser.post('/api/auth/passkeys/register/finish').send({ + response: { + attestationObject: again.attestationObject.toString('base64url'), + clientDataJSON: again.clientDataJSON.toString('base64url'), + }, + }); + + expect(finished.status).toBe(400); + expect(finished.body.error.message).toMatch(/already/i); + }); + + it('spends the question, so one ceremony makes one passkey', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const started = await browser.post('/api/auth/passkeys/register/start').send({}); + const answer = (credential) => ({ + response: { + attestationObject: credential.attestationObject.toString('base64url'), + clientDataJSON: credential.clientDataJSON.toString('base64url'), + }, + }); + const made = () => + createCredential({ + challenge: started.body.options.challenge, + rpId: started.body.options.rp.id, + origin: started.body.origins[0], + }); + + expect( + (await browser.post('/api/auth/passkeys/register/finish').send(answer(made()))).status + ).toBe(201); + // A different credential, from the same question: the question is gone. + const second = await browser.post('/api/auth/passkeys/register/finish').send(answer(made())); + + expect(second.status).toBe(400); + expect((await browser.get('/api/auth/passkeys')).body.passkeys).toHaveLength(1); + }); + + it('will not let a sign-in question be answered with a new passkey', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + // The challenge is real and this browser was just given it — for the other + // ceremony. Answering the wrong one is how a signature made for signing in + // would be offered as a registration. + const started = await browser.post('/api/auth/login/passkey/start').send({}); + const credential = createCredential({ + challenge: started.body.options.challenge, + rpId: started.body.options.rpId, + origin: started.body.origins[0], + }); + + const finished = await browser.post('/api/auth/passkeys/register/finish').send({ + response: { + attestationObject: credential.attestationObject.toString('base64url'), + clientDataJSON: credential.clientDataJSON.toString('base64url'), + }, + }); + + expect(finished.status).toBe(400); + expect((await browser.get('/api/auth/passkeys')).body.passkeys).toEqual([]); + }); + + it('refuses one from a session the identity provider opened', async () => { + const { app, appForProviderSession } = await build(); + const browser = await setUpOwner(app); + const me = (await browser.get('/api/auth/status')).body.user; + + const federated = request.agent(appForProviderSession(me.id)); + const started = await federated.post('/api/auth/passkeys/register/start').send({}); + // And the far end of the ceremony, which is the one that would write a row. + const finished = await federated + .post('/api/auth/passkeys/register/finish') + .send({ response: {} }); + + expect(started.status).toBe(403); + expect(finished.status).toBe(403); + expect((await browser.get('/api/auth/passkeys')).body.passkeys).toEqual([]); + }); + + it('spends the question, so the same answer cannot be given twice', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const started = await browser.post('/api/auth/passkeys/register/start').send({}); + const credential = createCredential({ + challenge: started.body.options.challenge, + rpId: started.body.options.rp.id, + origin: started.body.origins[0], + }); + const body = { + response: { + attestationObject: credential.attestationObject.toString('base64url'), + clientDataJSON: credential.clientDataJSON.toString('base64url'), + }, + }; + + expect((await browser.post('/api/auth/passkeys/register/finish').send(body)).status).toBe(201); + expect((await browser.post('/api/auth/passkeys/register/finish').send(body)).status).toBe(400); + }); +}); + +describe('signing in with one', { timeout: 30_000 }, () => { + it('opens the account, without a password', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { credential } = await addPasskey(browser); + await browser.post('/api/auth/logout').send({}); + expect(await signedIn(browser)).toBe(false); + + const response = await signInWith(browser, credential); + + expect(response.status).toBe(200); + expect(response.body.user).toMatchObject({ username: 'owner' }); + expect(await signedIn(browser)).toBe(true); + }); + + it.each([['ES256'], ['EdDSA'], ['RS256']])('takes a %s key', async (name) => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { credential } = await addPasskey(browser, { algorithm: ALGORITHMS[name] }); + await browser.post('/api/auth/logout').send({}); + + expect((await signInWith(browser, credential)).status).toBe(200); + }); + + it('writes down when it was last used', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { credential } = await addPasskey(browser); + await browser.post('/api/auth/logout').send({}); + await signInWith(browser, credential); + + const listed = await browser.get('/api/auth/passkeys'); + + expect(listed.body.passkeys[0].lastUsedAt).toEqual(expect.any(String)); + }); + + it('refuses a recording of a sign-in that already happened', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { credential } = await addPasskey(browser); + await browser.post('/api/auth/logout').send({}); + + const started = await browser.post('/api/auth/login/passkey/start').send({}); + const assertion = signAssertion({ + credential, + challenge: started.body.options.challenge, + rpId: started.body.options.rpId, + origin: started.body.origins[0], + signCount: 4, + }); + const body = { + response: { + id: credential.credentialId.toString('base64url'), + authenticatorData: assertion.authenticatorData.toString('base64url'), + clientDataJSON: assertion.clientDataJSON.toString('base64url'), + signature: assertion.signature.toString('base64url'), + }, + }; + + expect((await browser.post('/api/auth/login/passkey/finish').send(body)).status).toBe(200); + await browser.post('/api/auth/logout').send({}); + // The same answer again, to a question that has been spent. + expect((await browser.post('/api/auth/login/passkey/finish').send(body)).status).toBe(401); + }); + + it('refuses a passkey that has been used with that counter before', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { credential } = await addPasskey(browser); + await browser.post('/api/auth/logout').send({}); + + expect((await signInWith(browser, credential, { signCount: 8 })).status).toBe(200); + await browser.post('/api/auth/logout').send({}); + const replayed = await signInWith(browser, credential, { signCount: 8 }); + + expect(replayed.status).toBe(401); + expect(replayed.body.error.code).toBe('AUTH_PASSKEY_REJECTED'); + }); + + it('refuses a credential it has never seen, saying no more than that', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + await addPasskey(browser); + await browser.post('/api/auth/logout').send({}); + const stranger = createCredential({ challenge: 'unused', rpId: 'files.example.test' }); + + const response = await signInWith(browser, stranger); + + expect(response.status).toBe(401); + expect(response.body.error.code).toBe('AUTH_PASSKEY_REJECTED'); + expect(JSON.stringify(response.body)).not.toMatch(/owner/); + }); + + it('refuses a signature made for another site', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { credential } = await addPasskey(browser); + await browser.post('/api/auth/logout').send({}); + + const response = await signInWith(browser, credential, { + rpId: 'evil.test', + origin: 'https://evil.test', + }); + + expect(response.status).toBe(401); + expect(await signedIn(browser)).toBe(false); + }); + + it('refuses one while the account is locked out by wrong passwords', async () => { + const { app } = await build({ AUTH_MAX_FAILED: '2' }); + const browser = await setUpOwner(app); + const { credential } = await addPasskey(browser); + await browser.post('/api/auth/logout').send({}); + for (let attempt = 0; attempt < 2; attempt += 1) { + await browser.post('/api/auth/login').send({ identifier: 'owner', password: 'wrong' }); + } + + const response = await signInWith(browser, credential); + + expect(response.status).toBe(429); + expect(response.body.error.code).toBe('AUTH_ACCOUNT_LOCKED'); + expect(await signedIn(browser)).toBe(false); + }); + + it('refuses an answer with no question behind it', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { credential } = await addPasskey(browser); + await browser.post('/api/auth/logout').send({}); + const assertion = signAssertion({ + credential, + challenge: 'bmV2ZXItYXNrZWQ', + rpId: 'files.example.test', + origin: PUBLIC_URL, + }); + + const response = await request + .agent(app) + .post('/api/auth/login/passkey/finish') + .send({ + response: { + id: credential.credentialId.toString('base64url'), + authenticatorData: assertion.authenticatorData.toString('base64url'), + clientDataJSON: assertion.clientDataJSON.toString('base64url'), + signature: assertion.signature.toString('base64url'), + }, + }); + + expect(response.status).toBe(401); + }); +}); + +describe('a passkey and a second factor', { timeout: 40_000 }, () => { + const turnOnTotp = async (browser, totpCode) => { + const started = await browser.post('/api/auth/totp/start').send({}); + expect(started.status).toBe(200); + const confirmed = await browser + .post('/api/auth/totp/confirm') + .send({ code: totpCode(started.body.secret) }); + expect(confirmed.status).toBe(200); + return started.body.secret; + }; + + it('is the whole sign-in when the passkey was unlocked', async () => { + const { app, totpCode } = await build(); + const browser = await setUpOwner(app); + await turnOnTotp(browser, totpCode); + const { credential } = await addPasskey(browser); + await browser.post('/api/auth/logout').send({}); + + const response = await signInWith(browser, credential, { userVerified: true }); + + expect(response.status).toBe(200); + expect(response.body.totpRequired).toBeUndefined(); + expect(await signedIn(browser)).toBe(true); + }); + + it('still asks for the code when the passkey was not unlocked', async () => { + const { app, totpCode } = await build(); + const browser = await setUpOwner(app); + const secret = await turnOnTotp(browser, totpCode); + const { credential } = await addPasskey(browser); + await browser.post('/api/auth/logout').send({}); + + const response = await signInWith(browser, credential, { userVerified: false }); + + expect(response.body).toEqual({ totpRequired: true }); + expect(await signedIn(browser)).toBe(false); + + // The next window's code: the one that turned it on has been spent. + const second = await browser + .post('/api/auth/login/totp') + .send({ code: totpCode(secret, { at: Date.now() + 30_000 }) }); + expect(second.status).toBe(200); + expect(await signedIn(browser)).toBe(true); + }); +}); + +describe('managing them', { timeout: 30_000 }, () => { + it('renames one', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { finished } = await addPasskey(browser, { name: 'Old name' }); + + const renamed = await browser + .patch(`/api/auth/passkeys/${finished.body.passkey.id}`) + .send({ name: ' The blue one ' }); + + expect(renamed.body.passkey.name).toBe('The blue one'); + expect((await browser.get('/api/auth/passkeys')).body.passkeys[0].name).toBe('The blue one'); + }); + + it('refuses to rename one that belongs to somebody else', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { finished } = await addPasskey(browser); + + const other = request.agent(app); + await other + .post('/api/auth/login') + .send({ identifier: 'owner@example.com', password: PASSWORD }); + + const renamed = await request + .agent(app) + .patch(`/api/auth/passkeys/${finished.body.passkey.id}`) + .send({ name: 'mine now' }); + + expect(renamed.status).toBe(401); + }); + + it('asks for the password before taking one away', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { finished } = await addPasskey(browser); + const id = finished.body.passkey.id; + + const refused = await browser.delete(`/api/auth/passkeys/${id}`).send({ password: 'wrong' }); + expect(refused.status).toBe(401); + expect(refused.body.error.code).toBe('AUTH_PASSWORD_INCORRECT'); + + const removed = await browser.delete(`/api/auth/passkeys/${id}`).send({ password: PASSWORD }); + expect(removed.status).toBe(204); + expect((await browser.get('/api/auth/passkeys')).body.passkeys).toEqual([]); + }); + + it('has nothing to say about a passkey that is not there', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + + const removed = await browser + .delete('/api/auth/passkeys/00000000-0000-0000-0000-000000000000') + .send({ password: PASSWORD }); + + expect(removed.status).toBe(404); + }); + + it('closes the door on a passkey that is gone', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + const { credential, finished } = await addPasskey(browser); + await browser + .delete(`/api/auth/passkeys/${finished.body.passkey.id}`) + .send({ password: PASSWORD }); + await browser.post('/api/auth/logout').send({}); + + expect((await signInWith(browser, credential)).status).toBe(401); + }); +}); + +describe('the site a passkey is bound to', { timeout: 30_000 }, () => { + it('is the public URL when there is one', async () => { + const { app } = await build(); + const browser = await setUpOwner(app); + + const started = await browser.post('/api/auth/passkeys/register/start').send({}); + + expect(started.body.options.rp.id).toBe('files.example.test'); + expect(started.body.origins).toEqual([PUBLIC_URL]); + }); + + it('is the name the request arrived on when nothing is configured', async () => { + const { app } = await build({ PUBLIC_URL: undefined }); + const browser = await setUpOwner(app); + + const started = await browser + .post('/api/auth/passkeys/register/start') + .set('Host', 'files.lan:3000') + .send({}); + + expect(started.body.options.rp.id).toBe('files.lan'); + expect(started.body.origins).toEqual(['http://files.lan:3000']); + }); + + it('is what the operator settled, over both', async () => { + const { app } = await build({ WEBAUTHN_RP_ID: 'example.test', WEBAUTHN_RP_NAME: 'Home files' }); + const browser = await setUpOwner(app); + + const started = await browser.post('/api/auth/passkeys/register/start').send({}); + + expect(started.body.options.rp).toEqual({ id: 'example.test', name: 'Home files' }); + }); + + it('offers passkeys on the sign-in screen where local accounts are offered', async () => { + const { app } = await build(); + + expect((await request(app).get('/api/auth/status')).body.strategies).toMatchObject({ + local: true, + passkey: true, + }); + }); + + it('offers none when accounts come from the identity provider', async () => { + const { app } = await build({ AUTH_MODE: 'oidc' }); + + const status = await request(app).get('/api/auth/status'); + expect(status.body.strategies.passkey).toBe(false); + expect((await request(app).post('/api/auth/login/passkey/start').send({})).status).toBe(403); + }); +}); + +/** + * The laptop that was the passkey, gone with the passkey on it. + * + * The same deliberate act as taking somebody's second factor off, by the same + * person: an administrator who can already reset that account's password. + */ +describe('an administrator handing an account back', { timeout: 40_000 }, () => { + const createRegular = async (owner) => + ( + await owner.post('/api/users').send({ + email: 'regular@example.com', + username: 'regular', + password: PASSWORD, + roles: ['user'], + }) + ).body.user; + + const buildWithUsers = async () => { + const built = await build(); + const userRoutes = currentEnv.requireFresh('src/routes/users'); + built.app.use('/api', userRoutes); + return built; + }; + + it('takes every passkey off, and the account signs in with its password', async () => { + const { app } = await buildWithUsers(); + const owner = await setUpOwner(app); + const regular = await createRegular(owner); + + const theirs = request.agent(app); + await theirs.post('/api/auth/login').send({ identifier: 'regular', password: PASSWORD }); + const { credential } = await addPasskey(theirs); + expect((await theirs.get('/api/auth/passkeys')).body.passkeys).toHaveLength(1); + + expect((await owner.delete(`/api/users/${regular.id}/passkeys`)).status).toBe(204); + + expect((await theirs.get('/api/auth/passkeys')).body.passkeys).toEqual([]); + await theirs.post('/api/auth/logout').send({}); + expect((await signInWith(theirs, credential)).status).toBe(401); + + const again = request.agent(app); + const response = await again + .post('/api/auth/login') + .send({ identifier: 'regular', password: PASSWORD }); + expect(response.body.user.username).toBe('regular'); + }); + + it('refuses anybody who is not an administrator', async () => { + const { app } = await buildWithUsers(); + const owner = await setUpOwner(app); + const regular = await createRegular(owner); + + const theirs = request.agent(app); + await theirs.post('/api/auth/login').send({ identifier: 'regular', password: PASSWORD }); + await addPasskey(theirs); + + const response = await theirs.delete(`/api/users/${regular.id}/passkeys`); + + expect(response.status).toBe(403); + expect((await theirs.get('/api/auth/passkeys')).body.passkeys).toHaveLength(1); + }); + + it('has nothing to say about an account that is not there', async () => { + const { app } = await buildWithUsers(); + const owner = await setUpOwner(app); + + expect((await owner.delete('/api/users/nobody/passkeys')).status).toBe(404); + }); +}); diff --git a/backend/tests/routes/permissions.test.js b/backend/tests/routes/permissions.test.js new file mode 100644 index 000000000..20454d5ae --- /dev/null +++ b/backend/tests/routes/permissions.test.js @@ -0,0 +1,265 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The two routes that hand a path to `chmod` and `chown`. + * + * They are the pair that once shipped with no admin check at all, and they are + * the only place in the application where a value from a request reaches a + * system tool. Both facts are pinned here: that a regular account is refused, + * and that nothing shaped like an option can reach the argument list. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const REGULAR_USER = { id: 'user-1', username: 'regular', roles: ['user'] }; +const ADMIN_USER = { id: 'admin-1', username: 'admin', roles: ['admin'] }; + +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'permissions-' }); + const dbService = currentEnv.requireFresh('src/services/db'); + const db = await dbService.getDb(); + const now = new Date().toISOString(); + for (const user of [REGULAR_USER, ADMIN_USER]) { + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, ?, ?, ?)` + ).run( + user.id, + `${user.username}@example.com`, + user.username, + user.username, + JSON.stringify(user.roles), + now, + now + ); + } + const dir = path.join(currentEnv.volumeDir, 'Docs'); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, 'note.txt'), 'hello\n'); + return dir; +}; + +const buildApp = (user) => { + const routes = currentEnv.requireFresh('src/routes/permissions'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + if (user) req.user = user; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +describe('who may change permissions', () => { + it('refuses a regular account on chmod', async () => { + await seed(); + + const response = await request(buildApp(REGULAR_USER)) + .post('/api/permissions/chmod') + .send({ path: 'Docs/note.txt', mode: '777' }); + + expect(response.status).toBe(403); + }); + + it('refuses a regular account on chown', async () => { + await seed(); + + const response = await request(buildApp(REGULAR_USER)) + .post('/api/permissions/chown') + .send({ path: 'Docs/note.txt', owner: 'root' }); + + expect(response.status).toBe(403); + }); + + /** + * A write permission on a path is not consent to re-permission its tree — + * the refusal has to come from the role, not from the path being unreachable. + */ + it('refuses before it has looked at the path at all', async () => { + await seed(); + + const response = await request(buildApp(REGULAR_USER)) + .post('/api/permissions/chmod') + .send({ path: 'Docs/does-not-exist.txt', mode: '777' }); + + expect(response.status).toBe(403); + }); +}); + +describe('what may reach chmod', () => { + it('changes the mode of a real file', async () => { + const dir = await seed(); + + const response = await request(buildApp(ADMIN_USER)) + .post('/api/permissions/chmod') + .send({ path: 'Docs/note.txt', mode: '640' }); + + expect(response.status).toBe(200); + const stats = await fs.stat(path.join(dir, 'note.txt')); + expect(stats.mode & 0o777).toBe(0o640); + }); + + // Three digits say nothing about the setuid, setgid and sticky bits, and + // chmod writes the whole mode: unticking one box on a sticky or setgid folder + // used to take those bits away with it. + it('keeps the special bits the folder already had', async () => { + const dir = await seed(); + const shared = path.join(dir, 'drop-box'); + await fs.mkdir(shared); + await fs.chmod(shared, 0o1777); + + const response = await request(buildApp(ADMIN_USER)) + .post('/api/permissions/chmod') + .send({ path: 'Docs/drop-box', mode: '775' }); + + expect(response.status).toBe(200); + const stats = await fs.stat(shared); + expect(stats.mode & 0o777).toBe(0o775); + expect(stats.mode & 0o7000).toBe(0o1000); + expect(response.body.mode & 0o7777).toBe(0o1775); + }); + + // The mode is interpolated into a `chmod -R` argument list on the recursive + // path. Only three octal digits can get that far. + it.each([['755 --reference=/etc/shadow'], ['7555'], ['75\n5'], ['a+x'], ['']])( + 'refuses the mode %j', + async (mode) => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)) + .post('/api/permissions/chmod') + .send({ path: 'Docs/note.txt', mode }); + + expect(response.status).toBe(400); + } + ); + + // The status alone proves nothing here: a share path is unreachable anyway, + // so it is refused either way. Only the reason says which check fired. + it('refuses a path reached through a share, and says so', async () => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)) + .post('/api/permissions/chmod') + .send({ path: 'share/abc/note.txt', mode: '640' }); + + expect(response.status).toBe(403); + expect(response.body.error.message).toBe('Permissions cannot be changed through a share.'); + }); + + it('requires a path', async () => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)) + .post('/api/permissions/chmod') + .send({ mode: '640' }); + + expect(response.status).toBe(400); + }); +}); + +describe('what may reach chown', () => { + /** + * `chown` takes its arguments as an array and never through a shell, so a + * semicolon is harmless — an argument that reads as an *option* is not. + * `--reference=FILE` makes chown copy another file's ownership, and a + * leading dash is what the pattern exists to refuse. + */ + it.each([['--reference=/etc/shadow'], ['-R'], ['root nobody'], ['root;id'], ['.hidden'], ['-']])( + 'refuses the owner %j', + async (owner) => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)) + .post('/api/permissions/chown') + .send({ path: 'Docs/note.txt', owner }); + + expect(response.status).toBe(400); + } + ); + + it('refuses a group of the same shape', async () => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)) + .post('/api/permissions/chown') + .send({ path: 'Docs/note.txt', group: '--reference=/etc/shadow' }); + + expect(response.status).toBe(400); + }); + + it('accepts an ordinary account name', async () => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)) + .post('/api/permissions/chown') + .send({ path: 'Docs/note.txt', owner: 'nobody' }); + + // Changing ownership needs root, which the test process is not; what + // matters is that the name passed validation and the call was attempted. + expect(response.status).not.toBe(400); + }); + + it('requires an owner or a group', async () => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)) + .post('/api/permissions/chown') + .send({ path: 'Docs/note.txt' }); + + expect(response.status).toBe(400); + }); + + it('refuses a path reached through a share, and says so', async () => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)) + .post('/api/permissions/chown') + .send({ path: 'share/abc/note.txt', owner: 'nobody' }); + + expect(response.status).toBe(403); + expect(response.body.error.message).toBe('Ownership cannot be changed through a share.'); + }); +}); + +describe('reading permissions', () => { + it('reports the mode, owner and group of a file', async () => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)).get('/api/permissions/Docs/note.txt'); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ path: 'Docs/note.txt' }); + }); + + it('requires a path', async () => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)).get('/api/permissions/'); + + expect(response.status).toBe(400); + }); + + it('says not found rather than failing, for a path that is not there', async () => { + await seed(); + + const response = await request(buildApp(ADMIN_USER)).get('/api/permissions/Docs/absent.txt'); + + expect(response.status).toBe(404); + }); +}); diff --git a/backend/tests/routes/personal-folder-isolation.test.js b/backend/tests/routes/personal-folder-isolation.test.js index 97b6d238b..e58140c76 100644 --- a/backend/tests/routes/personal-folder-isolation.test.js +++ b/backend/tests/routes/personal-folder-isolation.test.js @@ -61,8 +61,7 @@ const setup = async ({ userRootEnv = {} } = {}) => { await fs.writeFile(path.join(aliceRoot, 'salary.txt'), 'alice private'); const bob = { id: 'bob', username: 'bob', roles: ['user'] }; - const asBob = (router) => - createTestApp({ router, mountPath: '/api', user: bob, errorHandler }); + const asBob = (router) => createTestApp({ router, mountPath: '/api', user: bob, errorHandler }); return { env, @@ -176,7 +175,7 @@ describe('a volume that does not hold the personal folders', () => { const browseRoutes = env.requireFresh('src/routes/browse'); const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); - const db = await (env.requireFresh('src/services/db').getDb()); + const db = await env.requireFresh('src/services/db').getDb(); const now = new Date().toISOString(); db.prepare( `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) diff --git a/backend/tests/routes/public-endpoints-hardening.test.js b/backend/tests/routes/public-endpoints-hardening.test.js index dcb6ed5b9..a3f982b64 100644 --- a/backend/tests/routes/public-endpoints-hardening.test.js +++ b/backend/tests/routes/public-endpoints-hardening.test.js @@ -64,7 +64,6 @@ describe('checking a share password', () => { const { app, token } = await shareApp(); for (let attempt = 0; attempt < 20; attempt += 1) { - // eslint-disable-next-line no-await-in-loop const response = await request(app) .post(`/api/share/${token}/verify`) .send({ password: `guess-${attempt}` }); @@ -88,8 +87,14 @@ describe('checking a share password', () => { .post(`/api/share/${token}/verify`) .send({ password: 'open-sesame' }); + // The one being handed out, not the one being cleared. The cookie moved + // from /api to the root — an asking /static for a thumbnail cannot + // carry a header, and a cookie scoped to /api never reaches it — so the + // old one is cleared in the same answer, and a browser keeps both headers. const cookieOf = (response) => - [].concat(response.headers['set-cookie'] || []).find((c) => c.startsWith('guestSession=')); + [] + .concat(response.headers['set-cookie'] || []) + .find((c) => /^guestSession=.+/.test(c) && !/Expires=Thu, 01 Jan 1970/.test(c)); expect(overHttps.status).toBe(200); expect(cookieOf(overHttps)).toMatch(/;\s*Secure/i); // The control: plain HTTP cannot carry a Secure cookie back at all. @@ -103,7 +108,6 @@ describe('a failed sign-in for an address with no account', () => { const users = envContext.requireFresh('src/services/users'); for (let attempt = 0; attempt < 12; attempt += 1) { - // eslint-disable-next-line no-await-in-loop expect( await users.attemptLocalLogin({ email: 'newcomer@example.com', password: 'guess' }) ).toBeNull(); diff --git a/backend/tests/routes/recent-destinations.test.js b/backend/tests/routes/recent-destinations.test.js new file mode 100644 index 000000000..8e44c7c8c --- /dev/null +++ b/backend/tests/routes/recent-destinations.test.js @@ -0,0 +1,146 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The folders a user actually files things into. + * + * The destination picker opens on this list rather than at the root, so what it + * contains has to be true without anyone maintaining it: written by the + * transfers themselves, ordered by use, and holding nothing the person cannot + * still reach. A destination offered and then refused at the end of the flow + * would be worse than no list at all. + */ + +describe('recent destinations', () => { + let env; + const asUser = (id) => ({ id, roles: ['admin'] }); + + const setup = async () => { + env = await setupTestEnv({ + tag: 'recent-destinations-', + modules: [ + 'src/services/db', + 'src/services/recentDestinationsService', + 'src/services/accessManager', + 'src/routes/files', + 'src/middleware/errorHandler', + ], + }); + + for (const folder of ['Archive', 'Invoices', 'Photos']) { + await fs.mkdir(path.join(env.volumeDir, folder), { recursive: true }); + } + await fs.writeFile(path.join(env.volumeDir, 'report.txt'), 'contents'); + }; + + const appFor = (user) => { + const routes = env.requireFresh('src/routes/files'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + return createTestApp({ router: routes, mountPath: '/api', user, errorHandler }); + }; + + /** Move a file into a folder, the way the client does. */ + const moveInto = async (app, name, destination) => + request(app) + .post('/api/files/move') + .send({ + items: [{ name, path: '' }], + destination, + }); + + const listFor = async (app) => { + const response = await request(app).get('/api/files/recent-destinations'); + expect(response.status).toBe(200); + return response.body.items; + }; + + afterEach(async () => { + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('remembers where a transfer landed, without being asked to', async () => { + // Nothing in the client reports this: recording it from the transfer means + // a drag onto a favorite and a paste count exactly like a pick. + await setup(); + const app = appFor(asUser('alice')); + + expect(await listFor(app)).toEqual([]); + + expect((await moveInto(app, 'report.txt', 'Archive')).status).toBe(200); + + expect(await listFor(app)).toEqual(['Archive']); + }); + + it('puts the destination used most recently first', async () => { + await setup(); + const app = appFor(asUser('alice')); + + await moveInto(app, 'report.txt', 'Archive'); + await fs.writeFile(path.join(env.volumeDir, 'report.txt'), 'again'); + await moveInto(app, 'report.txt', 'Invoices'); + + expect(await listFor(app)).toEqual(['Invoices', 'Archive']); + }); + + it('moves a destination up rather than listing it twice', async () => { + await setup(); + const app = appFor(asUser('alice')); + + await moveInto(app, 'report.txt', 'Archive'); + await fs.writeFile(path.join(env.volumeDir, 'report.txt'), 'again'); + await moveInto(app, 'report.txt', 'Invoices'); + await fs.writeFile(path.join(env.volumeDir, 'report.txt'), 'once more'); + await moveInto(app, 'report.txt', 'Archive'); + + expect(await listFor(app)).toEqual(['Archive', 'Invoices']); + }); + + it('keeps one user habits out of another list', async () => { + // Where someone files their work says something about it; a favorite is + // shared deliberately, this is not. + await setup(); + const alice = appFor(asUser('alice')); + const bob = appFor(asUser('bob')); + + await moveInto(alice, 'report.txt', 'Archive'); + + expect(await listFor(alice)).toEqual(['Archive']); + expect(await listFor(bob)).toEqual([]); + }); + + it('drops a destination that has since been deleted', async () => { + // Offering it would only produce a failure at the end of the flow. + await setup(); + const app = appFor(asUser('alice')); + + await moveInto(app, 'report.txt', 'Archive'); + await fs.rm(path.join(env.volumeDir, 'Archive'), { recursive: true, force: true }); + + expect(await listFor(app)).toEqual([]); + + // And forgotten for good, rather than re-tested on every open. + await fs.mkdir(path.join(env.volumeDir, 'Archive'), { recursive: true }); + expect(await listFor(app)).toEqual([]); + }); + + it('swallows a write it cannot perform, rather than failing the transfer', async () => { + // The list is a convenience; the file arriving is not. A /config directory + // shared with an older image is the real way this happens — the schema is + // behind and the table simply isn't there, which must not turn a successful + // move into a 500. + await setup(); + + const { getDb } = env.requireFresh('src/services/db'); + const service = env.requireFresh('src/services/recentDestinationsService'); + const db = await getDb(); + db.exec('DROP TABLE recent_destinations'); + + await expect(service.record('alice', 'Archive')).resolves.toBeUndefined(); + }); +}); diff --git a/backend/tests/routes/request-body-shape.test.js b/backend/tests/routes/request-body-shape.test.js new file mode 100644 index 000000000..40a514140 --- /dev/null +++ b/backend/tests/routes/request-body-shape.test.js @@ -0,0 +1,102 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What `req.body` is when nobody sent one. + * + * Express 4 left an empty object; Express 5 leaves it undefined, and this + * application was written against the empty object. A route destructuring + * `const { path, mode } = req.body` does not merely misbehave against a request + * with no content-type header — it throws a TypeError and answers 500 to + * something whose only fault is a missing header, which is exactly what a curl + * one-liner sends. + * + * Every other suite sends JSON, so none of them can see this. This one + * deliberately does not, and it goes through the real application so that the + * middleware order is the one that ships. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const buildApp = async () => { + // Authentication off, so the request arrives as an administrator and reaches + // the line that destructures the body — with auth on it is refused above it, + // and the test proves nothing. + currentEnv = await setupTestEnv({ tag: 'body-shape-', env: { AUTH_ENABLED: 'false' } }); + const { createApp } = currentEnv.requireFresh('src/app'); + return createApp({ skipBootstrap: true }); +}; + +/** The two parsers and the normaliser, in the order `createApp` mounts them. */ +const parsersOnly = () => { + const app = express(); + app.use(express.json()); + app.use(express.urlencoded({ extended: true })); + app.use((req, _res, next) => { + if (req.body === undefined) req.body = {}; + next(); + }); + return app; +}; + +describe('a request that carries no body', () => { + it('reaches a route that destructures it, and is answered rather than crashing', async () => { + const app = await buildApp(); + + // No `.send()`, so no content-type: `express.json` does not run, and the + // chmod route destructures `req.body` on the very first line. + const response = await request(app).post('/api/permissions/chmod'); + + expect(response.status).not.toBe(500); + }); + + it('is refused for the reason a caller can act on', async () => { + const app = await buildApp(); + + const response = await request(app).post('/api/permissions/chmod'); + + // The route's own validation answers: a path is required. + expect(response.status).toBe(400); + }); +}); + +describe('the normaliser itself', () => { + it('leaves an empty object where the parsers left nothing', async () => { + const app = parsersOnly(); + app.post('/probe', (req, res) => { + res.json({ type: req.body === undefined ? 'undefined' : typeof req.body }); + }); + + const response = await request(app).post('/probe'); + + expect(response.body.type).toBe('object'); + }); + + it('lets a route ask whether a field is present without throwing', async () => { + const app = parsersOnly(); + app.patch('/probe', (req, res) => res.json({ present: 'accessMode' in req.body })); + + const response = await request(app).patch('/probe'); + + expect(response.status).toBe(200); + expect(response.body.present).toBe(false); + }); + + it('does not touch a body that was parsed', async () => { + const app = parsersOnly(); + app.post('/probe', (req, res) => res.json({ got: req.body.value })); + + const response = await request(app).post('/probe').send({ value: 'kept' }); + + expect(response.body.got).toBe('kept'); + }); +}); diff --git a/backend/tests/routes/route-registration.test.js b/backend/tests/routes/route-registration.test.js new file mode 100644 index 000000000..85588da63 --- /dev/null +++ b/backend/tests/routes/route-registration.test.js @@ -0,0 +1,113 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The wiring nobody looks at. A route file that stops being mounted breaks + * nothing at startup and nothing in the tests that exercise it directly — it + * simply stops answering, and the first report is a user saying a button does + * nothing. + * + * Asked by making a request rather than by reading the router's internals: a + * mounted route may answer 200, 401, 403 or 500 depending on what it needs, and + * only 404 means nothing is listening. Reading the internals instead is how the + * first version of this test broke on an Express upgrade while the application + * itself was fine. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const build = async (env = {}) => { + currentEnv = await setupTestEnv({ tag: 'route-registration-', env }); + const registerRoutes = currentEnv.requireFresh('src/routes/index'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: 'admin-1', email: 'a@example.com', roles: ['admin'] }; + next(); + }); + registerRoutes(app); + return app; +}; + +const answered = async (app, path) => (await request(app).get(path)).status !== 404; + +describe('every route file is reachable', () => { + it.each([ + ['/api/auth/me', 'sign-in'], + ['/api/browse/', 'browsing'], + ['/api/volumes', 'volumes'], + ['/api/favorites', 'favorites'], + ['/api/settings', 'settings'], + ['/api/search?q=x', 'search'], + ['/api/users', 'accounts'], + ['/api/metadata/', 'details'], + ['/api/permissions/', 'permissions'], + ['/api/thumbnails/', 'thumbnails'], + ['/api/features', 'features'], + ['/api/usage/', 'volume usage'], + ['/api/upload/finalizations', 'uploads'], + ])('answers on %s, which serves %s', async (path) => { + const app = await build(); + + expect(await answered(app, path)).toBe(true); + }); + + /** + * Folder sizes answer 404 by design when the feature is off, so this one has + * to be switched on to tell 'not mounted' from 'mounted and disabled'. + */ + it('answers on /api/folder-size once the feature is on', async () => { + const app = await build({ FOLDER_SIZE_MODE: 'full' }); + + expect(await answered(app, '/api/folder-size/Docs')).toBe(true); + }); + + it('answers 404 there when the feature is off, without being unmounted', async () => { + const app = await build(); + + const response = await request(app).get('/api/folder-size/Docs'); + expect(response.status).toBe(404); + expect(response.body.error).toMatch(/disabled/i); + }); +}); + +describe('the editors mount only when they are configured', () => { + const ONLYOFFICE_PATH = '/api/onlyoffice/users'; + const COLLABORA_PATH = '/api/collabora/wopi/files/abc'; + + it('leaves ONLYOFFICE unmounted when no server is set', async () => { + const app = await build(); + + expect(await answered(app, ONLYOFFICE_PATH)).toBe(false); + }); + + it('mounts ONLYOFFICE once a server is set', async () => { + const app = await build({ ONLYOFFICE_URL: 'https://office.example.com' }); + + expect(await answered(app, ONLYOFFICE_PATH)).toBe(true); + }); + + it('leaves Collabora unmounted without both its URL and its secret', async () => { + const app = await build({ COLLABORA_URL: 'https://collabora.example.com' }); + + expect(await answered(app, COLLABORA_PATH)).toBe(false); + }); + + it('mounts Collabora once both are set', async () => { + const app = await build({ + COLLABORA_URL: 'https://collabora.example.com', + COLLABORA_SECRET: 'a-secret', + }); + + expect(await answered(app, COLLABORA_PATH)).toBe(true); + }); +}); diff --git a/backend/tests/routes/search-content.test.js b/backend/tests/routes/search-content.test.js index e242c872d..3abce9f5b 100644 --- a/backend/tests/routes/search-content.test.js +++ b/backend/tests/routes/search-content.test.js @@ -112,7 +112,6 @@ describe('when the same term matches many filenames', () => { const dir = await seed(); await fs.mkdir(dir, { recursive: true }); for (let index = 0; index < 120; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(path.join(dir, `pangolin-${index}.txt`), 'nothing to see\n'); } await fs.writeFile(path.join(dir, 'zzz-notes.md'), 'the word pangolin is in here\n'); @@ -408,7 +407,6 @@ describe('how long a search may take', () => { const dir = await seed({ SEARCH_TIMEOUT_MS: '1' }); await fs.mkdir(dir, { recursive: true }); for (let index = 0; index < 60; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(path.join(dir, `doc-${index}.pdf`), buildPdf('nothing of interest')); } diff --git a/backend/tests/routes/search-documents.test.js b/backend/tests/routes/search-documents.test.js index 5492d2268..5ca9b6a0b 100644 --- a/backend/tests/routes/search-documents.test.js +++ b/backend/tests/routes/search-documents.test.js @@ -82,7 +82,9 @@ const writeDocx = async (absolutePath, text) => { }; const search = async (q, query = {}) => { - const response = await request(buildApp()).get('/api/search').query({ q, ...query }); + const response = await request(buildApp()) + .get('/api/search') + .query({ q, ...query }); expect(response.status).toBe(200); return response.body.items || []; }; @@ -178,7 +180,6 @@ describe('the bounds on how much it will read', () => { it('respects the result limit the caller asked for', async () => { const dir = await seed(); for (let i = 0; i < 5; i += 1) { - // eslint-disable-next-line no-await-in-loop await writeDocx(path.join(dir, `doc${i}.docx`), 'the word pangolin appears here'); } diff --git a/backend/tests/routes/search-in-a-share.test.js b/backend/tests/routes/search-in-a-share.test.js new file mode 100644 index 000000000..1c0418985 --- /dev/null +++ b/backend/tests/routes/search-in-a-share.test.js @@ -0,0 +1,142 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Searching inside a shared folder, as the visitor of a link. + * + * A share resolves to a folder inside the volume and is described by another + * name: `share//…`, which is what every result carries and what every + * permission check is made against. The index knows the same files under their + * volume paths. + * + * With the index on, the search consulted it for a base it had never heard of, + * matched nothing, and answered nothing — while having skipped the live scan + * precisely because the index was there. The visitor saw an empty result for a + * file in front of them. + * + * Both halves are covered because both were affected: contents since the index + * was added, names since the catalogue was. + */ + +let envContext; + +const app = (share) => { + const searchRoutes = envContext.requireFresh('src/routes/search'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const server = express(); + // A visitor of the link: no account, a guest session on this share. + server.use((req, _res, next) => { + req.guestSession = { shareId: share.id }; + next(); + }); + server.use('/api', searchRoutes); + server.use(errorHandler); + return server; +}; + +/** A shared folder holding one file, and one file outside it. */ +const seed = async (env = {}) => { + envContext = await setupTestEnv({ + tag: 'search-share-', + env: { SEARCH_DEEP: 'true', ...env }, + }); + const db = await envContext.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1', 'u@example.com', 1, 'u', 'U', '["admin"]', ?, ?)` + ).run(now, now); + + const shared = path.join(envContext.volumeDir, 'Docs', 'partage'); + await fs.mkdir(path.join(shared, 'sous'), { recursive: true }); + await fs.writeFile(path.join(shared, 'sous', 'rapport-2026.txt'), 'du texte pangolin'); + await fs.writeFile( + path.join(envContext.volumeDir, 'Docs', 'dehors-rapport.txt'), + 'pangolin dehors' + ); + + const shares = envContext.requireFresh('src/services/sharesService'); + return shares.createShare({ + ownerId: 'u1', + sourceSpace: 'volume', + sourcePath: 'Docs/partage', + isDirectory: true, + accessMode: 'readonly', + }); +}; + +/** One finished pass, and the mark that says so. */ +const buildIndex = async () => { + const db = await envContext.requireFresh('src/services/indexDb').getIndexDb(); + const { indexTree } = envContext.requireFresh('src/services/searchIndexer'); + const store = envContext.requireFresh('src/services/searchIndexStore'); + await indexTree({ db, rootAbs: envContext.volumeDir, cpuPercent: 100 }); + store.markPassComplete(db); + expect(store.hasNameCatalogue(db)).toBe(true); +}; + +const search = async (share, term) => { + const response = await request(app(share)) + .get('/api/search') + .query({ q: term, path: `share/${share.shareToken}` }); + expect(response.status).toBe(200); + return (response.body.items || []).map((item) => `${item.path}/${item.name}`); +}; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('with the index on', () => { + it('finds a file in the share by its name', async () => { + const share = await seed({ SEARCH_INDEX: 'true' }); + await buildIndex(); + + expect(await search(share, 'rapport')).toEqual([ + `share/${share.shareToken}/sous/rapport-2026.txt`, + ]); + }); + + it('finds one by what it says', async () => { + const share = await seed({ SEARCH_INDEX: 'true' }); + await buildIndex(); + + expect(await search(share, 'pangolin')).toEqual([ + `share/${share.shareToken}/sous/rapport-2026.txt`, + ]); + }); + + // The file outside carries both words. A visitor of the link has no business + // knowing it exists, and no business learning where the share sits on disk. + it('answers about the share and nothing above it', async () => { + const share = await seed({ SEARCH_INDEX: 'true' }); + await buildIndex(); + + for (const term of ['rapport', 'pangolin']) { + const found = await search(share, term); + // Said first: an empty answer satisfies every rule below it and proves + // none of them. + expect(found).toHaveLength(1); + expect(found.every((entry) => entry.startsWith(`share/${share.shareToken}/`))).toBe(true); + expect(found.some((entry) => entry.includes('dehors'))).toBe(false); + } + }); +}); + +describe('with the index off', () => { + it('answers the same, by reading the folder', async () => { + const share = await seed({ SEARCH_INDEX: 'false' }); + + expect(await search(share, 'rapport')).toEqual([ + `share/${share.shareToken}/sous/rapport-2026.txt`, + ]); + expect(await search(share, 'pangolin')).toEqual([ + `share/${share.shareToken}/sous/rapport-2026.txt`, + ]); + }); +}); diff --git a/backend/tests/routes/search-index-lines.test.js b/backend/tests/routes/search-index-lines.test.js new file mode 100644 index 000000000..50e7b6161 --- /dev/null +++ b/backend/tests/routes/search-index-lines.test.js @@ -0,0 +1,190 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The line shown under a result the index found by its contents. + * + * The index keeps words, not text, so that line is read back from the file. + * It was read for every candidate, one after the other, for up to three pages + * of them and before permissions were asked — and a search waited for all of + * it. The index had answered in milliseconds; on a network share, where every + * file comes back across the wire and a PDF is converted again, each search + * then ran to the end of its time budget (#11). + * + * A slow disk is played here by a line reader that takes its time, since what + * is being tested is what the search does while it waits, not the disk. + */ + +let envContext; + +const seed = async (env = {}) => { + envContext = await setupTestEnv({ + tag: 'search-index-lines-', + env: { SEARCH_INDEX: 'true', SEARCH_DEEP: 'true', ...env }, + }); + const db = await envContext.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1', 'u@example.com', 1, 'u', 'U', '["admin"]', ?, ?)` + ).run(now, now); + return envContext.volumeDir; +}; + +const writeMatches = async (folder, count) => { + await fs.mkdir(folder, { recursive: true }); + for (let index = 0; index < count; index += 1) { + const name = `note-${String(index).padStart(3, '0')}.txt`; + await fs.writeFile(path.join(folder, name), `ligne une\nle pangolin numero ${index}\n`); + } +}; + +const buildIndex = async () => { + const db = await envContext.requireFresh('src/services/indexDb').getIndexDb(); + const { indexTree } = envContext.requireFresh('src/services/searchIndexer'); + const store = envContext.requireFresh('src/services/searchIndexStore'); + await indexTree({ db, rootAbs: envContext.volumeDir, cpuPercent: 100 }); + store.markPassComplete(db); +}; + +/** + * Every line read through a reader that waits `ms` first, and counted. + * + * Installed on the module before the route is loaded, because the route takes + * its functions from it when it is loaded. + */ +const slowLines = (ms) => { + const documentText = envContext.requireFresh('src/services/documentText'); + const original = documentText.findPlainTextMatch; + const read = []; + vi.spyOn(documentText, 'findPlainTextMatch').mockImplementation(async (file, ...rest) => { + read.push(file); + if (ms) await new Promise((resolve) => setTimeout(resolve, ms)); + return original(file, ...rest); + }); + return read; +}; + +const search = async (term, query = {}) => { + const searchRoutes = envContext.requireFresh('src/routes/search'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'u1', email: 'u@example.com', roles: ['admin'] }; + next(); + }); + app.use('/api', searchRoutes); + app.use(errorHandler); + + const started = Date.now(); + const response = await request(app) + .get('/api/search') + .query({ q: term, ...query }); + expect(response.status).toBe(200); + return { body: response.body, elapsed: Date.now() - started }; +}; + +afterEach(async () => { + vi.restoreAllMocks(); + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('lines that are slow to read', () => { + for (const ripgrep of ['true', 'false']) { + it(`does not hold the answer for them (ripgrep ${ripgrep})`, async () => { + const volume = await seed({ SEARCH_TIMEOUT_MS: '6000', SEARCH_RIPGREP: ripgrep }); + await writeMatches(path.join(volume, 'Docs'), 30); + await buildIndex(); + // Thirty reads of eight hundred milliseconds: twenty-four seconds one + // after the other, and six even four at a time — the whole budget. + slowLines(800); + + const { body, elapsed } = await search('pangolin'); + + // Two seconds of lines, then the rest as the index gave them — well + // inside the budget, and nothing cut short. + expect(elapsed).toBeLessThan(4000); + expect(body.truncated).toBe(false); + expect(body.items).toHaveLength(30); + for (const item of body.items) expect(item.matchedContent).toBe(true); + + const withLine = body.items.filter((item) => item.matchLine); + const without = body.items.filter((item) => !item.matchLine); + // Read four at a time: two rounds of four fit in the two seconds, where + // one at a time would have shown two lines. + expect(withLine.length).toBeGreaterThanOrEqual(5); + expect(withLine[0].matchLine).toContain('pangolin'); + // The others say where they were found, and carry nothing that was not + // asked for. + expect(without.length).toBeGreaterThan(0); + for (const item of without) { + expect(item).not.toHaveProperty('inContents'); + expect(item).not.toHaveProperty('score'); + } + }, 20000); + } +}); + +describe('lines that come quickly', () => { + it('are all shown, as before', async () => { + const volume = await seed({ SEARCH_RIPGREP: 'false' }); + await writeMatches(path.join(volume, 'Docs'), 10); + await buildIndex(); + + const { body } = await search('pangolin'); + + expect(body.items).toHaveLength(10); + for (const item of body.items) { + expect(item.matchLine).toContain('pangolin'); + expect(item.matchLineNumber).toBe(2); + } + }); + + it('are not read for more results than the page can hold', async () => { + const volume = await seed({ SEARCH_RIPGREP: 'false' }); + await writeMatches(path.join(volume, 'Docs'), 60); + await buildIndex(); + const read = slowLines(0); + + const { body } = await search('pangolin', { limit: 10 }); + + expect(body.items).toHaveLength(10); + // Ten for the page, and no more than the few already being read when it + // filled. It used to be every row the index handed over — fifty here. + expect(read.length).toBeLessThanOrEqual(14); + }); + + it('leaves out a file that no longer says what the index remembers', async () => { + const volume = await seed({ SEARCH_RIPGREP: 'false' }); + await writeMatches(path.join(volume, 'Docs'), 3); + await buildIndex(); + await fs.writeFile(path.join(volume, 'Docs', 'note-001.txt'), 'plus rien\n'); + + const { body } = await search('pangolin'); + + expect(body.items.map((item) => item.name).sort()).toEqual(['note-000.txt', 'note-002.txt']); + }); +}); + +describe('a file the reader may not see', () => { + it('is not opened to find its line', async () => { + const volume = await seed({ SEARCH_RIPGREP: 'false' }); + await writeMatches(path.join(volume, 'Prive'), 5); + await writeMatches(path.join(volume, 'Public'), 5); + await buildIndex(); + const accessControl = envContext.requireFresh('src/services/accessControlService'); + await accessControl.setRules([{ path: 'Prive', recursive: true, permissions: 'hidden' }]); + const read = slowLines(0); + + const { body } = await search('pangolin'); + + expect(body.items.every((item) => item.path === 'Public')).toBe(true); + expect(body.items).toHaveLength(5); + expect(read.filter((file) => file.includes(`${path.sep}Prive${path.sep}`))).toEqual([]); + }); +}); diff --git a/backend/tests/routes/search-ordinary-folder-names.test.js b/backend/tests/routes/search-ordinary-folder-names.test.js new file mode 100644 index 000000000..f6ca2c2d0 --- /dev/null +++ b/backend/tests/routes/search-ordinary-folder-names.test.js @@ -0,0 +1,168 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Folder names an editor skips and a file server must not. + * + * Search carried `.git`, `node_modules`, `dist` and `build` as names to walk + * past — a habit that belongs in a code editor. Here they are folder names + * somebody may have put a year of work in, and a file under one of them could + * not be found by name or by content, with nothing in the answer to say why + * (#11). + * + * Both engines are exercised, because the names were hard-coded twice: once as + * ripgrep globs and once in the walker that runs when ripgrep is absent. + */ + +let envContext; + +const buildApp = () => { + const searchRoutes = envContext.requireFresh('src/routes/search'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'u1', email: 'u@example.com', roles: ['admin'] }; + next(); + }); + app.use('/api', searchRoutes); + app.use(errorHandler); + return app; +}; + +const seed = async (env = {}) => { + envContext = await setupTestEnv({ + tag: 'search-folder-names-', + env: { SEARCH_RIPGREP: 'true', SEARCH_DEEP: 'true', ...env }, + }); + const dbService = envContext.requireFresh('src/services/db'); + const db = await dbService.getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1', 'u@example.com', 1, 'u', 'U', '["admin"]', ?, ?)` + ).run(now, now); + return path.join(envContext.volumeDir, 'Docs'); +}; + +const search = async (term) => { + const response = await request(buildApp()).get('/api/search').query({ q: term }); + expect(response.status).toBe(200); + return (response.body.items || []).map((item) => `${item.path}/${item.name}`); +}; + +/** One file at `relDir/name`, and what searching for `term` returns. */ +const withFile = async ({ relDir, name, contents = 'nothing in particular', term, env }) => { + const docs = await seed(env); + const dir = path.join(docs, relDir); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, name), contents); + return search(term); +}; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('a folder named like a build directory', () => { + for (const folder of ['build', 'dist', 'node_modules']) { + it(`finds a file by name under ${folder}`, async () => { + const found = await withFile({ + relDir: `projects/${folder}/reports`, + name: 'quarterly.txt', + term: 'quarterly', + }); + expect(found).toContain(`Docs/projects/${folder}/reports/quarterly.txt`); + }); + } + + it('finds what such a folder holds, by its contents', async () => { + const found = await withFile({ + relDir: 'projects/build', + name: 'notes.txt', + contents: 'the measurement was taken at dawn', + term: 'dawn', + }); + expect(found).toContain('Docs/projects/build/notes.txt'); + }); + + it('finds the folder itself', async () => { + const found = await withFile({ + relDir: 'projects/build/reports', + name: 'quarterly.txt', + term: 'build', + }); + expect(found).toContain('Docs/projects/build'); + }); + + it('finds it with the walker too, when ripgrep is not there', async () => { + const found = await withFile({ + relDir: 'projects/build/reports', + name: 'quarterly.txt', + term: 'quarterly', + env: { SEARCH_RIPGREP: 'false' }, + }); + expect(found).toContain('Docs/projects/build/reports/quarterly.txt'); + }); +}); + +describe('what may still be left out', () => { + // The mechanism that decides is the administrator's, not a name in the + // source: removing the four must not remove this one. + it('honours the exclusion list', async () => { + const found = await withFile({ + relDir: 'private', + name: 'quarterly.txt', + term: 'quarterly', + env: { SEARCH_INDEX_EXCLUDE: 'Docs/private' }, + }); + expect(found).toEqual([]); + }); + + it('leaves hidden folders hidden while the reader has not asked', async () => { + const found = await withFile({ + relDir: '.private', + name: 'quarterly.txt', + term: 'quarterly', + }); + expect(found).toEqual([]); + }); +}); + +/** + * And the third place the same four names were written down. + * + * The index is a fourth engine, not a variant of the other two: it walks the + * volume itself, on its own schedule, and the search answers from what it + * catalogued. A name it never walked into is a name no amount of asking will + * return — and unlike the other two, being absent from the index shows up as + * an empty answer with nothing to explain it, however the search is run. + */ +describe('a folder named like a build directory, catalogued', () => { + const indexAndSearch = async (relDir, name, term) => { + const docs = await seed({ SEARCH_INDEX_ENABLED: 'true' }); + const dir = path.join(docs, relDir); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, name), 'le mot pangolin'); + + const indexDb = envContext.requireFresh('src/services/indexDb'); + const db = await indexDb.getIndexDb(); + const indexer = envContext.requireFresh('src/services/searchIndexer'); + await indexer.indexTree({ db, rootAbs: envContext.volumeDir, cpuPercent: 100 }); + + const store = envContext.requireFresh('src/services/searchIndexStore'); + return { catalogued: store.search(db, term) }; + }; + + for (const folder of ['build', 'dist', 'node_modules']) { + it(`catalogues what is under ${folder}`, async () => { + const { catalogued } = await indexAndSearch(`projets/${folder}`, 'rapport.txt', 'pangolin'); + + expect(catalogued).toContain(`Docs/projets/${folder}/rapport.txt`); + }); + } +}); diff --git a/backend/tests/routes/search-terms-and-permissions.test.js b/backend/tests/routes/search-terms-and-permissions.test.js new file mode 100644 index 000000000..376f2c389 --- /dev/null +++ b/backend/tests/routes/search-terms-and-permissions.test.js @@ -0,0 +1,215 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Two things the catalogue changed the shape of, and neither had a test. + * + * A name search used to be a walk, which compared strings in JavaScript and + * asked the permission resolver about every candidate. It is a SQL scan now: + * the term becomes part of a `LIKE`, where `%` and `_` mean something, and the + * rows come back from a table that knows nothing about who may read what. Both + * halves of that are worth stating rather than assuming. + */ + +let envContext; + +const buildApp = () => { + const searchRoutes = envContext.requireFresh('src/routes/search'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'u1', email: 'u@example.com', roles: ['admin'] }; + next(); + }); + app.use('/api', searchRoutes); + app.use(errorHandler); + return app; +}; + +const seed = async (env = {}) => { + envContext = await setupTestEnv({ + tag: 'search-terms-', + env: { SEARCH_INDEX: 'true', SEARCH_DEEP: 'true', ...env }, + }); + const db = await envContext.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1', 'u@example.com', 1, 'u', 'U', '["admin"]', ?, ?)` + ).run(now, now); + return envContext.volumeDir; +}; + +const buildIndex = async () => { + const db = await envContext.requireFresh('src/services/indexDb').getIndexDb(); + const { indexTree } = envContext.requireFresh('src/services/searchIndexer'); + const store = envContext.requireFresh('src/services/searchIndexStore'); + await indexTree({ db, rootAbs: envContext.volumeDir, cpuPercent: 100 }); + store.markPassComplete(db); + expect(store.hasNameCatalogue(db)).toBe(true); +}; + +const search = async (term) => { + const response = await request(buildApp()).get('/api/search').query({ q: term }); + expect(response.status).toBe(200); + return (response.body.items || []).map((item) => `${item.path}/${item.name}`); +}; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('what a typed term may contain', () => { + // `%` and `_` are the two characters SQL reads as "anything" — the first as + // any run, the second as any single character. A term is typed by a person, + // so both are just characters, and a search for a discount of 100% must not + // offer everything beginning with 100. + it('treats the wildcards of SQL as ordinary characters', async () => { + const volume = await seed(); + const docs = path.join(volume, 'Docs'); + await fs.mkdir(docs, { recursive: true }); + await fs.writeFile(path.join(docs, 'remise 100% acquise.txt'), 'x'); + await fs.writeFile(path.join(docs, 'remise 1006 acquise.txt'), 'x'); + await fs.writeFile(path.join(docs, 'un_sous_tiret.txt'), 'x'); + await fs.writeFile(path.join(docs, 'unXsousYtiret.txt'), 'x'); + await buildIndex(); + + expect(await search('100%')).toEqual(['Docs/remise 100% acquise.txt']); + expect(await search('un_sous')).toEqual(['Docs/un_sous_tiret.txt']); + }); + + // Everything after `--` is positional for ripgrep, which is what keeps a term + // starting with a dash from being read as a flag. The catalogue has no such + // hazard, and both have to answer the same. + it('takes a term that begins with a dash', async () => { + const volume = await seed(); + await fs.mkdir(path.join(volume, 'Docs'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Docs', '-commence-par-tiret.txt'), 'x'); + await buildIndex(); + + expect(await search('-commence')).toEqual(['Docs/-commence-par-tiret.txt']); + }); + + it('takes a term with a space in it', async () => { + const volume = await seed(); + await fs.mkdir(path.join(volume, 'Docs'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Docs', 'proces verbal 2026.txt'), 'x'); + await fs.writeFile(path.join(volume, 'Docs', 'proces-verbal-2026.txt'), 'x'); + await buildIndex(); + + expect(await search('proces verbal')).toEqual(['Docs/proces verbal 2026.txt']); + }); + + it('answers across the volumes, not only the first', async () => { + const volume = await seed(); + await fs.mkdir(path.join(volume, 'Usb', 'a'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Nvm'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Usb', 'a', 'rapport-usb.txt'), 'x'); + await fs.writeFile(path.join(volume, 'Nvm', 'rapport-nvm.txt'), 'x'); + await buildIndex(); + + expect(await search('rapport')).toEqual(['Nvm/rapport-nvm.txt', 'Usb/a/rapport-usb.txt']); + }); +}); + +describe('what the catalogue may not reveal', () => { + /** + * The rows come from a table that knows nothing about who may read what, and + * they no longer pass through the walk that used to be the thing asking. The + * permission resolver is consulted on every result instead — and the point of + * a test here is that a mistake in this direction is not a wrong answer, it + * is a disclosure. + */ + const withHiddenFolder = async () => { + const volume = await seed(); + await fs.mkdir(path.join(volume, 'Prive'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Public'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Prive', 'secret-rapport.txt'), 'le mot pangolin'); + await fs.writeFile(path.join(volume, 'Public', 'ouvert-rapport.txt'), 'le mot pangolin'); + await buildIndex(); + + // Both are there to begin with: an empty answer below would otherwise + // prove nothing at all. Ordered by the name, the two being the same length + // and equally close to the term. + expect(await search('rapport')).toEqual([ + 'Public/ouvert-rapport.txt', + 'Prive/secret-rapport.txt', + ]); + + const accessControl = envContext.requireFresh('src/services/accessControlService'); + await accessControl.setRules([{ path: 'Prive', recursive: true, permissions: 'hidden' }]); + }; + + it('keeps a hidden folder out of a search by name', async () => { + await withHiddenFolder(); + expect(await search('rapport')).toEqual(['Public/ouvert-rapport.txt']); + }); + + it('keeps it out of a search by contents', async () => { + await withHiddenFolder(); + expect(await search('pangolin')).toEqual(['Public/ouvert-rapport.txt']); + }); + + it('does not offer the folder itself either', async () => { + await withHiddenFolder(); + expect(await search('Prive')).toEqual([]); + }); +}); + +describe('a term too short to be worth answering', () => { + // One or two characters describe most of a volume. The catalogue would + // answer — a scan is a scan — with the first hundred rows that happen to + // hold the letter, which is not something anybody asked for; the walk reads + // the whole storage to say the same. + it('refuses fewer than three characters', async () => { + await seed(); + + for (const term of ['a', 'ab']) { + const response = await request(buildApp()).get('/api/search').query({ q: term }); + expect(response.status).toBe(400); + } + }); + + it('answers three', async () => { + const volume = await seed(); + await fs.mkdir(path.join(volume, 'Docs'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Docs', 'abc-rapport.txt'), 'x'); + await buildIndex(); + + expect(await search('abc')).toEqual(['Docs/abc-rapport.txt']); + }); +}); + +describe('a word the reader has only begun to type', () => { + /** + * FTS5 matches whole words. Searching `azul` found nothing at all while + * `azules` found the document holding it — the index and the live scan + * answering two different questions from the same box, and the shorter term + * being the one that failed, which reads as the search being broken. + */ + const withSpanish = async (env) => { + const volume = await seed(env); + await fs.mkdir(path.join(volume, 'Docs'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Docs', 'film.json'), '{"Title":"Los azules del mar"}'); + return volume; + }; + + it('finds the word from its beginning, through the index', async () => { + await withSpanish(); + await buildIndex(); + + expect(await search('azul')).toEqual(['Docs/film.json']); + expect(await search('azules')).toEqual(['Docs/film.json']); + }); + + it('finds it the same way without the index', async () => { + await withSpanish({ SEARCH_INDEX: 'false' }); + + expect(await search('azul')).toEqual(['Docs/film.json']); + }); +}); diff --git a/backend/tests/routes/settings-concurrent-saves.test.js b/backend/tests/routes/settings-concurrent-saves.test.js new file mode 100644 index 000000000..f1c38f344 --- /dev/null +++ b/backend/tests/routes/settings-concurrent-saves.test.js @@ -0,0 +1,137 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { fileURLToPath } from 'node:url'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { substituteModule } from '../helpers/substitute-module.js'; + +/** + * Two saves of one settings section, in flight at the same moment. + * + * The page sends what changed rather than the whole document, so the server + * merges each section over what is stored. That merge used to read the + * settings at the start of the request and write the result two awaits later: + * two administrators saving at once, or one with the settings open in two + * tabs, both started from the same stored value and the second wrote over the + * first's field — while telling the person who set it that it was saved. + * Branding was taken out of that path already, where a lost save also left a + * logo file behind; every other section had the same hole. + * + * The database answers synchronously, so a read and a write with nothing + * awaited between them cannot be interleaved. That is what is pinned here: the + * moment where the second request could slip in is the `await` on the database + * handle, and both requests are held at it until both have arrived. With the + * read and the write on either side of it, the second overwrites the first; + * with both after it, the second reads what the first has just written. + */ + +const SERVICE_FILE = fileURLToPath( + new URL('../../src/services/settingsService.js', import.meta.url) +); + +let currentEnv; +let restore = null; + +afterEach(async () => { + restore?.(); + restore = null; + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +/** + * Hold the first two callers until both have arrived, and let everything after + * them straight through. + */ +const holdFirstTwo = () => { + let arrived = 0; + let release; + const both = new Promise((resolve) => { + release = resolve; + }); + return async () => { + arrived += 1; + if (arrived > 2) return; + if (arrived === 2) release(); + await both; + }; +}; + +const buildApp = () => { + const routes = currentEnv.requireFresh('src/routes/settings'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: 'u1', email: 'u@example.com', roles: ['admin'] }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +/** + * An application whose settings service waits on the gate every time it asks + * for the database handle. + */ +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'settings-concurrent-' }); + const db = currentEnv.requireFresh('src/services/db'); + await db.getDb(); + + const gate = holdFirstTwo(); + // Substituted after the environment cleared the module registry, so the + // settings service required next is the one that receives this. + restore = substituteModule(SERVICE_FILE, './db', { + ...db, + getDb: async () => { + await gate(); + return db.getDb(); + }, + }); + + return buildApp(); +}; + +const patch = (app, payload) => request(app).patch('/api/settings').send(payload); +const readAsAdmin = async (app) => (await request(app).get('/api/settings')).body; + +describe('two saves of one settings section at once', () => { + it.each([ + ['the trash', 'trash', { retentionDays: 90 }, { maxPercent: 40 }], + ['file versions', 'versions', { maxPerFile: 7 }, { dailyDays: 60 }], + ['thumbnails', 'thumbnails', { size: 320 }, { quality: 55 }], + ['uploads', 'uploads', { chunkSizeBytes: 16 * 1024 * 1024 }, { chunkedEnabled: true }], + ])('keep both fields: %s', async (_label, section, first, second) => { + const app = await seed(); + + const answers = await Promise.all([ + patch(app, { [section]: first }), + patch(app, { [section]: second }), + ]); + + expect(answers.map((answer) => answer.status)).toEqual([200, 200]); + expect((await readAsAdmin(app))[section]).toMatchObject({ ...first, ...second }); + }); + + /** + * The exclusions are a list rather than a set of fields, so the two saves + * cannot both survive — the second replaces the list. What must hold is that + * the one the person is told about is the one that is stored. + */ + it('leaves the folder size exclusions as the last answer says they are', async () => { + const app = await seed(); + + const answers = await Promise.all([ + patch(app, { folderSize: { excludedPaths: ['Archive'] } }), + patch(app, { folderSize: { excludedPaths: ['Archive', 'Backups'] } }), + ]); + + expect(answers.map((answer) => answer.status)).toEqual([200, 200]); + const stored = (await readAsAdmin(app)).folderSize.excludedPaths; + expect(stored).toEqual(answers.at(-1).body.folderSize.excludedPaths); + }); +}); diff --git a/backend/tests/routes/settings-exclusions.test.js b/backend/tests/routes/settings-exclusions.test.js new file mode 100644 index 000000000..485d62480 --- /dev/null +++ b/backend/tests/routes/settings-exclusions.test.js @@ -0,0 +1,124 @@ +import { describe, it, expect } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Written at the layer the browser actually talks to. + * + * `SEARCH_INDEX_EXCLUDE` was set, the service that reads settings reported it, + * a test asserted exactly that — and the page still said "no path configured", + * because the route does not call that function. It calls one that assembles + * the admin payload field by field, and the new field was not in the list. A + * test one layer below the defect cannot see the defect. + */ +let envContext; + +const buildApp = (roles) => { + const settingsRoutes = envContext.requireFresh('src/routes/settings'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: 'u1', email: 'u@example.com', roles }; + next(); + }); + app.use('/api', settingsRoutes); + app.use(errorHandler); + return app; +}; + +const seed = async (env) => { + envContext = await setupTestEnv({ tag: 'settings-exclusions-', env }); + const dbService = envContext.requireFresh('src/services/db'); + await dbService.getDb(); +}; + +describe('what GET /api/settings tells an administrator', () => { + it('carries the search index exclusions the environment set', async () => { + await seed({ SEARCH_INDEX: 'true', SEARCH_INDEX_EXCLUDE: 'Stacks/docker' }); + try { + const response = await request(buildApp(['admin'])).get('/api/settings'); + + expect(response.status).toBe(200); + expect(response.body.searchIndex).toBeTruthy(); + expect(response.body.searchIndex.environmentExcludedPaths).toEqual(['Stacks/docker']); + // Beside the folder-size ones, which have always been there. + expect(response.body.folderSize).toBeTruthy(); + } finally { + await envContext.cleanup(); + } + }); + + it('does not carry them to someone who is not an administrator', async () => { + await seed({ SEARCH_INDEX: 'true', SEARCH_INDEX_EXCLUDE: 'Stacks/docker' }); + try { + const response = await request(buildApp(['user'])).get('/api/settings'); + + expect(response.status).toBe(200); + expect(response.body.searchIndex).toBeUndefined(); + } finally { + await envContext.cleanup(); + } + }); + + it('takes a path an administrator adds and gives it back', async () => { + await seed({ SEARCH_INDEX: 'true' }); + try { + const app = buildApp(['admin']); + const saved = await request(app) + .patch('/api/settings') + .send({ searchIndex: { excludedPaths: ['Sauvegardes/2024'] } }); + expect(saved.status).toBe(200); + + const response = await request(app).get('/api/settings'); + expect(response.body.searchIndex.excludedPaths).toEqual(['Sauvegardes/2024']); + } finally { + await envContext.cleanup(); + } + }); +}); + +/** + * Written down is not the same as in effect. + * + * Saving the list and telling the worker about it are two separate steps, and + * a test that reads the setting back sees only the first. Skipping the second + * leaves the running indexer walking a folder an administrator has just + * excluded, with the settings page showing it excluded — which is the worst + * shape a setting can take. + */ +describe('an exclusion an administrator adds while the index is running', () => { + it('reaches the worker, not only the stored settings', async () => { + await seed({ SEARCH_INDEX: 'true' }); + try { + const exclusions = envContext.requireFresh('src/services/searchIndexExclusions'); + expect(exclusions.effectivePaths()).not.toContain('Sauvegardes/2024'); + + const response = await request(buildApp(['admin'])) + .patch('/api/settings') + .send({ searchIndex: { excludedPaths: ['Sauvegardes/2024'] } }); + expect(response.status).toBe(200); + + // The worker decides what it walks from this list, not from the database. + expect(exclusions.effectivePaths()).toContain('Sauvegardes/2024'); + } finally { + await envContext.cleanup(); + } + }); + + it('does the same for folder sizes', async () => { + await seed({ FOLDER_SIZE_MODE: 'full' }); + try { + const exclusions = envContext.requireFresh('src/services/folderSizeExclusions'); + + await request(buildApp(['admin'])) + .patch('/api/settings') + .send({ folderSize: { excludedPaths: ['Media/raw'] } }); + + expect(exclusions.effectivePaths()).toContain('Media/raw'); + } finally { + await envContext.cleanup(); + } + }); +}); diff --git a/backend/tests/routes/settings-partial-updates.test.js b/backend/tests/routes/settings-partial-updates.test.js new file mode 100644 index 000000000..e21cd3a97 --- /dev/null +++ b/backend/tests/routes/settings-partial-updates.test.js @@ -0,0 +1,587 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A settings write that sends part of a section, or sends a field in the wrong + * shape. + * + * The settings page sends what changed, not the whole document, so the route + * merges each section over what is stored. And it drops a field that is not in + * the shape the field takes before anything is stored — which matters more than + * it looks, because the service underneath repairs a bad value by putting the + * *default* in its place. Without the route's check, a trash retention of + * ninety days sent back as "forever" becomes thirty, and an access rule list + * sent as anything other than a list becomes no rules at all: every folder an + * administrator had hidden, visible again. + * + * So every case here first stores a value that differs from the default, then + * sends the bad one, then reads back — a test that started from the default + * could not tell the route's refusal from the service's repair. + * + * Who may write which section is pinned in `settings-write-boundary.test.js`. + */ + +const MiB = 1024 * 1024; + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'settings-partial-' }); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1','u@example.com',1,'u','U','["user"]', ?, ?)` + ).run(now, now); + return db; +}; + +const buildApp = (roles) => { + const routes = currentEnv.requireFresh('src/routes/settings'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: 'u1', email: 'u@example.com', roles }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +const patch = (roles, payload) => request(buildApp(roles)).patch('/api/settings').send(payload); +const readAsAdmin = async () => (await request(buildApp(['admin'])).get('/api/settings')).body; + +describe('a field sent in a shape it does not take', () => { + it.each([ + [ + 'thumbnails.quality', + { thumbnails: { quality: 55 } }, + { thumbnails: { quality: 'best' } }, + (s) => s.thumbnails.quality, + 55, + ], + [ + 'uploads.chunkSizeBytes', + { uploads: { chunkSizeBytes: 16 * MiB } }, + { uploads: { chunkSizeBytes: 'huge' } }, + (s) => s.uploads.chunkSizeBytes, + 16 * MiB, + ], + [ + 'uploads.chunkedEnabled', + { uploads: { chunkedEnabled: true } }, + { uploads: { chunkedEnabled: 'yes' } }, + (s) => s.uploads.chunkedEnabled, + true, + ], + [ + 'trash.retentionDays', + { trash: { retentionDays: 90 } }, + { trash: { retentionDays: 'forever' } }, + (s) => s.trash.retentionDays, + 90, + ], + [ + 'trash.maxBytes', + { trash: { maxBytes: 5_000_000_000 } }, + { trash: { maxBytes: 'lots' } }, + (s) => s.trash.maxBytes, + 5_000_000_000, + ], + [ + 'versions.maxPerFile', + { versions: { maxPerFile: 7 } }, + { versions: { maxPerFile: 'many' } }, + (s) => s.versions.maxPerFile, + 7, + ], + [ + 'branding.appName', + { branding: { appName: 'Files' } }, + { branding: { appName: 42 } }, + (s) => s.branding.appName, + 'Files', + ], + ])( + 'leaves %s as it was, not reset to its default', + async (_field, stored, sent, readBack, kept) => { + await seed(); + await patch(['admin'], stored); + expect(readBack(await readAsAdmin())).toEqual(kept); + + const response = await patch(['admin'], sent); + + expect(response.status).toBe(200); + expect(readBack(await readAsAdmin())).toEqual(kept); + } + ); + + it('leaves thumbnails as they were when "enabled" is not a boolean', async () => { + await seed(); + await patch(['admin'], { thumbnails: { enabled: false } }); + + // Anything present used to count, and the service reads what is not a + // boolean as on: "false" switched thumbnails on for everybody. + const response = await patch(['admin'], { thumbnails: { enabled: 'false' } }); + + expect(response.status).toBe(200); + expect((await readAsAdmin()).thumbnails.enabled).toBe(false); + }); + + /** The one field where "nothing" is a value: no cap on the trash. */ + it('takes null for the trash size cap, which removes the cap', async () => { + await seed(); + await patch(['admin'], { trash: { maxBytes: 5_000_000_000 } }); + + await patch(['admin'], { trash: { maxBytes: null } }); + + expect((await readAsAdmin()).trash.maxBytes).toBeNull(); + }); +}); + +/** + * A number, but not one anybody chose: what an emptied or mistyped field sends. + * + * The shape is right, so the check above let these through, and the service + * brought each up to its lowest bound — a chunk size of 0 stored as 1 MiB, a + * thumbnail size of 0 as 64 pixels — in place of what the administrator had. + * A positive value beyond a bound is still brought within it. + */ +describe('a size or a count of nothing', () => { + it.each([ + ['uploads.chunkSizeBytes', 0, 'uploads', 'chunkSizeBytes', 16 * MiB], + ['uploads.chunkSizeBytes', -MiB, 'uploads', 'chunkSizeBytes', 16 * MiB], + ['thumbnails.size', 0, 'thumbnails', 'size', 320], + ['thumbnails.quality', -5, 'thumbnails', 'quality', 55], + ['thumbnails.concurrency', 0, 'thumbnails', 'concurrency', 4], + // The trash and the file versions, which the settings page already refuses + // with these bounds — this is what an API client saw instead. + ['trash.retentionDays', 0, 'trash', 'retentionDays', 90], + ['trash.retentionDays', -5, 'trash', 'retentionDays', 90], + ['trash.maxPercent', 0, 'trash', 'maxPercent', 40], + ['trash.maxBytes', 0, 'trash', 'maxBytes', 5_000_000_000], + ['versions.keepAllHours', 0, 'versions', 'keepAllHours', 48], + ['versions.hourlyDays', -3, 'versions', 'hourlyDays', 14], + ['versions.dailyDays', 0, 'versions', 'dailyDays', 60], + ['versions.maxPerFile', 0, 'versions', 'maxPerFile', 7], + ['versions.sessionCheckpointMinutes', -1, 'versions', 'sessionCheckpointMinutes', 30], + ])('leaves %s as it was when sent %j', async (_label, sent, section, field, kept) => { + await seed(); + await patch(['admin'], { [section]: { [field]: kept } }); + + const response = await patch(['admin'], { [section]: { [field]: sent } }); + + expect(response.status).toBe(200); + expect((await readAsAdmin())[section][field]).toBe(kept); + }); + + it('still brings a positive value beyond its bounds within them', async () => { + await seed(); + + await patch(['admin'], { + thumbnails: { size: 5000 }, + uploads: { chunkSizeBytes: 1024 }, + trash: { retentionDays: 9000 }, + versions: { maxPerFile: 5000 }, + }); + + const settings = await readAsAdmin(); + expect(settings.thumbnails.size).toBe(1024); + expect(settings.uploads.chunkSizeBytes).toBe(MiB); + expect(settings.trash.retentionDays).toBe(3650); + expect(settings.versions.maxPerFile).toBe(1000); + }); +}); + +describe('the application name', () => { + it.each([[''], [' ']])('is left as it was when sent as %j', async (appName) => { + await seed(); + await patch(['admin'], { branding: { appName: 'Files' } }); + + const response = await patch(['admin'], { branding: { appName } }); + + expect(response.status).toBe(200); + expect(response.body.branding.appName).toBe('Files'); + expect((await readAsAdmin()).branding.appName).toBe('Files'); + }); + + it('reads as the default where an empty one was stored before', async () => { + const db = await seed(); + db.prepare( + `INSERT INTO system_settings (id, category, key, value, updated_at) + VALUES ('b1', 'branding', 'branding', ?, ?)` + ).run(JSON.stringify({ appName: ' ', appLogoUrl: '/logo.svg' }), new Date().toISOString()); + + const response = await request(buildApp([])).get('/api/branding'); + + expect(response.body.appName).toBe('Explorer'); + }); +}); + +describe('a section sent with only some of its fields', () => { + it('changes those fields and leaves the rest of the section as it was', async () => { + await seed(); + await patch(['admin'], { + trash: { retentionDays: 90, maxPercent: 40 }, + versions: { maxPerFile: 7, dailyDays: 60 }, + }); + + await patch(['admin'], { trash: { retentionDays: 7 }, versions: { maxPerFile: 9 } }); + + const { trash, versions } = await readAsAdmin(); + expect(trash).toMatchObject({ retentionDays: 7, maxPercent: 40 }); + expect(versions).toMatchObject({ maxPerFile: 9, dailyDays: 60 }); + }); +}); + +describe('a list sent as something that is not a list', () => { + const HIDDEN_RULE = { path: 'Private', permissions: 'hidden', recursive: true }; + const rulesOf = (settings) => settings.access.rules.map((r) => `${r.path}:${r.permissions}`); + + it.each([ + [ + 'the access rules', + { access: { rules: [HIDDEN_RULE] } }, + { access: { rules: 'none' } }, + rulesOf, + ['Private:hidden'], + ], + [ + 'the access rules, when the list is missing', + { access: { rules: [HIDDEN_RULE] } }, + { access: {} }, + rulesOf, + ['Private:hidden'], + ], + [ + 'the search index exclusions', + { searchIndex: { excludedPaths: ['Private'] } }, + { searchIndex: { excludedPaths: 'Elsewhere' } }, + (s) => s.searchIndex.excludedPaths, + ['Private'], + ], + [ + 'the folder size exclusions', + { folderSize: { excludedPaths: ['Private'] } }, + { folderSize: { excludedPaths: null } }, + (s) => s.folderSize.excludedPaths, + ['Private'], + ], + ])('leaves %s in place', async (_label, stored, sent, readBack, kept) => { + await seed(); + await patch(['admin'], stored); + expect(readBack(await readAsAdmin())).toEqual(kept); + + const response = await patch(['admin'], sent); + + expect(response.status).toBe(200); + expect(readBack(await readAsAdmin())).toEqual(kept); + }); +}); + +/** + * A rule the server cannot store as it was written. + * + * Every one of these used to be sanitised away with a 200: the row for + * `../Secret` vanished from the page the moment it was saved, and an + * administrator was left believing a folder was hidden that never was. Worse, + * permissions that were not one of the three became `rw`, so a mistyped + * `readonly` opened a folder for writing instead of refusing the word. + * + * Each one stores a good rule first, so a refusal can be told from a list that + * was replaced by nothing. + */ +describe('an access rule the server cannot store', () => { + const STORED = { id: 'kept', path: 'Private', permissions: 'hidden', recursive: true }; + + it.each([ + [ + 'a path that climbs out of the volume', + { path: '../Secret', permissions: 'hidden' }, + /Traversal outside the volume root/, + ], + ['no path at all', { path: '', permissions: 'ro' }, /a rule needs the path of a folder/], + [ + 'permissions that are not one of the three', + { path: 'Legal', permissions: 'readonly' }, + /is not one of the permissions/, + ], + [ + 'a recursive flag that is not one', + { path: 'Legal', permissions: 'ro', recursive: 'yes' }, + /does not say whether the rule covers what is inside/, + ], + ['something that is not a rule', 'Legal', /this is not a rule/], + ])('is refused, with the reason, and changes nothing: %s', async (_label, rule, reason) => { + await seed(); + await patch(['admin'], { access: { rules: [STORED] } }); + + const response = await patch(['admin'], { access: { rules: [STORED, rule] } }); + + expect(response.status).toBe(400); + expect(response.body.error.message).toMatch(reason); + // Numbered as the page numbers the rows, so the reason names the one to fix. + expect(response.body.error.message).toMatch(/^Access rule 2: /); + expect((await readAsAdmin()).access.rules).toEqual([expect.objectContaining(STORED)]); + }); + + /** + * Read back, the same rule is still dropped rather than refused. A value an + * older version stored, or one edited into app.db by hand, must not make the + * settings unreadable — unreadable settings are every hidden folder visible. + */ + it('is dropped, not refused, when it is already in the database', async () => { + const db = await seed(); + db.prepare( + `INSERT INTO system_settings (id, category, key, value, updated_at) + VALUES ('a1', 'system', 'access', ?, ?)` + ).run( + JSON.stringify({ rules: [STORED, { path: '../Secret', permissions: 'hidden' }] }), + new Date().toISOString() + ); + + const settings = await readAsAdmin(); + + expect(settings.access.rules).toEqual([expect.objectContaining(STORED)]); + }); +}); + +describe('what a regular account may not change', () => { + /** + * `settings-write-boundary.test.js` covers one field of five sections. These + * are the other three, and the access rules are the ones that decide which + * folders anybody may see. + */ + it.each([ + ['the access rules', { access: { rules: [] } }, (s) => s.access.rules.length, 1], + ['the trash', { trash: { retentionDays: 1 } }, (s) => s.trash.retentionDays, 90], + ['the file versions', { versions: { maxPerFile: 1 } }, (s) => s.versions.maxPerFile, 7], + ])('is refused, and unchanged: %s', async (_label, sent, readBack, kept) => { + await seed(); + await patch(['admin'], { + access: { rules: [{ path: 'Private', permissions: 'hidden', recursive: true }] }, + trash: { retentionDays: 90 }, + versions: { maxPerFile: 7 }, + }); + + const response = await patch(['user'], sent); + + expect(response.status).toBe(403); + expect(response.body.error).toBe('Admin access required for system settings.'); + expect(readBack(await readAsAdmin())).toBe(kept); + }); +}); + +/** + * A save the route refuses halfway. + * + * One payload carries a section per group, and the sections used to be applied + * one after another: a valid one before a refused one was stored, and the + * answer was still 400. The person saw their save refused, the page kept the + * values it had sent, and the server had taken some of them — the two + * disagreed until the next reload, which is the worst state of the three. + * + * The access rules are the only section that refuses what it was sent, so they + * are what makes this reachable. The sections are checked in the order they + * are declared, and thumbnails come first: it is written before access is + * reached, or it is not written at all. + */ +describe('a payload with a valid section and a refused one', () => { + const REFUSED = { access: { rules: [{ path: 'Private', permissions: 'sideways' }] } }; + + it('stores none of it, and says which rule it refused', async () => { + await seed(); + await patch(['admin'], { thumbnails: { size: 321 } }); + + const response = await patch(['admin'], { ...REFUSED, thumbnails: { size: 654 } }); + + expect(response.status).toBe(400); + expect(response.body.error.message).toMatch(/Access rule 1/); + expect((await readAsAdmin()).thumbnails.size).toBe(321); + }); + + /** + * The other direction, so this cannot pass on the order of the sections + * alone: branding is written after access, and must be no more stored than + * thumbnails was. + */ + it('stores nothing that comes after the refusal either', async () => { + await seed(); + await patch(['admin'], { branding: { appName: 'Before' } }); + + const response = await patch(['admin'], { ...REFUSED, branding: { appName: 'After' } }); + + expect(response.status).toBe(400); + expect((await readAsAdmin()).branding.appName).toBe('Before'); + }); + + /** A preference of one's own is not stored by a save the server refuses. */ + it('leaves the sender’s own preferences alone', async () => { + await seed(); + await patch(['admin'], { user: { showHiddenFiles: true } }); + + const response = await patch(['admin'], { ...REFUSED, user: { showHiddenFiles: false } }); + + expect(response.status).toBe(400); + expect((await readAsAdmin()).user.showHiddenFiles).toBe(true); + }); + + /** And a save with nothing wrong in it still writes every section it carries. */ + it('still writes every section when none of them is refused', async () => { + await seed(); + + const response = await patch(['admin'], { + thumbnails: { size: 654 }, + branding: { appName: 'After' }, + access: { rules: [{ path: 'Private', permissions: 'hidden', recursive: true }] }, + }); + + expect(response.status).toBe(200); + const settings = await readAsAdmin(); + expect(settings.thumbnails.size).toBe(654); + expect(settings.branding.appName).toBe('After'); + expect(settings.access.rules.map((rule) => rule.path)).toEqual(['Private']); + }); +}); + +/** + * A rule that names no folder, and an exclusion list stored as it came. + * + * Both are the same kind of defect: something an administrator saved, that the + * page then showed back to them, doing nothing. A path of nothing but spaces + * normalises to itself, so it was stored and matched no folder. The search + * index had no sanitiser on its way into storage, so its list kept whatever + * spacing and repetition it arrived with — the worker was handed a clean copy + * and behaved, which is exactly why nobody noticed the stored one. + */ +describe('what a rule and an exclusion list are held to', () => { + it.each([[' '], ['\t'], [''], [' \n ']])( + 'refuses an access rule whose path is %j', + async (blank) => { + await seed(); + + const response = await patch(['admin'], { + access: { rules: [{ path: blank, permissions: 'hidden' }] }, + }); + + expect(response.status).toBe(400); + expect(response.body.error.message).toMatch(/Access rule 1.*folder/); + expect((await readAsAdmin()).access.rules).toEqual([]); + } + ); + + it('keeps a folder whose name has spaces in it', async () => { + await seed(); + + const response = await patch(['admin'], { + access: { rules: [{ path: 'My Documents/Q1 2026', permissions: 'ro' }] }, + }); + + expect(response.status).toBe(200); + expect((await readAsAdmin()).access.rules.map((rule) => rule.path)).toEqual([ + 'My Documents/Q1 2026', + ]); + }); + + it('stores the search index exclusions as the worker is given them', async () => { + const db = await seed(); + + const response = await patch(['admin'], { + searchIndex: { excludedPaths: [' Private ', 'Private', '', '/Cache/'] }, + }); + + expect(response.status).toBe(200); + const stored = JSON.parse( + db + .prepare( + "SELECT value FROM system_settings WHERE category = 'system' AND key = 'searchIndex'" + ) + .get().value + ); + + // Trimmed, emptied of nothing, and each folder once — the list the worker + // is handed, rather than what the request happened to carry. + expect(stored.excludedPaths).not.toContain(' Private '); + expect(stored.excludedPaths).not.toContain(''); + expect(stored.excludedPaths.filter((entry) => entry === 'Private')).toHaveLength(1); + expect(stored.excludedPaths).toEqual((await readAsAdmin()).searchIndex.excludedPaths); + }); +}); + +/** + * The access section is saved from two controls: the list of rules, and the one + * switch above them that holds administrators to every rule. + * + * The route used to forward the rules alone. Saving them switched the setting + * back off — silently widening what administrators could reach — and a request + * that carried only the switch stored nothing at all, so turning it on did + * nothing whatever the page showed. The service was right either way, which is + * why only a test that goes through the route catches it. + */ +describe('the access section, saved half at a time', () => { + const RULE = { path: 'Team', recursive: true, permissions: 'ro', appliesToAdmins: true }; + + const storeBoth = () => + patch(['admin'], { access: { rules: [RULE], applyToAdmins: true } }).expect(200); + + it('keeps the switch when only the rules are sent', async () => { + await seed(); + await storeBoth(); + + await patch(['admin'], { + access: { rules: [{ ...RULE, path: 'Finance' }] }, + }).expect(200); + + const { access } = await readAsAdmin(); + expect(access.applyToAdmins).toBe(true); + expect(access.rules.map((rule) => rule.path)).toEqual(['Finance']); + }); + + it('keeps the rules when only the switch is sent', async () => { + await seed(); + await storeBoth(); + + await patch(['admin'], { access: { applyToAdmins: false } }).expect(200); + + const { access } = await readAsAdmin(); + expect(access.applyToAdmins).toBe(false); + expect(access.rules.map((rule) => rule.path)).toEqual(['Team']); + }); + + it('stores what each rule says about administrators', async () => { + await seed(); + + await patch(['admin'], { + access: { + rules: [ + { path: 'Team', recursive: true, permissions: 'ro', appliesToAdmins: true }, + { path: 'Vault', recursive: true, permissions: 'hidden', appliesToAdmins: false }, + ], + }, + }).expect(200); + + const { access } = await readAsAdmin(); + expect(access.rules.map((rule) => rule.appliesToAdmins)).toEqual([true, false]); + }); + + it('refuses a switch that is not a yes or a no, and stores nothing', async () => { + await seed(); + await storeBoth(); + + await patch(['admin'], { access: { applyToAdmins: 'yes' } }).expect(400); + + const { access } = await readAsAdmin(); + expect(access.applyToAdmins).toBe(true); + }); +}); diff --git a/backend/tests/routes/settings-preferences.test.js b/backend/tests/routes/settings-preferences.test.js index 26c503722..ac95cee92 100644 --- a/backend/tests/routes/settings-preferences.test.js +++ b/backend/tests/routes/settings-preferences.test.js @@ -66,26 +66,34 @@ describe('a preference the screen offers', () => { /** * Every key the service knows how to sanitise is a key this route accepts: * one list, so neither can gain a preference the other drops. + * + * A value each preference actually takes, and the value is what is asserted + * rather than the key being present: the answer carries the settings as they + * now stand, so a key stored by an earlier turn of this loop would still be + * there after the one that dropped it. */ + const A_VALUE_IT_TAKES = { + defaultShareExpiration: null, + skipHome: null, + locale: 'fr', + defaultView: 'list', + }; + it('accepts exactly what the settings service calls a preference', async () => { - const { USER_SETTING_KEYS } = load('src/services/settingsService'); - - for (const key of USER_SETTING_KEYS) { - const value = - key === 'defaultShareExpiration' || key === 'skipHome' - ? null - : key === 'locale' - ? 'fr' - : true; + const { WRITABLE_USER_SETTINGS } = load('src/services/settingsService'); + + for (const key of WRITABLE_USER_SETTINGS) { + const value = key in A_VALUE_IT_TAKES ? A_VALUE_IT_TAKES[key] : true; const response = await save({ [key]: value }); - expect(response.body.user, `${key} was dropped`).toHaveProperty(key); + expect(response.body.user?.[key], `${key} was dropped`).toEqual(value); + expect((await stored())[key], `${key} was not stored`).toEqual(value); } }); it('ignores a key that is not a preference', async () => { const response = await save({ isAdmin: true }); - expect(response.body.user ?? {}).toEqual({}); + expect(response.body.user).not.toHaveProperty('isAdmin'); expect((await stored()).isAdmin).toBeUndefined(); }); }); diff --git a/backend/tests/routes/settings-user-preferences.test.js b/backend/tests/routes/settings-user-preferences.test.js new file mode 100644 index 000000000..2c196b9d3 --- /dev/null +++ b/backend/tests/routes/settings-user-preferences.test.js @@ -0,0 +1,323 @@ +import { describe, it, expect } from 'vitest'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +const MODULES = [ + 'src/services/settingsService', + 'src/services/db', + 'src/routes/settings', + 'src/middleware/errorHandler', +]; + +/** + * The route, not the service underneath it. + * + * A preference used to have to be listed in two places — sanitised in the + * service and allowed in the route — and a key present in one but not the other + * was accepted by the API, silently dropped, and answered with its previous + * value. The client applied that answer, so the switch flicked itself back off. + * Testing setUserSetting directly could not see it: the route was the half that + * was missing. + */ +const buildContext = async () => { + const envContext = await setupTestEnv({ tag: 'settings-route-test-', modules: MODULES }); + const settingsService = envContext.requireFresh('src/services/settingsService'); + const settingsRoutes = envContext.requireFresh('src/routes/settings'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + + const dbService = envContext.requireFresh('src/services/db'); + const db = await dbService.getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ).run('user-1', 'user-1@example.com', 1, 'user-1', 'User 1', '["user"]', now, now); + + const express = require('express'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: 'user-1', email: 'user-1@example.com', roles: ['user'] }; + next(); + }); + app.use('/api', settingsRoutes); + app.use(errorHandler); + + return { envContext, app, settingsService }; +}; + +// A value that is different from every default, so "it came back" cannot be +// confused with "it was already like that". +const NON_DEFAULT = { + showHiddenFiles: true, + showThumbnails: false, + showSidebarFavorites: false, + showSidebarShares: false, + showSidebarTools: false, + markdownOpensInEditor: true, + documentsOpenInNewTab: true, + // On by default, so off is the value that has to survive a round trip. + showVersionMarks: false, + defaultShareExpiration: { value: 3, unit: 'days' }, + skipHome: true, + defaultView: 'list', + // Null by default, which means "follow the browser". + locale: 'nl', +}; + +describe('PATCH /api/settings — user preferences', () => { + it('saves the markdown preference and reads it back', async () => { + const { envContext, app } = await buildContext(); + try { + const saved = await request(app) + .patch('/api/settings') + .send({ user: { markdownOpensInEditor: true } }) + .expect(200); + + // The response is what the client applies to its own state, so the value + // has to be in it — not merely stored somewhere. + expect(saved.body.user?.markdownOpensInEditor).toBe(true); + + const reread = await request(app).get('/api/settings').expect(200); + expect(reread.body.user.markdownOpensInEditor).toBe(true); + } finally { + await envContext.cleanup(); + } + }); + + // Every writable preference, so the next one added is covered without anyone + // having to remember to write a test for it. + it('saves and reads back every writable preference', async () => { + const { envContext, app, settingsService } = await buildContext(); + try { + const writable = [...settingsService.WRITABLE_USER_SETTINGS]; + + // Guard against the list and this test drifting apart. + for (const key of writable) { + expect(NON_DEFAULT, `add ${key} to NON_DEFAULT`).toHaveProperty(key); + } + + const payload = Object.fromEntries(writable.map((key) => [key, NON_DEFAULT[key]])); + const saved = await request(app).patch('/api/settings').send({ user: payload }).expect(200); + + for (const key of writable) { + expect(saved.body.user?.[key], `${key} missing from the response`).toEqual( + NON_DEFAULT[key] + ); + } + + const reread = await request(app).get('/api/settings').expect(200); + for (const key of writable) { + expect(reread.body.user[key], `${key} was not persisted`).toEqual(NON_DEFAULT[key]); + } + } finally { + await envContext.cleanup(); + } + }); + + /** + * A default expiry that is not one used to be stored as no default: minus + * three weeks sent from the page removed the default the person had, and the + * page then showed an empty field. + */ + it.each([ + [{ value: -3, unit: 'weeks' }], + [{ value: 0, unit: 'days' }], + [{ value: 3, unit: 'years' }], + [5], + ['soon'], + ])('leaves the default share expiry as it was when sent %j', async (sent) => { + const { envContext, app } = await buildContext(); + try { + await request(app) + .patch('/api/settings') + .send({ user: { defaultShareExpiration: { value: 3, unit: 'days' } } }) + .expect(200); + + const saved = await request(app) + .patch('/api/settings') + .send({ user: { defaultShareExpiration: sent } }) + .expect(200); + + expect(saved.body.user.defaultShareExpiration).toEqual({ value: 3, unit: 'days' }); + const reread = await request(app).get('/api/settings').expect(200); + expect(reread.body.user.defaultShareExpiration).toEqual({ value: 3, unit: 'days' }); + } finally { + await envContext.cleanup(); + } + }); + + it('removes the default share expiry when sent null', async () => { + const { envContext, app } = await buildContext(); + try { + await request(app) + .patch('/api/settings') + .send({ user: { defaultShareExpiration: { value: 3, unit: 'days' } } }) + .expect(200); + + await request(app) + .patch('/api/settings') + .send({ user: { defaultShareExpiration: null } }) + .expect(200); + + const reread = await request(app).get('/api/settings').expect(200); + expect(reread.body.user.defaultShareExpiration).toBeNull(); + } finally { + await envContext.cleanup(); + } + }); + + /** + * A switch used to be `Boolean(whatever came)`, which has an opinion about + * everything: `'false'` — what a form field, a query string or a shell + * client sends — was true, and `0` was false. Either way the preference was + * set to something nobody had chosen, and answered as though they had. + * + * Each case stores the opposite of what the coercion would have made of the + * value, so "it stayed" cannot be confused with "it was already like that". + */ + it.each([ + ['showHiddenFiles', 'false', false], + ['showThumbnails', 0, true], + ['showSidebarFavorites', 'no', false], + ['markdownOpensInEditor', '', true], + ['skipHome', 0, true], + ])('leaves %s as it was when sent %j', async (key, sent, stored) => { + const { envContext, app } = await buildContext(); + try { + await request(app) + .patch('/api/settings') + .send({ user: { [key]: stored } }) + .expect(200); + + const saved = await request(app) + .patch('/api/settings') + .send({ user: { [key]: sent } }) + .expect(200); + + expect(saved.body.user[key]).toBe(stored); + const reread = await request(app).get('/api/settings').expect(200); + expect(reread.body.user[key]).toBe(stored); + } finally { + await envContext.cleanup(); + } + }); + + /** + * A view mode we do not have used to become null, and null is a value here: + * the built-in default. One unknown word therefore put every folder back to + * the built-in view rather than being refused. + */ + it('leaves the default view as it was when sent a mode there is no such thing as', async () => { + const { envContext, app } = await buildContext(); + try { + await request(app) + .patch('/api/settings') + .send({ user: { defaultView: 'list' } }) + .expect(200); + + const saved = await request(app) + .patch('/api/settings') + .send({ user: { defaultView: 'mosaic' } }) + .expect(200); + + expect(saved.body.user.defaultView).toBe('list'); + const reread = await request(app).get('/api/settings').expect(200); + expect(reread.body.user.defaultView).toBe('list'); + } finally { + await envContext.cleanup(); + } + }); + + it('still takes null for the default view, which is the built-in one', async () => { + const { envContext, app } = await buildContext(); + try { + await request(app) + .patch('/api/settings') + .send({ user: { defaultView: 'list' } }) + .expect(200); + + await request(app) + .patch('/api/settings') + .send({ user: { defaultView: null } }) + .expect(200); + + const reread = await request(app).get('/api/settings').expect(200); + expect(reread.body.user.defaultView).toBeNull(); + } finally { + await envContext.cleanup(); + } + }); + + /** + * The language an account reads in, which is the account's and not the + * browser's: the only other way to choose one is the picker on the sign-in + * page, which writes into the browser and is never seen again once somebody + * is signed in (nxzai/NextExplorer discussion #408). + */ + describe('the language', () => { + it('follows the browser until an account says otherwise', async () => { + const { envContext, app } = await buildContext(); + try { + const fresh = await request(app).get('/api/settings').expect(200); + expect(fresh.body.user.locale ?? null).toBeNull(); + + const saved = await request(app) + .patch('/api/settings') + .send({ user: { locale: 'pt-BR' } }) + .expect(200); + expect(saved.body.user.locale).toBe('pt-BR'); + + const back = await request(app) + .patch('/api/settings') + .send({ user: { locale: null } }) + .expect(200); + expect(back.body.user.locale).toBeNull(); + } finally { + await envContext.cleanup(); + } + }); + + /** + * Refused rather than read as "follow the browser": a value that is not a + * language tag is a mistake, and turning it into the default would put the + * choice back where it was with nothing to show for it. + */ + it.each([['not a language'], ['en_US!'], [42], [{ code: 'fr' }], [['fr']]])( + 'leaves the language as it was when sent %j', + async (sent) => { + const { envContext, app } = await buildContext(); + try { + await request(app) + .patch('/api/settings') + .send({ user: { locale: 'nl' } }) + .expect(200); + + const saved = await request(app) + .patch('/api/settings') + .send({ user: { locale: sent } }) + .expect(200); + + expect(saved.body.user.locale).toBe('nl'); + } finally { + await envContext.cleanup(); + } + } + ); + }); + + it('ignores a key that is not a user preference', async () => { + const { envContext, app } = await buildContext(); + try { + await request(app) + .patch('/api/settings') + .send({ user: { notASetting: 'x' } }) + .expect(200); + + const reread = await request(app).get('/api/settings').expect(200); + expect(reread.body.user.notASetting).toBeUndefined(); + } finally { + await envContext.cleanup(); + } + }); +}); diff --git a/backend/tests/routes/settings-write-boundary.test.js b/backend/tests/routes/settings-write-boundary.test.js new file mode 100644 index 000000000..7fb754c41 --- /dev/null +++ b/backend/tests/routes/settings-write-boundary.test.js @@ -0,0 +1,189 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Who may change what, on the one endpoint that writes every setting. + * + * `PATCH /api/settings` takes a single payload with a section per group and + * decides section by section: anyone signed in may change their own + * preferences, only an administrator may change the ones that affect everybody. + * Fifty-five paths through one function, and only the read side of that + * boundary had a test — a regular account being refused the *write* did not. + * + * The response is the whole settings document rather than a list of changes, + * so what is asserted is the value before and after, not the shape of a reply. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async (env = {}) => { + currentEnv = await setupTestEnv({ tag: 'settings-write-', env }); + const dbService = currentEnv.requireFresh('src/services/db'); + const db = await dbService.getDb(); + // A preference is stored against an account that has to exist; without the + // row the write fails on a foreign key and the route answers 500. + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1','u@example.com',1,'u','U','["user"]', ?, ?)` + ).run(now, now); +}; + +const buildApp = (roles) => { + const routes = currentEnv.requireFresh('src/routes/settings'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: 'u1', email: 'u@example.com', roles }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +const patch = (roles, payload) => request(buildApp(roles)).patch('/api/settings').send(payload); +const read = (roles) => request(buildApp(roles)).get('/api/settings'); + +/** + * One writable field per section that only an administrator may touch, with a + * value that differs from the default, so a change is visible either way. + */ +const SYSTEM_CHANGES = [ + ['thumbnails', { size: 321 }, (settings) => settings.thumbnails?.size], + ['uploads', { chunkedEnabled: true }, (settings) => settings.uploads?.chunkedEnabled], + ['branding', { appName: 'Renamed' }, (settings) => settings.branding?.appName], + [ + 'folderSize', + { excludedPaths: ['Sneaked/in'] }, + (settings) => settings.folderSize?.excludedPaths?.join(), + ], + [ + 'searchIndex', + { excludedPaths: ['Sneaked/in'] }, + (settings) => settings.searchIndex?.excludedPaths?.join(), + ], +]; + +describe('what only an administrator may change', () => { + /** + * Refused outright rather than quietly dropped. Answering 200 to a change + * that was not made is worse than saying no: the page that asked has no way + * to tell, and shows the value the person typed. + */ + it.each(SYSTEM_CHANGES)( + 'is refused, and unchanged, when a regular account asks: %s', + async (section, value, readBack) => { + await seed(); + const before = readBack((await read(['admin'])).body); + + const response = await patch(['user'], { [section]: value }); + + expect(response.status).toBe(403); + expect(readBack((await read(['admin'])).body)).toEqual(before); + } + ); + + it.each(SYSTEM_CHANGES)( + 'is applied when an administrator asks: %s', + async (section, value, readBack) => { + await seed(); + const before = readBack((await read(['admin'])).body); + + await patch(['admin'], { [section]: value }); + + const after = readBack((await read(['admin'])).body); + expect(after).not.toEqual(before); + } + ); + + /** + * A payload that mixes the two is refused whole, and the preference in it is + * not kept either. + * + * It used to be: the user section was applied first and the refusal raised + * afterwards, so this answered 403 with the preference already saved. A + * request reported as refused that changed something is the one answer a + * caller cannot act on. + */ + it('refuses a payload that mixes its own preference with a system one', async () => { + await seed(); + + const response = await patch(['user'], { + branding: { appName: 'Taken over' }, + user: { markdownOpensInEditor: true }, + }); + + expect(response.status).toBe(403); + expect((await read(['user'])).body.user?.markdownOpensInEditor).not.toBe(true); + expect((await read(['admin'])).body.branding?.appName).not.toBe('Taken over'); + }); + + it('takes a preference on its own from a regular account', async () => { + await seed(); + + const response = await patch(['user'], { user: { markdownOpensInEditor: true } }); + + expect(response.status).toBe(200); + expect(response.body.user?.markdownOpensInEditor).toBe(true); + }); +}); + +describe('what a value has to look like to be stored', () => { + it('takes a boolean from anything truthy, since a checkbox may send either', async () => { + await seed(); + + const response = await patch(['admin'], { thumbnails: { enabled: 'yes' } }); + + expect(response.body.thumbnails.enabled).toBe(true); + }); + + /** + * A size that is not a number is a size nobody chose. Storing it would put + * something that is not a pixel count where one belongs, and every thumbnail + * generated afterwards would carry it. + * + * Refused twice — once by the route and once by the service that stores it — + * so neither mutation alone fails this. Removing both does. Said out loud + * because a single surviving mutation reads like a gap and is not one. + */ + it.each([['not-a-number'], [null], [Infinity]])( + 'keeps the size it had when given %s', + async (size) => { + await seed(); + const before = (await read(['admin'])).body.thumbnails.size; + + await patch(['admin'], { thumbnails: { size } }); + + expect((await read(['admin'])).body.thumbnails.size).toBe(before); + } + ); + + it('takes a size that is a number', async () => { + await seed(); + + await patch(['admin'], { thumbnails: { size: 256 } }); + + expect((await read(['admin'])).body.thumbnails.size).toBe(256); + }); + + it('ignores a section that is not an object', async () => { + await seed(); + const before = (await read(['admin'])).body.thumbnails; + + const response = await patch(['admin'], { thumbnails: 'enabled please' }); + + expect(response.status).toBe(200); + expect((await read(['admin'])).body.thumbnails).toEqual(before); + }); +}); diff --git a/backend/tests/routes/share-counters.test.js b/backend/tests/routes/share-counters.test.js index ddfc9ac7b..78cc9e1fd 100644 --- a/backend/tests/routes/share-counters.test.js +++ b/backend/tests/routes/share-counters.test.js @@ -98,7 +98,15 @@ describe('a share link', () => { expect(await numbers(id)).toEqual({ opened: 2, downloaded: 0 }); }); - it('counts a file leaving as a download', async () => { + /** + * Counted the way the file is delivered, not by which route asked for it. + * + * Every fetch of `/file/...` used to be a download, so reading a text file in + * the browser — which never leaves the page — was written down as a copy + * taken away. An owner reading "downloaded 40 times" was reading the number + * of times somebody had looked at it. + */ + it('counts a file shown in the page as an opening', async () => { const { token, id } = await seedShare(); const app = buildApp(); @@ -106,6 +114,19 @@ describe('a share link', () => { const file = await request(app).get(`/api/share/${token}/file/file.txt`); expect(file.status).toBe(200); + expect(file.headers['content-disposition'] || '').not.toMatch(/attachment/); + expect(await numbers(id)).toEqual({ opened: 2, downloaded: 0 }); + }); + + it('counts a file handed over as a download', async () => { + const { token, id } = await seedShare(); + const app = buildApp(); + + await request(app).get(`/api/share/${token}/access`); + const file = await request(app).get(`/api/share/${token}/file/file.txt?mode=download`); + + expect(file.status).toBe(200); + expect(file.headers['content-disposition'] || '').toMatch(/attachment/); expect(await numbers(id)).toEqual({ opened: 1, downloaded: 1 }); }); diff --git a/backend/tests/routes/share-door.test.js b/backend/tests/routes/share-door.test.js new file mode 100644 index 000000000..ca1b6ea96 --- /dev/null +++ b/backend/tests/routes/share-door.test.js @@ -0,0 +1,615 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; + +import { setupTestEnv, clearModuleCache } from '../helpers/env-test-utils.js'; + +/** + * The three routes a shared link is opened through. + * + * `/info` is answered to anyone holding a token, before any password has been + * typed — so what it does *not* say matters as much as what it does: where the + * file lives on the server, and who the share was named to, are not a visitor's + * to learn from a link they may not even be able to open. + * + * `/verify` and `/access` are the door itself. Between them they decide who + * gets a guest session, who is sent to sign in, and who is turned away — and + * the rules are not symmetrical: a password protects a link from everyone but + * its owner, being signed in is not the same as knowing it, and a share named + * to people is not opened by a password at all. + */ + +let envContext; + +beforeAll(async () => { + envContext = await setupTestEnv({ + tag: 'share-door-test-', + env: { USER_VOLUMES: 'true' }, + modules: [ + 'src/services/db', + 'src/services/users', + 'src/services/userVolumesService', + 'src/services/sharesService', + 'src/services/guestSessionService', + 'src/utils/pathUtils', + 'src/middleware/authMiddleware', + 'src/middleware/errorHandler', + 'src/routes/shares', + ], + }); +}); + +afterAll(async () => { + await envContext.cleanup(); +}); + +const buildApp = ({ user } = {}) => { + clearModuleCache('src/config/env'); + clearModuleCache('src/config/index'); + + const sharesRoutes = envContext.requireFresh('src/routes/shares'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const authMiddleware = envContext.requireFresh('src/middleware/authMiddleware'); + + const app = express(); + app.use(express.json()); + app.use(cookieParser()); + app.use((req, _res, next) => { + req.session = user ? { localUserId: user.id } : {}; + next(); + }); + app.use(authMiddleware); + app.use('/api/shares', sharesRoutes); + app.use('/api/share', sharesRoutes); + app.use(errorHandler); + return app; +}; + +let seq = 0; + +/** An owner with a volume of their own, and something in it. */ +const makeOwner = async (files = { 'hello.txt': 'bonjour' }) => { + seq += 1; + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + + const root = path.join(envContext.tmpRoot, `door-volume-${seq}`); + await fs.mkdir(root, { recursive: true }); + for (const [name, contents] of Object.entries(files)) { + await fs.mkdir(path.dirname(path.join(root, name)), { recursive: true }); + await fs.writeFile(path.join(root, name), contents); + } + + const user = await usersService.createLocalUser({ + email: `door-${seq}@example.com`, + username: `door-${seq}`, + displayName: `Door ${seq}`, + password: 'secret123', + roles: ['user'], + }); + const label = `DoorVol${seq}`; + await userVolumesService.addVolumeToUser({ + userId: user.id, + label, + volumePath: root, + accessMode: 'readwrite', + }); + + return { user, label, root }; +}; + +const createShare = async (user, body) => { + const response = await request(buildApp({ user })).post('/api/shares').send(body); + expect(response.status).toBe(201); + return response.body; +}; + +const visitor = () => buildApp(); + +describe('what a link tells someone holding it', () => { + it('names the share and says what it is', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + label: 'Le mot de passe du wifi', + }); + + const info = await request(visitor()).get(`/api/share/${share.shareToken}/info`); + + expect(info.status).toBe(200); + expect(info.body).toMatchObject({ + shareToken: share.shareToken, + label: 'Le mot de passe du wifi', + isDirectory: false, + hasPassword: false, + sharingType: 'anyone', + isExpired: false, + }); + }); + + /** + * A token is not a permission. Anyone who has one — from a forwarded message, + * a browser history, a proxy log — can call this before typing a password, so + * it must not describe the server's filesystem or name the people the share + * was made for. + */ + it('says nothing about where the file lives or who it was made for', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + }); + + const info = await request(visitor()).get(`/api/share/${share.shareToken}/info`); + + expect(Object.keys(info.body).sort()).toEqual( + [ + 'expiresAt', + 'hasPassword', + 'isDirectory', + 'isExpired', + 'label', + 'requiresPassword', + 'sharingType', + 'shareToken', + ].sort() + ); + expect(JSON.stringify(info.body)).not.toContain(label); + }); + + it('says a protected link wants a password', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + password: 'ouvre-toi', + }); + + const info = await request(visitor()).get(`/api/share/${share.shareToken}/info`); + + expect(info.body).toMatchObject({ hasPassword: true, requiresPassword: true }); + }); + + /** + * The owner of a protected link is not sent to a prompt the API would let + * them skip — the router reads this field, so the two have to agree. + */ + it('does not ask its own owner for the password', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + password: 'ouvre-toi', + }); + + const info = await request(buildApp({ user })).get(`/api/share/${share.shareToken}/info`); + + expect(info.body).toMatchObject({ hasPassword: true, requiresPassword: false }); + }); + + it('says an expired link has expired', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }); + const sharesService = envContext.requireFresh('src/services/sharesService'); + const dbService = envContext.requireFresh('src/services/db'); + const db = await dbService.getDb(); + db.prepare('UPDATE shares SET expires_at = ? WHERE share_token = ?').run( + new Date(Date.now() - 60_000).toISOString(), + share.shareToken + ); + expect(sharesService).toBeTruthy(); + + const info = await request(visitor()).get(`/api/share/${share.shareToken}/info`); + + expect(info.body.isExpired).toBe(true); + }); + + it('answers a token that was never a share with a plain not-found', async () => { + const info = await request(visitor()).get('/api/share/pas-un-jeton/info'); + + expect(info.status).toBe(404); + }); +}); + +describe('typing the password on a link', () => { + const lockedShare = async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + password: 'ouvre-toi', + }); + return { user, share }; + }; + + it('opens it, and hands back a session for this share', async () => { + const { share } = await lockedShare(); + + const verified = await request(visitor()) + .post(`/api/share/${share.shareToken}/verify`) + .send({ password: 'ouvre-toi' }); + + expect(verified.status).toBe(200); + expect(verified.body.success).toBe(true); + expect(verified.body.guestSessionId).toBeTruthy(); + }); + + it('sets the session as a cookie, so a reload keeps it', async () => { + const { share } = await lockedShare(); + + const verified = await request(visitor()) + .post(`/api/share/${share.shareToken}/verify`) + .send({ password: 'ouvre-toi' }); + + expect(String(verified.headers['set-cookie'])).toContain('guest'); + }); + + it('refuses the wrong one, and hands back nothing', async () => { + const { share } = await lockedShare(); + + const refused = await request(visitor()) + .post(`/api/share/${share.shareToken}/verify`) + .send({ password: 'au-hasard' }); + + expect(refused.status).toBe(401); + expect(refused.body.guestSessionId).toBeUndefined(); + }); + + it('refuses an empty one', async () => { + const { share } = await lockedShare(); + + const refused = await request(visitor()).post(`/api/share/${share.shareToken}/verify`).send({}); + + expect(refused.status).toBe(401); + }); + + /** A link that has run out is not opened by the right password either. */ + it('refuses an expired link whatever is typed', async () => { + const { share } = await lockedShare(); + const db = await envContext.requireFresh('src/services/db').getDb(); + db.prepare('UPDATE shares SET expires_at = ? WHERE share_token = ?').run( + new Date(Date.now() - 60_000).toISOString(), + share.shareToken + ); + + const refused = await request(visitor()) + .post(`/api/share/${share.shareToken}/verify`) + .send({ password: 'ouvre-toi' }); + + expect(refused.status).toBe(403); + }); + + /** + * A share named to people is not opened by knowing something. The visitor is + * told to sign in rather than handed a session. + */ + it('sends a named share to sign in rather than opening it', async () => { + const { user, label } = await makeOwner(); + const other = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'users', + userIds: [other.user.id], + password: 'ouvre-toi', + }); + + const verified = await request(visitor()) + .post(`/api/share/${share.shareToken}/verify`) + .send({ password: 'ouvre-toi' }); + + expect(verified.status).toBe(200); + expect(verified.body).toEqual({ success: true, requiresAuth: true }); + expect(verified.body.guestSessionId).toBeUndefined(); + }); + + it('opens a link with no password at all, for anyone', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + }); + + const verified = await request(visitor()) + .post(`/api/share/${share.shareToken}/verify`) + .send({}); + + expect(verified.status).toBe(200); + expect(verified.body.guestSessionId).toBeTruthy(); + }); + + it('still requires signing in for a named share with no password', async () => { + const { user, label } = await makeOwner(); + const other = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'users', + userIds: [other.user.id], + }); + + const refused = await request(visitor()).post(`/api/share/${share.shareToken}/verify`).send({}); + + expect(refused.status).toBe(401); + }); + + it('answers a token that was never a share with a plain not-found', async () => { + const refused = await request(visitor()) + .post('/api/share/pas-un-jeton/verify') + .send({ password: 'x' }); + + expect(refused.status).toBe(404); + }); +}); + +describe('opening a link', () => { + it('gives a visitor a session and describes what they may do', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + }); + + const opened = await request(visitor()).get(`/api/share/${share.shareToken}/access`); + + expect(opened.status).toBe(200); + expect(opened.body.guestSessionId).toBeTruthy(); + expect(opened.body.share).toMatchObject({ + shareToken: share.shareToken, + sourcePath: `share/${share.shareToken}`, + accessMode: 'readonly', + allowDownload: true, + isDirectory: false, + }); + }); + + /** Being signed in is not knowing the password. */ + it('refuses a protected link to a visitor who has not typed the password', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + password: 'ouvre-toi', + }); + + const refused = await request(visitor()).get(`/api/share/${share.shareToken}/access`); + + expect(refused.status).toBe(401); + }); + + it('refuses it to a signed-in stranger too', async () => { + const { user, label } = await makeOwner(); + const stranger = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + password: 'ouvre-toi', + }); + + const refused = await request(buildApp({ user: stranger.user })).get( + `/api/share/${share.shareToken}/access` + ); + + expect(refused.status).toBe(401); + }); + + it('opens it for its owner without a password', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + password: 'ouvre-toi', + }); + + const opened = await request(buildApp({ user })).get(`/api/share/${share.shareToken}/access`); + + expect(opened.status).toBe(200); + }); + + /** + * Reloading the page calls here again. A visitor who has just typed the + * password holds a session that says so, and asking for it a second time — + * for a share they were just given — is how this went wrong before. + */ + it('accepts the session a visitor was just handed', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + password: 'ouvre-toi', + }); + const app = visitor(); + const agent = request.agent(app); + const verified = await agent + .post(`/api/share/${share.shareToken}/verify`) + .send({ password: 'ouvre-toi' }); + expect(verified.status).toBe(200); + + const opened = await agent.get(`/api/share/${share.shareToken}/access`); + + expect(opened.status).toBe(200); + expect(opened.body.guestSessionId).toBe(verified.body.guestSessionId); + }); + + it('refuses a named share to somebody not on it', async () => { + const { user, label } = await makeOwner(); + const invited = await makeOwner(); + const stranger = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'users', + userIds: [invited.user.id], + }); + + const refused = await request(buildApp({ user: stranger.user })).get( + `/api/share/${share.shareToken}/access` + ); + + expect(refused.status).toBe(403); + }); + + it('opens it for somebody who is on it', async () => { + const { user, label } = await makeOwner(); + const invited = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'users', + userIds: [invited.user.id], + }); + + const opened = await request(buildApp({ user: invited.user })).get( + `/api/share/${share.shareToken}/access` + ); + + expect(opened.status).toBe(200); + }); + + it('asks a signed-out visitor of a named share to sign in', async () => { + const { user, label } = await makeOwner(); + const invited = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'users', + userIds: [invited.user.id], + }); + + const refused = await request(visitor()).get(`/api/share/${share.shareToken}/access`); + + expect(refused.status).toBe(401); + }); + + it('refuses an expired link to everyone, its owner included', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + }); + const db = await envContext.requireFresh('src/services/db').getDb(); + db.prepare('UPDATE shares SET expires_at = ? WHERE share_token = ?').run( + new Date(Date.now() - 60_000).toISOString(), + share.shareToken + ); + + expect((await request(visitor()).get(`/api/share/${share.shareToken}/access`)).status).toBe( + 403 + ); + expect( + (await request(buildApp({ user })).get(`/api/share/${share.shareToken}/access`)).status + ).toBe(403); + }); + + it('answers a token that was never a share with a plain not-found', async () => { + const refused = await request(visitor()).get('/api/share/pas-un-jeton/access'); + + expect(refused.status).toBe(404); + }); +}); + +describe('browsing a share that is one file', () => { + const openFileShare = async (body = {}) => { + const { user, label } = await makeOwner({ 'photo.png': 'pas vraiment une image' }); + const share = await createShare(user, { + sourcePath: `${label}/photo.png`, + accessMode: 'readonly', + sharingType: 'anyone', + ...body, + }); + const agent = request.agent(visitor()); + await agent.get(`/api/share/${share.shareToken}/access`); + return { agent, share, user }; + }; + + it('answers with the one file, and says it is not a folder', async () => { + const { agent, share } = await openFileShare(); + + const listing = await agent.get(`/api/share/${share.shareToken}/browse/`); + + expect(listing.status).toBe(200); + expect(listing.body.items).toHaveLength(1); + expect(listing.body.items[0].name).toBe('photo.png'); + expect(listing.body.current.isDirectory).toBe(false); + }); + + /** + * A file share is a file, so the things a folder allows are refused whatever + * the share's own access mode says: there is nowhere to upload to, nothing to + * create, and a link is not a right to hand out more links. + */ + it('offers none of the things a folder would', async () => { + const { agent, share } = await openFileShare({ accessMode: 'readwrite' }); + + const listing = await agent.get(`/api/share/${share.shareToken}/browse/`); + + expect(listing.body.access).toMatchObject({ + canUpload: false, + canCreateFolder: false, + canCreateFile: false, + canShare: false, + }); + }); + + it('follows the share on whether the file may be downloaded', async () => { + const { agent, share } = await openFileShare({ allowDownload: false }); + + const listing = await agent.get(`/api/share/${share.shareToken}/browse/`); + + expect(listing.body.access.canDownload).toBe(false); + }); + + it('names the folder the file came from, for the breadcrumb', async () => { + const { user, label } = await makeOwner({ 'Rapports/mars.txt': 'bonjour' }); + const share = await createShare(user, { + sourcePath: `${label}/Rapports/mars.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + label: 'Le rapport de mars', + }); + const agent = request.agent(visitor()); + await agent.get(`/api/share/${share.shareToken}/access`); + + const listing = await agent.get(`/api/share/${share.shareToken}/browse/`); + + expect(listing.body.shareInfo).toEqual({ + label: 'Le rapport de mars', + sourceFolderName: 'mars.txt', + }); + }); + + it('is refused to a visitor who has not opened the link', async () => { + const { user, label } = await makeOwner(); + const share = await createShare(user, { + sourcePath: `${label}/hello.txt`, + accessMode: 'readonly', + sharingType: 'anyone', + password: 'ouvre-toi', + }); + + const refused = await request(visitor()).get(`/api/share/${share.shareToken}/browse/`); + + expect(refused.status).toBe(401); + }); +}); diff --git a/backend/tests/routes/share-download-permission.test.js b/backend/tests/routes/share-download-permission.test.js new file mode 100644 index 000000000..4530a1e9e --- /dev/null +++ b/backend/tests/routes/share-download-permission.test.js @@ -0,0 +1,309 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A share that may be read but not taken away. + * + * `canDownload` existed across four call sites and was set to `true` at every + * place that set it — the only `false` was in the denied-access object, where + * `canAccess` had already answered. Removing the check from the download route + * broke no test, because nothing could ever withhold it. It was a promise the + * code did not keep, and the frontend gated a button on it. + * + * `allowDownload` on a share is what makes it mean something: "read this" + * rather than "take a copy of this". It is deliberately not tied to read-write + * like the other granular permissions — a read-only share is exactly where + * withholding a download is the point. + * + * The default is allowed, everywhere, so every share made before this existed + * behaves as it always did. That is the property most worth pinning: a + * permission added to a live system must not silently take something away. + */ + +let ctx; + +const setup = async () => { + ctx = await setupTestEnv({ + tag: 'share-download-', + modules: [ + 'src/config/env', + 'src/config/index', + 'src/services/db', + 'src/services/sharesService', + 'src/services/accessManager', + 'src/routes/shares', + 'src/routes/files/download', + 'src/middleware/errorHandler', + ], + }); + + await fs.mkdir(path.join(ctx.volumeDir, 'Docs'), { recursive: true }); + await fs.writeFile(path.join(ctx.volumeDir, 'Docs', 'report.txt'), 'the contents'); + + const { getDb } = ctx.requireFresh('src/services/db'); + const db = await getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('owner', 'owner@example.com', 1, 'owner', 'Owner', '["admin"]', ?, ?)` + ).run(now, now); + + const shares = ctx.requireFresh('src/services/sharesService'); + const routes = ctx.requireFresh('src/routes/shares'); + const { errorHandler } = ctx.requireFresh('src/middleware/errorHandler'); + + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: 'owner', roles: ['admin'] }; + next(); + }); + // Mounted at both paths, as the application mounts it: `/api/shares` is the + // owner's view of their shares, `/api/share` is what a link resolves to. + app.use('/api/shares', routes); + app.use('/api/share', routes); + app.use(errorHandler); + + // The ordinary download endpoint, which resolves a `share//...` path + // through the same access manager as everything else. + const downloadRoutes = ctx.requireFresh('src/routes/files/download'); + const downloadApp = express(); + downloadApp.use(express.json()); + downloadApp.use((req, _res, next) => { + req.user = { id: 'owner', roles: ['admin'] }; + next(); + }); + downloadApp.use('/api', downloadRoutes); + downloadApp.use(errorHandler); + + return { shares, app, db, downloadApp }; +}; + +afterEach(async () => { + if (ctx) { + await ctx.cleanup(); + ctx = null; + } +}); + +const makeShare = (shares, overrides = {}) => + shares.createShare({ + ownerId: 'owner', + sourceSpace: 'volume', + sourcePath: 'Docs/report.txt', + isDirectory: false, + accessMode: 'readonly', + sharingType: 'anyone', + ...overrides, + }); + +describe('a share that allows downloads', () => { + it('is the default, so nothing that already exists changes', async () => { + const { shares } = await setup(); + + const share = await makeShare(shares); + + expect(share.allowDownload).toBe(true); + }); + + it('serves the file', async () => { + const { shares, app } = await setup(); + const share = await makeShare(shares); + + const response = await request(app).get(`/api/share/${share.shareToken}/file`); + + expect(response.status).toBe(200); + }); + + /** + * A row written before the column existed has it filled in by the migration + * default. Anything else would take downloads away from live share links on + * an upgrade. + */ + it('is what a row from before the column reads as', async () => { + const { shares, db } = await setup(); + const share = await makeShare(shares); + db.prepare('UPDATE shares SET allow_download = 1 WHERE id = ?').run(share.id); + + const reloaded = await shares.getShareById(share.id); + + expect(reloaded.allowDownload).toBe(true); + }); +}); + +describe('a share that withholds them', () => { + it('records the choice', async () => { + const { shares } = await setup(); + + const share = await makeShare(shares, { allowDownload: false }); + + expect(share.allowDownload).toBe(false); + }); + + it('refuses the file', async () => { + const { shares, app } = await setup(); + const share = await makeShare(shares, { allowDownload: false }); + + const response = await request(app).get(`/api/share/${share.shareToken}/file`); + + expect(response.status).toBe(403); + }); + + /** + * The whole point: reading still works. A share nobody can open is not a + * read-only share, it is a broken one. + */ + it('still lets the share be opened', async () => { + const { shares, app } = await setup(); + const share = await makeShare(shares, { allowDownload: false }); + + const response = await request(app).get(`/api/share/${share.shareToken}/access`); + + expect(response.status).toBe(200); + }); + + /** + * Told to the client before it browses anything, so a share view can hide the + * button instead of offering a click whose only outcome is a 403. + */ + it('says so in the payload that opens the share', async () => { + const { shares, app } = await setup(); + const share = await makeShare(shares, { allowDownload: false }); + + const response = await request(app).get(`/api/share/${share.shareToken}/access`); + + expect(response.body?.share?.allowDownload).toBe(false); + }); + + it('says the opposite for one that allows them', async () => { + const { shares, app } = await setup(); + const share = await makeShare(shares); + + const response = await request(app).get(`/api/share/${share.shareToken}/access`); + + expect(response.body?.share?.allowDownload).toBe(true); + }); + + /** + * And per row in the listing, which used to be hard-coded true: every file in + * a share with downloads withheld still showed the button. + */ + it('says so for each file in the listing too', async () => { + const { shares, app } = await setup(); + const folderShare = await shares.createShare({ + ownerId: 'owner', + sourceSpace: 'volume', + sourcePath: 'Docs', + isDirectory: true, + accessMode: 'readonly', + sharingType: 'anyone', + allowDownload: false, + }); + + const response = await request(app).get(`/api/share/${folderShare.shareToken}/browse/`); + + expect(response.status).toBe(200); + const items = response.body?.items || []; + expect(items.length).toBeGreaterThan(0); + expect(items.every((item) => item.access?.canDownload === false)).toBe(true); + }); + + /** + * Not tied to read-write, unlike delete, upload and the create permissions. + * A read-write share where downloads are withheld is coherent — collaborate + * on it, do not take it home — and gating it the way the others are gated + * would make that impossible to express. + */ + it('withholds them on a read-write share too', async () => { + const { shares, app } = await setup(); + const share = await makeShare(shares, { accessMode: 'readwrite', allowDownload: false }); + + const response = await request(app).get(`/api/share/${share.shareToken}/file`); + + expect(response.status).toBe(403); + }); +}); + +describe('the ordinary download route, on a share path', () => { + /** + * A signed-in person can reach a share through the normal explorer, and the + * normal download endpoint resolves `share//...` through the same + * access manager. That endpoint's own `canDownload` check was the one nothing + * could reach — it is reachable now, and this is what reaches it. + */ + it('refuses a file inside a share that withholds downloads', async () => { + const { shares, app, downloadApp } = await setup(); + const share = await shares.createShare({ + ownerId: 'owner', + sourceSpace: 'volume', + sourcePath: 'Docs', + isDirectory: true, + accessMode: 'readonly', + sharingType: 'anyone', + allowDownload: false, + }); + expect(app).toBeTruthy(); + + const response = await request(downloadApp) + .post('/api/download') + .send({ paths: [`share/${share.shareToken}/report.txt`] }); + + expect(response.status).toBe(403); + }); + + it('serves it from a share that allows them', async () => { + const { shares, downloadApp } = await setup(); + const share = await shares.createShare({ + ownerId: 'owner', + sourceSpace: 'volume', + sourcePath: 'Docs', + isDirectory: true, + accessMode: 'readonly', + sharingType: 'anyone', + }); + + const response = await request(downloadApp) + .post('/api/download') + .send({ paths: [`share/${share.shareToken}/report.txt`] }); + + expect(response.status).toBe(200); + }); +}); + +describe('changing it afterwards', () => { + it('can be switched off on an existing share', async () => { + const { shares, app } = await setup(); + const share = await makeShare(shares); + + await request(app).put(`/api/shares/${share.id}`).send({ allowDownload: false }); + const response = await request(app).get(`/api/share/${share.shareToken}/file`); + + expect(response.status).toBe(403); + }); + + it('can be switched back on', async () => { + const { shares, app } = await setup(); + const share = await makeShare(shares, { allowDownload: false }); + + await request(app).put(`/api/shares/${share.id}`).send({ allowDownload: true }); + const response = await request(app).get(`/api/share/${share.shareToken}/file`); + + expect(response.status).toBe(200); + }); + + /** An update that says nothing about it must not reset it. */ + it('is left alone by an update that does not mention it', async () => { + const { shares, app } = await setup(); + const share = await makeShare(shares, { allowDownload: false }); + + await request(app).put(`/api/shares/${share.id}`).send({ label: 'Renamed' }); + const reloaded = await shares.getShareById(share.id); + + expect(reloaded.allowDownload).toBe(false); + }); +}); diff --git a/backend/tests/routes/share-editor-save.test.js b/backend/tests/routes/share-editor-save.test.js index bba4fd87b..f0fc15573 100644 --- a/backend/tests/routes/share-editor-save.test.js +++ b/backend/tests/routes/share-editor-save.test.js @@ -62,8 +62,8 @@ const buildApp = ({ user } = {}) => { return application; }; -/** An "anyone" share of `sourceName` inside the owner's assigned volume. */ -const shareOf = async (sourceName, { accessMode = 'readwrite' } = {}) => { +/** A writable "anyone" share of `sourceName` inside the owner's assigned volume. */ +const shareOf = async (sourceName) => { const owner = await load('src/services/users').createLocalUser({ email: `owner-${sourceName.replace(/\W/g, '-')}@example.com`, username: `owner-${sourceName.replace(/\W/g, '-')}`, @@ -81,7 +81,7 @@ const shareOf = async (sourceName, { accessMode = 'readwrite' } = {}) => { .post('/api/shares') .send({ sourcePath: `Assigned/${sourceName}`, - accessMode, + accessMode: 'readwrite', sharingType: 'anyone', }); expect(create.status).toBe(201); @@ -149,24 +149,4 @@ describe('saving a text file through a share link', () => { expect(response.status).toBe(400); expect((await fsp.stat(path.join(assignedRoot, 'minutes'))).isDirectory()).toBe(true); }); - - /** - * A link that only reads is the case this route exists to keep apart from - * the one that writes: the editor opens on both, and offers to save on one. - */ - it('refuses a link that was not made writable', async () => { - const target = path.join(assignedRoot, 'lecture.txt'); - await fsp.writeFile(target, 'à lire seulement'); - const token = await shareOf('lecture.txt', { accessMode: 'readonly' }); - - // It still opens: reading is what the link is for. - const opened = await request(buildApp()).get(`/api/share/${token}/editor`); - expect(opened.status).toBe(200); - expect(opened.body).toMatchObject({ canWrite: false }); - - const response = await save(token, 'réécrit quand même'); - - expect(response.status).toBe(403); - expect(await fsp.readFile(target, 'utf8')).toBe('à lire seulement'); - }); }); diff --git a/backend/tests/routes/shareLinksUsers.test.js b/backend/tests/routes/shareLinksUsers.test.js index 430edc4b3..a2c6c21d8 100644 --- a/backend/tests/routes/shareLinksUsers.test.js +++ b/backend/tests/routes/shareLinksUsers.test.js @@ -18,6 +18,7 @@ beforeAll(async () => { 'src/middleware/errorHandler', 'src/routes/auth', 'src/routes/shares', + 'src/services/textEditorService', ], }); }); @@ -66,7 +67,12 @@ const buildApp = () => { describe('Share Links for Specific Users', () => { describe('User-Specific Share Access', () => { - it('should allow /api/share/:token/access when logged in as recipient', async () => { + /** + * Slow for the same reason as the local-auth walk-through: bcryptjs is pure + * JavaScript, the share password is hashed, and v8 coverage instrumentation + * multiplies that cost past the five-second default. + */ + it('should restrict the shared editor to the intended recipient', async () => { const usersService = envContext.requireFresh('src/services/users'); const app = buildApp(); @@ -86,6 +92,13 @@ describe('Share Links for Specific Users', () => { password: 'secret123', roles: ['user'], }); + const outsider = await usersService.createLocalUser({ + email: 'outsider@example.com', + username: 'outsider', + displayName: 'Outsider', + password: 'secret123', + roles: ['user'], + }); // Create a folder to share under the volume root. const sharedFolder = path.join(envContext.volumeDir, 'docs'); @@ -122,6 +135,19 @@ describe('Share Links for Specific Users', () => { expect(access.status).toBe(200); expect(access.body?.share?.shareToken).toBe(token); expect(access.body?.share?.sourcePath).toBe(`share/${token}`); - }); + + const editor = await recipientAgent.get(`/api/share/${token}/editor/hello.txt`); + expect(editor.status).toBe(200); + expect(editor.body).toMatchObject({ name: 'hello.txt', content: 'hello' }); + + const outsiderAgent = request.agent(app); + const outsiderLogin = await outsiderAgent + .post('/api/auth/login') + .send({ email: outsider.email, password: 'secret123' }); + expect(outsiderLogin.status).toBe(200); + + const forbidden = await outsiderAgent.get(`/api/share/${token}/editor/hello.txt`); + expect(forbidden.status).toBe(403); + }, 30_000); }); }); diff --git a/backend/tests/routes/shares.test.js b/backend/tests/routes/shares.test.js index 027a81a55..8ca901cc3 100644 --- a/backend/tests/routes/shares.test.js +++ b/backend/tests/routes/shares.test.js @@ -2,6 +2,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import path from 'node:path'; import fs from 'node:fs/promises'; import express from 'express'; +import cookieParser from 'cookie-parser'; import request from 'supertest'; import { setupTestEnv, clearModuleCache } from '../helpers/env-test-utils.js'; @@ -20,9 +21,13 @@ beforeAll(async () => { 'src/services/sharesService', 'src/services/guestSessionService', 'src/utils/pathUtils', + 'src/middleware/authMiddleware', 'src/middleware/errorHandler', 'src/routes/shares', 'src/routes/files/delete', + 'src/routes/files/folder', + 'src/routes/files/file', + 'src/routes/permissions', 'src/services/fileTransferService', ], }); @@ -40,29 +45,253 @@ const buildApp = ({ user } = {}) => { const sharesRoutes = envContext.requireFresh('src/routes/shares'); const deleteRoutes = envContext.requireFresh('src/routes/files/delete'); + const folderRoutes = envContext.requireFresh('src/routes/files/folder'); + const fileRoutes = envContext.requireFresh('src/routes/files/file'); + const permissionsRoutes = envContext.requireFresh('src/routes/permissions'); const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); const app = express(); app.use(express.json()); - - app.use(async (req, _res, next) => { - if (user) req.user = user; - const guestSessionId = req.headers['x-guest-session']; - if (guestSessionId) { - const { getGuestSession } = envContext.requireFresh('src/services/guestSessionService'); - req.guestSession = await getGuestSession(guestSessionId); - } + app.use(cookieParser()); + + // The real middleware, not a stand-in for it. An earlier version of this + // harness attached req.guestSession unconditionally, which the middleware + // does not do — and a bug that lived in exactly that gap shipped green. + const authMiddleware = envContext.requireFresh('src/middleware/authMiddleware'); + app.use((req, _res, next) => { + // express-session normally provides this; the middleware loads the user + // from the database, exactly as it does in production. + req.session = user ? { localUserId: user.id } : {}; next(); }); + app.use(authMiddleware); app.use('/api/shares', sharesRoutes); app.use('/api/share', sharesRoutes); app.use('/api', deleteRoutes); + app.use('/api', folderRoutes); + app.use('/api', fileRoutes); + app.use('/api', permissionsRoutes); app.use(errorHandler); return app; }; describe('Shares Routes', () => { + describe('Share updates', () => { + it('should replace recipient permissions and clear them when changing to an anyone link', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-share-updates'); + await fs.mkdir(assignedRoot, { recursive: true }); + await fs.writeFile(path.join(assignedRoot, 'shared.txt'), 'shared'); + + const owner = await usersService.createLocalUser({ + email: 'share-owner@example.com', + username: 'share-owner', + displayName: 'Share Owner', + password: 'secret123', + roles: ['user'], + }); + const recipient = await usersService.createLocalUser({ + email: 'share-recipient@example.com', + username: 'share-recipient', + displayName: 'Share Recipient', + password: 'secret123', + roles: ['user'], + }); + await userVolumesService.addVolumeToUser({ + userId: owner.id, + label: 'ShareUpdateVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const app = buildApp({ user: owner }); + const created = await request(app) + .post('/api/shares') + .send({ + sourcePath: 'ShareUpdateVol/shared.txt', + accessMode: 'readonly', + sharingType: 'users', + userIds: [recipient.id], + }); + expect(created.status).toBe(201); + expect(created.body.permittedUserIds).toEqual([recipient.id]); + + const updated = await request(app).put(`/api/shares/${created.body.id}`).send({ + accessMode: 'readwrite', + sharingType: 'anyone', + userIds: [], + allowDelete: false, + allowCreateFolder: false, + allowCreateFile: false, + allowUpload: false, + label: 'Updated share', + }); + expect(updated.status).toBe(200); + expect(updated.body).toMatchObject({ + accessMode: 'readwrite', + sharingType: 'anyone', + allowDelete: false, + allowCreateFolder: false, + allowCreateFile: false, + allowUpload: false, + label: 'Updated share', + }); + + const recipientApp = buildApp({ user: recipient }); + const received = await request(recipientApp).get('/api/shares/shared-with-me'); + expect(received.status).toBe(200); + expect(received.body.shares).toEqual([]); + }); + }); + + describe('Shared item permissions', () => { + it('should allow viewing permissions through a share but reject permission changes', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-share-permissions'); + await fs.mkdir(assignedRoot, { recursive: true }); + await fs.writeFile(path.join(assignedRoot, 'shared.txt'), 'shared'); + + const user = await usersService.createLocalUser({ + email: 'share-permissions@example.com', + username: 'share-permissions', + displayName: 'Share Permissions', + password: 'secret123', + roles: ['user'], + }); + await userVolumesService.addVolumeToUser({ + userId: user.id, + label: 'SharePermissionsVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const app = buildApp({ user }); + const created = await request(app).post('/api/shares').send({ + sourcePath: 'SharePermissionsVol/shared.txt', + accessMode: 'readwrite', + sharingType: 'anyone', + }); + expect(created.status).toBe(201); + + const sharedPath = `share/${created.body.shareToken}/shared.txt`; + const view = await request(app).get(`/api/permissions/${sharedPath}`); + expect(view.status).toBe(200); + + const chmod = await request(app) + .post('/api/permissions/chmod') + .send({ path: sharedPath, mode: '644' }); + expect(chmod.status).toBe(403); + + const chown = await request(app) + .post('/api/permissions/chown') + .send({ path: sharedPath, owner: 'root', group: 'root' }); + expect(chown.status).toBe(403); + }); + }); + + describe('Granular write permissions', () => { + it('should apply directory write permissions to share access and mutation routes', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-granular-permissions'); + const sharedFolder = path.join(assignedRoot, 'shared'); + await fs.mkdir(sharedFolder, { recursive: true }); + await fs.writeFile(path.join(sharedFolder, 'existing.txt'), 'existing'); + + const user = await usersService.createLocalUser({ + email: 'permissions@example.com', + username: 'permissions', + displayName: 'Permissions', + password: 'secret123', + roles: ['user'], + }); + + await userVolumesService.addVolumeToUser({ + userId: user.id, + label: 'PermissionsVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const ownerApp = buildApp({ user }); + const create = await request(ownerApp).post('/api/shares').send({ + sourcePath: 'PermissionsVol/shared', + accessMode: 'readwrite', + allowDelete: false, + allowCreateFolder: false, + allowCreateFile: false, + allowUpload: false, + sharingType: 'anyone', + }); + + expect(create.status).toBe(201); + expect(create.body.allowDelete).toBe(false); + expect(create.body.allowCreateFolder).toBe(false); + expect(create.body.allowCreateFile).toBe(false); + expect(create.body.allowUpload).toBe(false); + + const guestApp = buildApp(); + const access = await request(guestApp).get(`/api/share/${create.body.shareToken}/access`); + expect(access.status).toBe(200); + expect(access.body.guestSessionId).toBeTruthy(); + + const sessionHeader = { 'X-Guest-Session': access.body.guestSessionId }; + const browse = await request(guestApp) + .get(`/api/share/${create.body.shareToken}/browse/`) + .set(sessionHeader); + expect(browse.status).toBe(200); + expect(browse.body.access).toMatchObject({ + canWrite: true, + canDelete: false, + canUpload: false, + canCreateFolder: false, + canCreateFile: false, + }); + + const createFolder = await request(guestApp) + .post('/api/files/folder') + .set(sessionHeader) + .send({ path: `share/${create.body.shareToken}`, name: 'blocked-folder' }); + expect(createFolder.status).toBe(403); + + const createFile = await request(guestApp) + .post('/api/files/file') + .set(sessionHeader) + .send({ path: `share/${create.body.shareToken}`, name: 'blocked.txt' }); + expect(createFile.status).toBe(403); + }); + + it('should default granular permissions to the current full read-write behavior', async () => { + const sharesService = envContext.requireFresh('src/services/sharesService'); + const usersService = envContext.requireFresh('src/services/users'); + const owner = await usersService.createLocalUser({ + email: 'default-permissions@example.com', + username: 'default-permissions', + displayName: 'Default Permissions', + password: 'secret123', + roles: ['user'], + }); + const share = await sharesService.createShare({ + ownerId: owner.id, + sourceSpace: 'volume', + sourcePath: 'Volume/default-permissions', + isDirectory: true, + accessMode: 'readwrite', + }); + + expect(share).toMatchObject({ + allowDelete: true, + allowCreateFolder: true, + allowCreateFile: true, + allowUpload: true, + }); + }); + }); + describe('User Volumes', () => { it('should create and browse share from assigned volume path', async () => { const usersService = envContext.requireFresh('src/services/users'); @@ -339,10 +568,11 @@ describe('Shares Routes', () => { }); expect(create.status).toBe(201); - expect(create.body.directFileUrl).toContain(`/api/share/${create.body.shareToken}/file`); + expect(create.body.directFileUrl).toContain(`/api/share/${create.body.shareToken}`); + expect(create.body.directFileUrl).not.toContain('/file'); const publicApp = buildApp(); - const direct = await request(publicApp).get(`/api/share/${create.body.shareToken}/file`); + const direct = await request(publicApp).get(`/api/share/${create.body.shareToken}`); expect(direct.status).toBe(200); expect(direct.headers['content-disposition']).toContain('inline'); @@ -350,12 +580,164 @@ describe('Shares Routes', () => { expect(direct.text).toBe('hello direct link'); const download = await request(publicApp).get( - `/api/share/${create.body.shareToken}/file?mode=download` + `/api/share/${create.body.shareToken}?mode=download` ); expect(download.status).toBe(200); expect(download.headers['content-disposition']).toContain('attachment'); expect(download.headers['content-disposition']).toContain('hello.txt'); + + const raw = await request(publicApp).get(`/api/share/${create.body.shareToken}?mode=raw`); + expect(raw.status).toBe(200); + expect(raw.text).toBe('hello direct link'); + + const legacyDirect = await request(publicApp).get( + `/api/share/${create.body.shareToken}/file` + ); + expect(legacyDirect.status).toBe(200); + expect(legacyDirect.text).toBe('hello direct link'); + }); + + it('records the client IP when a shared file is accessed directly', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-direct-ip'); + await fs.mkdir(assignedRoot, { recursive: true }); + await fs.writeFile(path.join(assignedRoot, 'ip.txt'), 'track my ip'); + + const user = await usersService.createLocalUser({ + email: 'direct-ip@example.com', + username: 'direct-ip', + displayName: 'Direct Ip', + password: 'secret123', + roles: ['user'], + }); + + await userVolumesService.addVolumeToUser({ + userId: user.id, + label: 'DirectIpVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const ownerApp = buildApp({ user }); + const create = await request(ownerApp).post('/api/shares').send({ + sourcePath: 'DirectIpVol/ip.txt', + accessMode: 'readonly', + sharingType: 'anyone', + }); + expect(create.status).toBe(201); + + // Accessing the file directly is the common path for viewing a share; it + // must record the access IP (regression: it previously tracked with none). + const direct = await request(buildApp()).get(`/api/share/${create.body.shareToken}/file`); + expect(direct.status).toBe(200); + + const details = await request(ownerApp).get(`/api/shares/${create.body.id}`); + expect(details.status).toBe(200); + expect(details.body.stats.accessCount).toBeGreaterThan(0); + expect(typeof details.body.stats.lastAccessIp).toBe('string'); + expect(details.body.stats.lastAccessIp.length).toBeGreaterThan(0); + }); + + it('counts direct attachment deliveries as downloads, inline views as accesses', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-direct-download'); + await fs.mkdir(assignedRoot, { recursive: true }); + await fs.writeFile(path.join(assignedRoot, 'notes.txt'), 'count my download'); + + const user = await usersService.createLocalUser({ + email: 'direct-download@example.com', + username: 'direct-download', + displayName: 'Direct Download', + password: 'secret123', + roles: ['user'], + }); + + await userVolumesService.addVolumeToUser({ + userId: user.id, + label: 'DirectDownloadVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const ownerApp = buildApp({ user }); + const create = await request(ownerApp).post('/api/shares').send({ + sourcePath: 'DirectDownloadVol/notes.txt', + accessMode: 'readonly', + sharingType: 'anyone', + }); + expect(create.status).toBe(201); + + // Explicit download mode serves an attachment: it must increment the + // download counter (regression: it only ever counted an access). + const download = await request(buildApp()).get( + `/api/share/${create.body.shareToken}/file?mode=download` + ); + expect(download.status).toBe(200); + expect(download.headers['content-disposition']).toContain('attachment'); + + const afterDownload = await request(ownerApp).get(`/api/shares/${create.body.id}`); + expect(afterDownload.status).toBe(200); + expect(afterDownload.body.stats.downloadCount).toBe(1); + expect(afterDownload.body.stats.accessCount).toBe(0); + expect(afterDownload.body.stats.lastDownloadedAt).toBeTruthy(); + expect(typeof afterDownload.body.stats.lastDownloadIp).toBe('string'); + expect(afterDownload.body.stats.lastDownloadIp.length).toBeGreaterThan(0); + + // An inline view of the same link still counts as an access. + const view = await request(buildApp()).get(`/api/share/${create.body.shareToken}/file`); + expect(view.status).toBe(200); + expect(view.headers['content-disposition']).toContain('inline'); + + const afterView = await request(ownerApp).get(`/api/shares/${create.body.id}`); + expect(afterView.status).toBe(200); + expect(afterView.body.stats.accessCount).toBe(1); + expect(afterView.body.stats.downloadCount).toBe(1); + }); + + it('counts a direct directory ZIP delivery as a download', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-direct-zip-count'); + await fs.mkdir(path.join(assignedRoot, 'folder'), { recursive: true }); + await fs.writeFile(path.join(assignedRoot, 'folder', 'nested.txt'), 'nested file'); + + const user = await usersService.createLocalUser({ + email: 'direct-zip-count@example.com', + username: 'direct-zip-count', + displayName: 'Direct Zip Count', + password: 'secret123', + roles: ['user'], + }); + + await userVolumesService.addVolumeToUser({ + userId: user.id, + label: 'DirectZipCountVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const ownerApp = buildApp({ user }); + const create = await request(ownerApp).post('/api/shares').send({ + sourcePath: 'DirectZipCountVol/folder', + accessMode: 'readonly', + sharingType: 'anyone', + }); + expect(create.status).toBe(201); + + const direct = await request(buildApp()).get(`/api/share/${create.body.shareToken}/file`); + expect(direct.status).toBe(200); + expect(direct.headers['content-disposition']).toContain('attachment'); + + const details = await request(ownerApp).get(`/api/shares/${create.body.id}`); + expect(details.status).toBe(200); + expect(details.body.stats.downloadCount).toBe(1); + expect(details.body.stats.accessCount).toBe(0); }); it('should redirect a password-protected direct file until the password is verified', async () => { @@ -393,7 +775,7 @@ describe('Shares Routes', () => { const publicApp = buildApp(); const directBeforePassword = await request(publicApp).get( - `/api/share/${create.body.shareToken}/file` + `/api/share/${create.body.shareToken}` ); expect(directBeforePassword.status).toBe(302); expect(directBeforePassword.headers.location).toContain(`/share/${create.body.shareToken}`); @@ -407,7 +789,7 @@ describe('Shares Routes', () => { expect(verify.body.guestSessionId).toBeDefined(); const directAfterPassword = await request(publicApp) - .get(`/api/share/${create.body.shareToken}/file`) + .get(`/api/share/${create.body.shareToken}`) .set('X-Guest-Session', verify.body.guestSessionId); expect(directAfterPassword.status).toBe(200); @@ -445,10 +827,11 @@ describe('Shares Routes', () => { }); expect(create.status).toBe(201); - expect(create.body.directFileUrl).toContain(`/api/share/${create.body.shareToken}/file`); + expect(create.body.directFileUrl).toContain(`/api/share/${create.body.shareToken}`); + expect(create.body.directFileUrl).not.toContain('/file'); const publicApp = buildApp(); - const direct = await request(publicApp).get(`/api/share/${create.body.shareToken}/file`); + const direct = await request(publicApp).get(`/api/share/${create.body.shareToken}`); expect(direct.status).toBe(200); expect(direct.headers['content-type']).toContain('application/zip'); @@ -542,4 +925,271 @@ describe('Shares Routes', () => { expect(direct.status).toBe(403); }); }); + + describe('Shared Pastebin Editor', () => { + it('should keep a read-only public text share read-only', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-shared-editor'); + await fs.mkdir(assignedRoot, { recursive: true }); + await fs.writeFile( + path.join(assignedRoot, 'Analyze-FileServerData.ps1'), + 'Write-Output hello' + ); + + const user = await usersService.createLocalUser({ + email: 'shared-editor@example.com', + username: 'shared-editor', + displayName: 'Shared Editor', + password: 'secret123', + roles: ['user'], + }); + + await userVolumesService.addVolumeToUser({ + userId: user.id, + label: 'SharedEditorVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const ownerApp = buildApp({ user }); + const create = await request(ownerApp).post('/api/shares').send({ + sourcePath: 'SharedEditorVol/Analyze-FileServerData.ps1', + accessMode: 'readonly', + sharingType: 'anyone', + }); + expect(create.status).toBe(201); + + const publicApp = buildApp(); + const editor = await request(publicApp).get(`/api/share/${create.body.shareToken}/editor`); + expect(editor.status).toBe(200); + expect(editor.headers['cache-control']).toBe('private, no-cache'); + expect(editor.body).toMatchObject({ + name: 'Analyze-FileServerData.ps1', + content: 'Write-Output hello', + canDownload: true, + canWrite: false, + }); + + // Friendly links may include the source filename, but no arbitrary child path. + const friendly = await request(publicApp).get( + `/api/share/${create.body.shareToken}/editor/Analyze-FileServerData.ps1` + ); + expect(friendly.status).toBe(200); + expect(friendly.body.path).toBe(''); + + const write = await request(publicApp) + .put(`/api/share/${create.body.shareToken}/editor`) + .send({ content: 'should never be written' }); + expect(write.status).toBe(403); + expect( + await fs.readFile(path.join(assignedRoot, 'Analyze-FileServerData.ps1'), 'utf-8') + ).toBe('Write-Output hello'); + }); + + it('should send a large shared text file compressed', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-shared-editor-large'); + await fs.mkdir(assignedRoot, { recursive: true }); + const content = '# Journal\n\nUne ligne de texte, encore une.\n'.repeat(2000); + await fs.writeFile(path.join(assignedRoot, 'journal.md'), content); + + const user = await usersService.createLocalUser({ + email: 'shared-editor-large@example.com', + username: 'shared-editor-large', + displayName: 'Shared Editor Large', + password: 'secret123', + roles: ['user'], + }); + await userVolumesService.addVolumeToUser({ + userId: user.id, + label: 'SharedEditorLargeVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const create = await request(buildApp({ user })).post('/api/shares').send({ + sourcePath: 'SharedEditorLargeVol/journal.md', + accessMode: 'readonly', + sharingType: 'anyone', + }); + expect(create.status).toBe(201); + + const editor = await request(buildApp()) + .get(`/api/share/${create.body.shareToken}/editor`) + .set('Accept-Encoding', 'gzip, deflate'); + expect(editor.status).toBe(200); + expect(editor.headers['content-encoding']).toBe('gzip'); + expect(editor.headers.vary).toMatch(/accept-encoding/i); + expect(editor.body).toMatchObject({ name: 'journal.md', content, canWrite: false }); + }); + + /** + * The shared editor's answer says whether the visitor may save. Kept by the + * browser and revalidated, it must be read again when that changes, even + * though the file did not. + */ + it('should never hide a change of permission behind a 304', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-shared-editor-etag'); + await fs.mkdir(assignedRoot, { recursive: true }); + await fs.writeFile(path.join(assignedRoot, 'notes.txt'), 'Shared notes'); + + const user = await usersService.createLocalUser({ + email: 'shared-editor-etag@example.com', + username: 'shared-editor-etag', + displayName: 'Shared Editor Etag', + password: 'secret123', + roles: ['user'], + }); + await userVolumesService.addVolumeToUser({ + userId: user.id, + label: 'SharedEditorEtagVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const ownerApp = buildApp({ user }); + const create = await request(ownerApp).post('/api/shares').send({ + sourcePath: 'SharedEditorEtagVol/notes.txt', + accessMode: 'readonly', + sharingType: 'anyone', + }); + expect(create.status).toBe(201); + + const publicApp = buildApp(); + const editorUrl = `/api/share/${create.body.shareToken}/editor`; + const first = await request(publicApp).get(editorUrl); + expect(first.body).toMatchObject({ content: 'Shared notes', canWrite: false }); + expect(first.headers.etag).toMatch(/^W\/".+"$/); + + const unchanged = await request(publicApp) + .get(editorUrl) + .set('If-None-Match', first.headers.etag); + expect(unchanged.status).toBe(304); + + const updated = await request(ownerApp) + .put(`/api/shares/${create.body.id}`) + .send({ accessMode: 'readwrite' }); + expect(updated.status).toBe(200); + + const after = await request(publicApp) + .get(editorUrl) + .set('If-None-Match', first.headers.etag); + expect(after.status).toBe(200); + expect(after.body).toMatchObject({ content: 'Shared notes', canWrite: true }); + expect(after.headers.etag).not.toBe(first.headers.etag); + }); + + it('should save a text file only through a read-write share', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-shared-editor-write'); + await fs.mkdir(assignedRoot, { recursive: true }); + const filePath = path.join(assignedRoot, 'editable.txt'); + await fs.writeFile(filePath, 'initial content'); + + const user = await usersService.createLocalUser({ + email: 'shared-editor-write@example.com', + username: 'shared-editor-write', + displayName: 'Shared Editor Write', + password: 'secret123', + roles: ['user'], + }); + + await userVolumesService.addVolumeToUser({ + userId: user.id, + label: 'SharedEditorWriteVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const ownerApp = buildApp({ user }); + const create = await request(ownerApp).post('/api/shares').send({ + sourcePath: 'SharedEditorWriteVol/editable.txt', + accessMode: 'readwrite', + sharingType: 'anyone', + }); + expect(create.status).toBe(201); + + const publicApp = buildApp(); + const editor = await request(publicApp).get(`/api/share/${create.body.shareToken}/editor`); + expect(editor.status).toBe(200); + expect(editor.body.canWrite).toBe(true); + + const save = await request(publicApp) + .put(`/api/share/${create.body.shareToken}/editor`) + .send({ content: 'updated through the share' }); + expect(save.status).toBe(200); + expect(await fs.readFile(filePath, 'utf-8')).toBe('updated through the share'); + }); + + it('should require a verified guest session and reject binary shared files', async () => { + const usersService = envContext.requireFresh('src/services/users'); + const userVolumesService = envContext.requireFresh('src/services/userVolumesService'); + + const assignedRoot = path.join(envContext.tmpRoot, 'assigned-volume-shared-editor-protected'); + await fs.mkdir(assignedRoot, { recursive: true }); + await fs.writeFile(path.join(assignedRoot, 'protected.txt'), 'protected text'); + await fs.writeFile(path.join(assignedRoot, 'binary.dat'), Buffer.from([0, 1, 2, 3])); + + const user = await usersService.createLocalUser({ + email: 'shared-editor-protected@example.com', + username: 'shared-editor-protected', + displayName: 'Shared Editor Protected', + password: 'secret123', + roles: ['user'], + }); + + await userVolumesService.addVolumeToUser({ + userId: user.id, + label: 'SharedEditorProtectedVol', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + + const ownerApp = buildApp({ user }); + const protectedShare = await request(ownerApp).post('/api/shares').send({ + sourcePath: 'SharedEditorProtectedVol/protected.txt', + accessMode: 'readonly', + sharingType: 'anyone', + password: 'open-sesame', + }); + const binaryShare = await request(ownerApp).post('/api/shares').send({ + sourcePath: 'SharedEditorProtectedVol/binary.dat', + accessMode: 'readonly', + sharingType: 'anyone', + }); + expect(protectedShare.status).toBe(201); + expect(binaryShare.status).toBe(201); + + const publicApp = buildApp(); + const beforeVerification = await request(publicApp).get( + `/api/share/${protectedShare.body.shareToken}/editor` + ); + expect(beforeVerification.status).toBe(302); + + const verify = await request(publicApp) + .post(`/api/share/${protectedShare.body.shareToken}/verify`) + .send({ password: 'open-sesame' }); + expect(verify.status).toBe(200); + + const afterVerification = await request(publicApp) + .get(`/api/share/${protectedShare.body.shareToken}/editor`) + .set('X-Guest-Session', verify.body.guestSessionId); + expect(afterVerification.status).toBe(200); + expect(afterVerification.body.content).toBe('protected text'); + + const binary = await request(publicApp).get( + `/api/share/${binaryShare.body.shareToken}/editor` + ); + expect(binary.status).toBe(415); + }); + }); }); diff --git a/backend/tests/routes/sign-in-identifier.test.js b/backend/tests/routes/sign-in-identifier.test.js new file mode 100644 index 000000000..28b9e3646 --- /dev/null +++ b/backend/tests/routes/sign-in-identifier.test.js @@ -0,0 +1,119 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import request from 'supertest'; + +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * One box on the sign-in screen, and the same name for it all the way down. + * + * The box takes an email address or a username, so it is neither: it is + * whatever was typed. That name has to hold from the screen to the route, and + * when it did not, nothing said so. The screen sent `identifier`, the store + * passed on `email`, and `JSON.stringify` drops a key whose value is undefined + * — so the request went out carrying a password and nobody to sign in, and the + * answer was "invalid credentials", which is what a wrong password looks like. + * + * Both halves are asserted here: the route takes the name the screen sends, and + * the screen, the client and the store all use that one name. The second half + * is read off the frontend sources, because the chain is four files long and + * three of them have no runner here — and a chain that breaks silently in the + * middle is exactly what this is for. + */ + +const FRONTEND = path.join(__dirname, '..', '..', '..', 'frontend', 'src'); +const read = (relative) => fs.readFileSync(path.join(FRONTEND, relative), 'utf8'); + +describe('the name for what was typed into the sign-in box', () => { + let env; + let app; + + beforeEach(async () => { + env = await setupTestEnv({ + tag: 'sign-in-identifier-', + modules: ['src/services/db', 'src/routes/auth', 'src/middleware/errorHandler'], + envOverrides: { AUTH_ENABLED: 'true' }, + }); + await env.requireFresh('src/services/users').createLocalUser({ + email: 'alice@example.com', + username: 'alice', + displayName: 'Alice', + password: 'correct horse battery', + roles: ['admin'], + }); + app = createTestApp({ + router: env.requireFresh('src/routes/auth'), + mountPath: '/api/auth', + errorHandler: env.requireFresh('src/middleware/errorHandler').errorHandler, + }); + }); + + afterEach(async () => { + await env.cleanup(); + }); + + const signIn = (body) => request(app).post('/api/auth/login').send(body); + + it('signs in with the name the screen sends', async () => { + const response = await signIn({ + identifier: 'alice@example.com', + password: 'correct horse battery', + }); + + expect(response.status).toBe(200); + expect(response.body.user?.email).toBe('alice@example.com'); + }); + + it('takes a username in the same box', async () => { + const response = await signIn({ identifier: 'alice', password: 'correct horse battery' }); + + expect(response.status).toBe(200); + expect(response.body.user?.username).toBe('alice'); + }); + + it.each(['email', 'username'])('still takes the older name %s', async (name) => { + const response = await signIn({ + [name]: name === 'email' ? 'alice@example.com' : 'alice', + password: 'correct horse battery', + }); + + expect(response.status).toBe(200); + }); + + it('refuses a password that is wrong, and says nothing about which half', async () => { + const response = await signIn({ identifier: 'alice', password: 'not it' }); + + expect(response.status).toBe(401); + expect(response.body.error?.code).toBe('AUTH_INVALID_CREDENTIALS'); + }); + + /** + * The screen, the client and the store. A rename that stops at one of them + * leaves the next passing undefined, which is not an error anywhere — the key + * simply vanishes from the request body. + */ + it('is the name the screen, the client and the store all use', () => { + expect(read('views/AuthLoginView.vue')).toMatch(/auth\.login\(\{\s*identifier:/); + expect(read('api/auth.api.js')).toMatch(/const login = \(\{ identifier, password \}\)/); + expect(read('stores/auth.js')).toMatch(/const login = async \(\{ identifier, password \}\)/); + expect(read('stores/auth.js')).toMatch(/loginApi\(\{ identifier, password \}\)/); + }); + + /** + * Everything else the sign-in screen reads off the store. + * + * The same rename went through this screen and stopped before the store, and + * the identifier was only the half that failed loudly. `totpPending` reads + * undefined, so the box for the code from the authenticator never appears: + * a correct password on an account with a second factor lands on a screen + * that looks like it did nothing. Undefined is not an error in a template — + * it is a `v-if` that is false — so there is nothing to see but the absence. + */ + it.each(['totpPending', 'oidcStatus', 'cancelTotp', 'ensureStatus', 'forgetSession'])( + 'is on the store, because the screen reads it: %s', + (member) => { + expect(read('stores/auth.js')).toContain(member); + } + ); +}); diff --git a/backend/tests/routes/static-server.test.js b/backend/tests/routes/static-server.test.js new file mode 100644 index 000000000..cb1b9e4ca --- /dev/null +++ b/backend/tests/routes/static-server.test.js @@ -0,0 +1,154 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The single-page fallback, which every test until now skipped. + * + * `createApp({ skipStaticFiles: true })` is what the other suites pass, so the + * one route only production registers had no coverage at all — and a bare `*` + * left in it survived the Express 5 migration, was published, and crash-looped + * the container: path-to-regexp refuses the pattern while the route is being + * registered, so the server does not start. Seven hundred tests passed. + * + * A route nothing exercises is a route nothing protects. This suite builds the + * application the way it ships. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +/** The application as it ships: static files mounted, nothing skipped. */ +const buildShippedApp = async () => { + currentEnv = await setupTestEnv({ tag: 'static-server-', env: { AUTH_ENABLED: 'false' } }); + + // The frontend build lands next to the backend's source in the image. + const publicDir = path.join(currentEnv.tmpRoot, 'public'); + await fs.mkdir(publicDir, { recursive: true }); + await fs.writeFile(path.join(publicDir, 'index.html'), 'Explorer'); + await fs.writeFile(path.join(publicDir, 'app.js'), 'console.log("bundle");'); + + const staticServer = currentEnv.requireFresh('src/utils/staticServer'); + const express = require('express'); + const app = express(); + + // Mounted exactly as `createApp` mounts it, against a directory that exists. + staticServer.configureStaticFiles(app, publicDir); + return app; +}; + +/** + * A logo is served under the name it was written under, which is its own for + * every logo now, and a fixed name per type for one an earlier version wrote. + * An installation whose settings still point at the fixed name has nothing to + * migrate: that address keeps answering. + */ +describe('the custom logo', () => { + const OWN = 'logo-0b7f7c1e-3d44-4c55-9a8e-1f2a3b4c5d6e (1).png'; + + it.each([ + [ + 'under a name of its own', + OWN, + '/static/logos/logo-0b7f7c1e-3d44-4c55-9a8e-1f2a3b4c5d6e%20(1).png', + ], + [ + 'under the fixed name an earlier version used', + 'custom-logo.png', + '/static/logos/custom-logo.png', + ], + ])('is served %s, sandboxed', async (_label, name, url) => { + const app = await buildShippedApp(); + const logoDir = path.join(currentEnv.configDir, 'logos'); + await fs.mkdir(logoDir, { recursive: true }); + await fs.writeFile(path.join(logoDir, name), 'png bytes'); + + const response = await request(app).get(url); + + expect(response.status).toBe(200); + expect(response.body.toString()).toBe('png bytes'); + expect(response.headers['content-security-policy']).toBe('sandbox'); + }); + + it('is not served while it is still being written under its hidden name', async () => { + const app = await buildShippedApp(); + const logoDir = path.join(currentEnv.configDir, 'logos'); + await fs.mkdir(logoDir, { recursive: true }); + await fs.writeFile(path.join(logoDir, '.logo-0b7f7c1e.part'), 'half'); + + const response = await request(app).get('/static/logos/.logo-0b7f7c1e.part'); + + expect(response.text).not.toBe('half'); + }); +}); + +describe('the application as it ships', () => { + /** + * The failure this suite exists for does not produce a bad answer — it + * produces no server. Registering the routes is the assertion. + */ + it('registers its routes without refusing one', async () => { + await expect(buildShippedApp()).resolves.toBeTruthy(); + }); + + it('answers the address the application is opened at', async () => { + const app = await buildShippedApp(); + + const response = await request(app).get('/'); + + expect(response.status).toBe(200); + expect(response.text).toContain('Explorer'); + }); + + it('answers a deep route the browser owns, not the server', async () => { + const app = await buildShippedApp(); + + const response = await request(app).get('/browse/Docs/holiday/2026'); + + expect(response.status).toBe(200); + expect(response.text).toContain('Explorer'); + }); + + it('serves a real asset as itself rather than as the page', async () => { + const app = await buildShippedApp(); + + const response = await request(app).get('/app.js'); + + expect(response.status).toBe(200); + expect(response.text).toContain('bundle'); + }); + + it('leaves the API alone', async () => { + const app = await buildShippedApp(); + + const response = await request(app).get('/api/anything'); + + expect(response.status).toBe(404); + expect(response.text).not.toContain('Explorer'); + }); + + it('leaves the static asset routes alone', async () => { + const app = await buildShippedApp(); + + const response = await request(app).get('/static/thumbnails/missing.png'); + + expect(response.status).toBe(404); + expect(response.text).not.toContain('Explorer'); + }); + + it('does not answer a write as though it were a page', async () => { + const app = await buildShippedApp(); + + const response = await request(app).post('/browse/Docs'); + + expect(response.status).toBe(404); + }); +}); diff --git a/backend/tests/routes/text-compression.test.js b/backend/tests/routes/text-compression.test.js new file mode 100644 index 000000000..ac67e74a5 --- /dev/null +++ b/backend/tests/routes/text-compression.test.js @@ -0,0 +1,240 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import zlib from 'node:zlib'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A 19 MB Markdown file, opened from a server reached by its local address, + * travelled as 22 MB of JSON: nothing in the application compressed a response. + * + * Read with node's own client rather than supertest, which decompresses what it + * receives and would hide whether the bytes on the wire were compressed at all. + */ + +let envContext; + +const startServer = (server) => + new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => resolve(`http://127.0.0.1:${server.address().port}`)); + }); + +const closeServer = (server) => + new Promise((resolve, reject) => { + server.closeAllConnections?.(); + server.close((err) => (err ? reject(err) : resolve())); + }); + +const build = async () => { + envContext = await setupTestEnv({ tag: 'text-compression-test-' }); + const destination = path.join(envContext.volumeDir, 'Notes'); + await fs.mkdir(destination, { recursive: true }); + + const express = require('express'); + const { uploads } = envContext.requireFresh('src/config/index'); + const editorRoutes = envContext.requireFresh('src/routes/editor'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + + const app = express(); + app.use(express.json({ limit: uploads.maxJsonBodyBytes })); + app.use((req, _res, next) => { + req.user = { id: 'admin', email: 'admin@example.com', roles: ['admin'] }; + next(); + }); + app.use('/api', editorRoutes); + app.use(errorHandler); + + const server = http.createServer(app); + const baseUrl = await startServer(server); + return { destination, server, baseUrl }; +}; + +/** The response exactly as it arrived: status, headers and undecoded bytes. */ +const exchange = (baseUrl, { method = 'GET', target, headers = {}, body }) => + new Promise((resolve, reject) => { + const request = http.request(new URL(target, baseUrl), { method, headers }, (response) => { + const chunks = []; + response.on('data', (chunk) => chunks.push(chunk)); + response.on('end', () => + resolve({ + status: response.statusCode, + headers: response.headers, + body: Buffer.concat(chunks), + }) + ); + response.on('error', reject); + }); + request.on('error', reject); + if (body === undefined) { + request.end(); + } else { + request.setHeader('Content-Type', 'application/json'); + request.end(JSON.stringify(body)); + } + }); + +const openInEditor = (baseUrl, filePath, headers) => + exchange(baseUrl, { method: 'POST', target: '/api/editor', headers, body: { path: filePath } }); + +/** Prose and code with characters outside ASCII, well past the threshold. */ +const LARGE = Array.from( + { length: 2500 }, + (_, index) => + `## Section ${index} — été, 日本語\n\n\`\`\`js\nconst value${index} = "quoted \\"${index}\\"";\n\`\`\`\n` +).join('\n'); + +/** + * Under the threshold, and as compressible as text gets: a few bytes of text + * come out of gzip larger than they went in, and would stay uncompressed with + * no threshold at all. + */ +const SMALL = '# A short note\n\nNothing worth compressing here.\n'.repeat(300); + +const varies = (response) => + String(response.headers.vary || '') + .toLowerCase() + .split(',') + .map((field) => field.trim()) + .includes('accept-encoding'); + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('a large text file over the wire', () => { + it('goes gzip-compressed to a browser on plain http, and decodes to the exact JSON', async () => { + const { destination, server, baseUrl } = await build(); + await fs.writeFile(path.join(destination, 'big.md'), LARGE); + + try { + const response = await openInEditor(baseUrl, 'Notes/big.md', { + 'Accept-Encoding': 'gzip, deflate', + }); + + const expected = Buffer.from(JSON.stringify({ content: LARGE })); + expect(response.status).toBe(200); + expect(response.headers['content-encoding']).toBe('gzip'); + expect(varies(response)).toBe(true); + expect(Number(response.headers['content-length'])).toBe(response.body.length); + expect(response.body.length).toBeLessThan(expected.length / 2); + expect(zlib.gunzipSync(response.body).equals(expected)).toBe(true); + } finally { + await closeServer(server); + } + }); + + /** + * A browser caps each cache entry by the compressed bytes it stores, and a + * 20 MB Markdown file gzipped at level 1 came out just past the cap of a + * Chromium measured for this — so every opening downloaded it again. + */ + it('is gzipped at least as hard as level 4, so a large file still fits a browser cache', async () => { + const { destination, server, baseUrl } = await build(); + await fs.writeFile(path.join(destination, 'big.md'), LARGE); + + try { + const response = await openInEditor(baseUrl, 'Notes/big.md', { + 'Accept-Encoding': 'gzip, deflate', + }); + + const json = Buffer.from(JSON.stringify({ content: LARGE })); + expect(response.headers['content-encoding']).toBe('gzip'); + expect(response.body.length).toBeLessThanOrEqual(zlib.gzipSync(json, { level: 4 }).length); + } finally { + await closeServer(server); + } + }); + + it('goes as brotli when that is offered', async () => { + const { destination, server, baseUrl } = await build(); + await fs.writeFile(path.join(destination, 'big.md'), LARGE); + + try { + const response = await openInEditor(baseUrl, 'Notes/big.md', { + 'Accept-Encoding': 'gzip, deflate, br, zstd', + }); + + expect(response.headers['content-encoding']).toBe('br'); + expect(Number(response.headers['content-length'])).toBe(response.body.length); + expect(zlib.brotliDecompressSync(response.body).toString('utf8')).toBe( + JSON.stringify({ content: LARGE }) + ); + } finally { + await closeServer(server); + } + }); + + it('goes as it is to a client that accepts no coding, still varying on it', async () => { + const { destination, server, baseUrl } = await build(); + await fs.writeFile(path.join(destination, 'big.md'), LARGE); + + try { + const response = await openInEditor(baseUrl, 'Notes/big.md'); + + expect(response.headers['content-encoding']).toBeUndefined(); + expect(varies(response)).toBe(true); + expect(JSON.parse(response.body.toString('utf8'))).toEqual({ content: LARGE }); + } finally { + await closeServer(server); + } + }); + + it('is not compressed in a coding the client refused with q=0', async () => { + const { destination, server, baseUrl } = await build(); + await fs.writeFile(path.join(destination, 'big.md'), LARGE); + + try { + const refused = await openInEditor(baseUrl, 'Notes/big.md', { + 'Accept-Encoding': 'br;q=0, gzip;q=0', + }); + expect(refused.headers['content-encoding']).toBeUndefined(); + expect(JSON.parse(refused.body.toString('utf8'))).toEqual({ content: LARGE }); + + const onlyGzip = await openInEditor(baseUrl, 'Notes/big.md', { + 'Accept-Encoding': 'br;q=0, gzip', + }); + expect(onlyGzip.headers['content-encoding']).toBe('gzip'); + } finally { + await closeServer(server); + } + }); + + it('leaves a small file uncompressed', async () => { + const { destination, server, baseUrl } = await build(); + await fs.writeFile(path.join(destination, 'small.md'), SMALL); + + try { + const response = await openInEditor(baseUrl, 'Notes/small.md', { + 'Accept-Encoding': 'gzip, deflate, br', + }); + + expect(response.headers['content-encoding']).toBeUndefined(); + expect(varies(response)).toBe(true); + expect(JSON.parse(response.body.toString('utf8'))).toEqual({ content: SMALL }); + } finally { + await closeServer(server); + } + }); + + it('compresses the raw text the same way', async () => { + const { destination, server, baseUrl } = await build(); + await fs.writeFile(path.join(destination, 'big.md'), LARGE); + + try { + const response = await exchange(baseUrl, { + target: `/api/raw?path=${encodeURIComponent('Notes/big.md')}`, + headers: { 'Accept-Encoding': 'gzip, deflate' }, + }); + + expect(response.status).toBe(200); + expect(response.headers['content-type']).toBe('text/plain; charset=utf-8'); + expect(response.headers['content-encoding']).toBe('gzip'); + expect(varies(response)).toBe(true); + expect(zlib.gunzipSync(response.body).toString('utf8')).toBe(LARGE); + } finally { + await closeServer(server); + } + }); +}); diff --git a/backend/tests/routes/thumbnails.test.js b/backend/tests/routes/thumbnails.test.js new file mode 100644 index 000000000..d67aff738 --- /dev/null +++ b/backend/tests/routes/thumbnails.test.js @@ -0,0 +1,175 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import sharp from 'sharp'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The thumbnail endpoint is the only authorization a thumbnail ever gets: the + * image itself is served from /static, outside the auth middleware, and the + * token in the URL is what carries this route's decision there. That, and the + * refusals — which deliberately answer not-found rather than forbidden, so a + * request cannot be used to learn whether a file exists. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async (env = {}) => { + currentEnv = await setupTestEnv({ tag: 'thumbnails-', env }); + const dbService = currentEnv.requireFresh('src/services/db'); + const db = await dbService.getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1','u@example.com',1,'u','U','["admin"]', ?, ?)` + ).run(now, now); + return currentEnv.volumeDir; +}; + +const writeImage = async (volume, name) => { + const file = path.join(volume, name); + await fs.mkdir(path.dirname(file), { recursive: true }); + await sharp({ + create: { width: 32, height: 32, channels: 3, background: { r: 20, g: 90, b: 100 } }, + }) + .png() + .toFile(file); + return file; +}; + +const buildApp = () => { + const routes = currentEnv.requireFresh('src/routes/thumbnails'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'u1', email: 'u@example.com', roles: ['admin'] }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +describe('what may have a thumbnail', () => { + /** + * Accepted, not refused. What comes back depends on whether generation has + * finished — 200 with a URL, or 202 with nothing to point at yet — so + * asserting the shape here made the test fail on a slow runner and pass on + * mine. The URL is checked by the test below, which waits for it. + */ + it('accepts an image rather than refusing it', async () => { + const volume = await seed(); + await writeImage(volume, 'Photos/one.png'); + + const response = await request(buildApp()).get('/api/thumbnails/Photos/one.png'); + + expect([200, 202]).toContain(response.status); + }); + + /** + * The static handler that serves the file sits outside the auth middleware, + * so a thumbnail URL with no token is a file anyone can fetch. + */ + it('carries a token on the URL it hands back', async () => { + const volume = await seed(); + await writeImage(volume, 'Photos/two.png'); + const app = buildApp(); + + // Generation is queued, so the first answer is usually 202 with nothing to + // point at yet. Waiting for the real URL is the point: an answer with no + // thumbnail in it cannot show whether the token would have been on one. + let thumbnail = ''; + for (let attempt = 0; attempt < 40 && !thumbnail; attempt += 1) { + const response = await request(app).get('/api/thumbnails/Photos/two.png'); + thumbnail = response.body.thumbnail || ''; + if (!thumbnail) await new Promise((resolve) => setTimeout(resolve, 50)); + } + + expect(thumbnail).not.toBe(''); + expect(thumbnail).toMatch(/[?&]t=/); + }); + + it.each([['pdf'], ['txt'], ['zip']])('refuses a .%s', async (extension) => { + const volume = await seed(); + await fs.writeFile(path.join(volume, `file.${extension}`), 'content'); + + const response = await request(buildApp()).get(`/api/thumbnails/file.${extension}`); + + expect(response.status).toBe(400); + }); + + it('refuses a folder', async () => { + const volume = await seed(); + await fs.mkdir(path.join(volume, 'Photos'), { recursive: true }); + + const response = await request(buildApp()).get('/api/thumbnails/Photos'); + + expect(response.status).toBe(400); + expect(response.body.error.message).toBe('Thumbnails are only available for files.'); + }); + + it('requires a path', async () => { + await seed(); + + const response = await request(buildApp()).get('/api/thumbnails/'); + + expect(response.status).toBe(400); + }); +}); + +describe('what a refusal gives away', () => { + /** + * Not-found rather than forbidden, on purpose: two different answers would + * let someone map a volume they cannot read by watching which paths come + * back 403. + */ + it('says not found for a file that is not there', async () => { + await seed(); + + const response = await request(buildApp()).get('/api/thumbnails/Photos/absent.png'); + + expect(response.status).toBe(404); + }); + + it('says not found, not forbidden, for a file the caller may not read', async () => { + // USER_VOLUMES on and no volume assigned: the path resolves and exists, so + // the refusal comes from the access check and nowhere else. + const volume = await seed({ USER_VOLUMES: 'true' }); + await writeImage(volume, 'Photos/private.png'); + + const routes = currentEnv.requireFresh('src/routes/thumbnails'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'restricted-user', roles: [] }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + + const response = await request(app).get('/api/thumbnails/Photos/private.png'); + + expect(response.status).toBe(404); + }); +}); + +describe('when thumbnails are switched off', () => { + it('answers empty without looking at the path', async () => { + await seed({ THUMBNAILS_ENABLED: 'false' }); + + // A path that would otherwise be a 404 — the setting is checked first. + const response = await request(buildApp()).get('/api/thumbnails/Photos/absent.png'); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ thumbnail: '' }); + }); +}); diff --git a/backend/tests/routes/trash.test.js b/backend/tests/routes/trash.test.js index 5bd0fc337..5d3868e0a 100644 --- a/backend/tests/routes/trash.test.js +++ b/backend/tests/routes/trash.test.js @@ -629,7 +629,6 @@ describe('restoring into a chosen folder', () => { expect((await restoreTo('alice', { ids: [id] })).status).toBe(400); expect((await restoreTo('alice', { ids: [], destination: 'Archive' })).status).toBe(400); for (const destination of ['Archive/file.txt', 'Archive/nowhere', 'Projects/.nextexplorer']) { - // eslint-disable-next-line no-await-in-loop const response = await restoreTo('alice', { ids: [id], destination }); expect(refusedOutright(response), destination).toBe(true); } diff --git a/backend/tests/routes/tus-upload.test.js b/backend/tests/routes/tus-upload.test.js new file mode 100644 index 000000000..76618b9d0 --- /dev/null +++ b/backend/tests/routes/tus-upload.test.js @@ -0,0 +1,1457 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import fsSync from 'node:fs'; +import { Writable } from 'node:stream'; +import request from 'supertest'; +import { setupTestEnv, modulePath } from '../helpers/env-test-utils.js'; + +const MODULES = [ + 'src/config/env', + 'src/config/index', + 'src/services/db', + 'src/services/settingsService', + 'src/services/accessControlService', + 'src/services/accessManager', + 'src/services/authorizationService', + 'src/services/sharesService', + 'src/services/tusUploadService', + 'src/services/userVolumesService', + 'src/routes/upload', + 'src/middleware/errorHandler', + 'src/utils/pathUtils', +]; + +const encodeMetadata = (metadata) => + Object.entries(metadata) + .map(([key, value]) => `${key} ${Buffer.from(String(value)).toString('base64')}`) + .join(','); + +const startServer = (server) => + new Promise((resolve) => { + server.listen(0, () => { + const { port } = server.address(); + resolve(`http://127.0.0.1:${port}`); + }); + }); + +/** Sign in far enough to hold a session cookie, the way a browser does. */ +const establishSession = async (baseUrl) => { + const response = await request(baseUrl).get('/api/test-session'); + const cookies = response.headers['set-cookie']; + if (!cookies?.length) throw new Error('no session cookie was issued'); + return cookies.map((cookie) => cookie.split(';')[0]).join('; '); +}; + +const closeServer = (server) => + new Promise((resolve, reject) => { + server.closeAllConnections?.(); + server.close((err) => (err ? reject(err) : resolve())); + }); + +const TWO_HOURS_AGO = () => new Date(Date.now() - 2 * 60 * 60 * 1000); + +const codedError = (code, message) => Object.assign(new Error(`${code}: ${message}`), { code }); + +const createUpload = async (baseUrl, cookie, name, length) => { + const create = await request(baseUrl) + .post('/api/upload/tus') + .set('Cookie', cookie) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Length', String(length)) + .set( + 'Upload-Metadata', + encodeMetadata({ filename: name, relativePath: name, uploadTo: 'Nvm' }) + ); + expect(create.status).toBe(201); + return new URL(create.headers.location).pathname; +}; + +/** The whole file in one PATCH. Returns the supertest request, not yet sent. */ +const sendUpload = (baseUrl, uploadPath, content) => + request(baseUrl) + .patch(uploadPath) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Offset', '0') + .set('Content-Type', 'application/offset+octet-stream') + .send(content); + +/** Age both files of an upload in the cache past the default one-hour TTL. */ +const ageUpload = async (tusDir, uploadId) => { + const aged = TWO_HOURS_AGO(); + await fs.utimes(path.join(tusDir, uploadId), aged, aged); + await fs.utimes(path.join(tusDir, `${uploadId}.json`), aged, aged); +}; + +const expectUploadInCache = async (tusDir, uploadId, size) => { + expect((await fs.stat(path.join(tusDir, uploadId))).size).toBe(size); + await expect(fs.access(path.join(tusDir, `${uploadId}.json`))).resolves.toBeUndefined(); +}; + +const expectUploadGone = async (tusDir, uploadId) => { + await expect(fs.access(path.join(tusDir, uploadId))).rejects.toBeTruthy(); + await expect(fs.access(path.join(tusDir, `${uploadId}.json`))).rejects.toBeTruthy(); +}; + +/** + * Moves out of the upload cache, intercepted. A finished upload is moved by a + * hard link, or by a rename where the filesystem has none, so both are + * patched. `before` runs first — to hold the move, or to put something under + * the name at the last moment — and `code`, when given, then fails the move as + * a cache on another filesystem (EXDEV) or a folder the server may not write + * (EACCES) does. Answers the function that puts both back. + */ +const interceptCacheMoves = (tusDir, { code = null, before = null } = {}) => { + const originalLink = fs.link; + const originalRename = fs.rename; + const intercept = (original) => async (source, destination) => { + if (String(source).startsWith(tusDir + path.sep)) { + if (before) await before(source, destination); + if (code) throw codedError(code, 'intercepted move out of the upload cache'); + } + return original(source, destination); + }; + fs.link = intercept(originalLink); + fs.rename = intercept(originalRename); + return () => { + fs.link = originalLink; + fs.rename = originalRename; + }; +}; + +/** Ask for an upload's offset, as a client resuming it does. */ +const head = (baseUrl, uploadPath, user) => { + const req = request(baseUrl).head(uploadPath).set('Tus-Resumable', '1.0.0'); + return user ? req.set('X-Test-User', user) : req; +}; + +const finalizeError = (response) => { + const raw = response.headers['upload-finalize-error']; + return raw === undefined ? undefined : decodeURIComponent(raw); +}; + +const NOT_ALLOWED = + 'The file was received, but it could not be put in its folder: the server is not allowed to write there.'; + +describe('TUS upload route', () => { + let envContext; + + beforeEach(async () => { + envContext = await setupTestEnv({ + tag: 'tus-upload-test-', + modules: MODULES, + }); + }); + + afterEach(async () => { + await envContext.cleanup(); + }); + + const buildApp = () => { + const express = require('express'); + const http = require('node:http'); + const uploadRoutes = envContext.requireFresh('src/routes/upload'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + + const app = express(); + app.use(express.json()); + // A real session, deliberately: a route suite that mounts none is exercising + // a stack nobody runs. The seam it puts back — express-session replaces + // `res.end` with a version that reads the callback @tus/server passes as a + // body, which is what crashed the server before 3.0.2 — has a guard of its + // own in tests/middleware/response-end-compat.test.js. This does not + // replace it: removing the fix leaves these green. + const { configureSession } = envContext.requireFresh('src/middleware/session'); + configureSession(app); + // Something has to be written to the session for one to exist: the store's + // `touch` — the path the crash went through — only runs for a session that + // is already established and unmodified. + app.get('/api/test-session', (req, res) => { + req.session.establishedAt = new Date().toISOString(); + res.json({ ok: true }); + }); + const requestLog = []; + app.use((req, _res, next) => { + if (req.path.startsWith('/api/upload/tus')) requestLog.push(req.method); + // Someone other than the person uploading, when a test names one. + const other = req.headers['x-test-user']; + req.user = other + ? { id: other, email: `${other}@example.com`, roles: ['user'] } + : { id: 'admin', email: 'admin@example.com', roles: ['admin'] }; + next(); + }); + app.use('/api', uploadRoutes); + app.use(errorHandler); + const server = http.createServer(app); + // What reached the chunked upload route, by method, in order. + server.requestLog = requestLog; + return server; + }; + + // Both switches, not one. TUS carries forced chunking *and* the client-side + // automatic fallback, so it is refused only when neither is on — which the + // old name of this case ("when chunked uploads are disabled") did not say, + // leaving the more interesting half untested below. + it('refuses an upload when neither forced chunking nor the fallback is on', async () => { + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + + try { + const response = await request(baseUrl) + .post('/api/upload/tus') + .set('Cookie', cookie) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Length', '5') + .set( + 'Upload-Metadata', + encodeMetadata({ + filename: 'hello.txt', + relativePath: 'S05E09 - Épisode 9.avi', + uploadTo: 'Nvm', + }) + ); + + expect(response.status).toBe(403); + // The status alone would be satisfied by any refusal — an unmounted + // route, a failed authorisation. This is the one being tested. + expect(String(response.text)).toMatch(/chunked uploads are disabled/i); + } finally { + await closeServer(server); + } + }); + + /** + * A chunked upload cannot make a volume either. + * + * A folder at the top of the storage is a mount, not something the + * application creates — `POST /api/files/folder` has always refused one + * there. Started from the list of volumes, an uploaded folder tree made one + * (nxzai/NextExplorer#409), and the large files go through this route. + */ + it('refuses one that would make a folder at the top of the storage', async () => { + const settingsService = envContext.requireFresh('src/services/settingsService'); + await settingsService.setSystemSetting('system', 'uploads', { + chunkedEnabled: true, + chunkSizeBytes: 1024 * 1024, + }); + + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + + try { + const response = await request(baseUrl) + .post('/api/upload/tus') + .set('Cookie', cookie) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Length', '7') + .set( + 'Upload-Metadata', + encodeMetadata({ + filename: 'planted.txt', + relativePath: 'NouveauVolume/planted.txt', + uploadTo: '', + }) + ); + + expect(response.status).toBe(400); + expect(String(response.text)).toMatch(/root path/i); + expect(await fs.readdir(envContext.volumeDir)).toEqual([]); + } finally { + await closeServer(server); + } + }); + + // The gate reads `chunkedEnabled || chunkedAutoFallback`, and the fallback + // half had no test. Getting it wrong once already rejected every fallback + // upload with a 403 that reached the client as "network error". + it('accepts an upload when only the automatic fallback is on', async () => { + const settingsService = envContext.requireFresh('src/services/settingsService'); + await settingsService.setSystemSetting('system', 'uploads', { + chunkedEnabled: false, + chunkedAutoFallback: true, + chunkSizeBytes: 1024 * 1024, + }); + + await fs.mkdir(path.join(envContext.volumeDir, 'Nvm'), { recursive: true }); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + + try { + const response = await request(baseUrl) + .post('/api/upload/tus') + .set('Cookie', cookie) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Length', '5') + .set( + 'Upload-Metadata', + encodeMetadata({ filename: 'hello.txt', relativePath: 'hello.txt', uploadTo: 'Nvm' }) + ); + + expect(response.status).toBe(201); + expect(response.headers.location).toBeTruthy(); + } finally { + await closeServer(server); + } + }); + + it('stores a completed TUS upload in the authorized target directory', async () => { + const settingsService = envContext.requireFresh('src/services/settingsService'); + await settingsService.setSystemSetting('system', 'uploads', { + chunkedEnabled: true, + chunkSizeBytes: 1024 * 1024, + }); + + await fs.mkdir(path.join(envContext.volumeDir, 'Nvm'), { recursive: true }); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const content = Buffer.from('hello through tus'); + + try { + const create = await request(baseUrl) + .post('/api/upload/tus') + .set('Cookie', cookie) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Length', String(content.length)) + .set( + 'Upload-Metadata', + encodeMetadata({ + filename: 'hello.txt', + relativePath: 'S05E09 - Épisode 9.avi', + uploadTo: 'Nvm', + }) + ); + + expect(create.status).toBe(201); + expect(create.headers.location).toBeTruthy(); + + const uploadPath = new URL(create.headers.location).pathname; + const patch = await request(baseUrl) + .patch(uploadPath) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Offset', '0') + .set('Content-Type', 'application/offset+octet-stream') + .send(content); + + expect(patch.status).toBe(204); + await expect( + fs.readFile(path.join(envContext.volumeDir, 'Nvm', 'S05E09 - Épisode 9.avi'), 'utf8') + ).resolves.toBe('hello through tus'); + } finally { + await closeServer(server); + } + }); + + it('rejects TUS upload creation when storage is insufficient', async () => { + const settingsService = envContext.requireFresh('src/services/settingsService'); + await settingsService.setSystemSetting('system', 'uploads', { + chunkedEnabled: true, + chunkSizeBytes: 1024 * 1024, + }); + + await fs.mkdir(path.join(envContext.volumeDir, 'Nvm'), { recursive: true }); + const statfsSpy = vi.spyOn(fs, 'statfs').mockResolvedValue({ + bavail: 1, + bsize: 1024, + }); + + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + + try { + const response = await request(baseUrl) + .post('/api/upload/tus') + .set('Cookie', cookie) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Length', String(1024 * 1024)) + .set( + 'Upload-Metadata', + encodeMetadata({ + filename: 'large.bin', + relativePath: 'large.bin', + uploadTo: 'Nvm', + }) + ); + + expect(response.status).toBe(507); + expect(response.text).toContain('Not enough storage available'); + } finally { + statfsSpy.mockRestore(); + await closeServer(server); + } + }); + + it('cleans stale incomplete TUS uploads from the cache', async () => { + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + await fs.mkdir(tusDir, { recursive: true }); + + const staleUploadId = 'stale-upload'; + const activeUploadId = 'active-upload'; + const staleDate = new Date(Date.now() - 2 * 60 * 60 * 1000); + + await fs.writeFile(path.join(tusDir, staleUploadId), 'partial'); + await fs.writeFile( + path.join(tusDir, `${staleUploadId}.json`), + JSON.stringify({ + id: staleUploadId, + size: 1024, + metadata: { filename: 'stale.bin' }, + creation_date: staleDate.toISOString(), + }) + ); + await fs.utimes(path.join(tusDir, staleUploadId), staleDate, staleDate); + await fs.utimes(path.join(tusDir, `${staleUploadId}.json`), staleDate, staleDate); + + await fs.writeFile(path.join(tusDir, activeUploadId), 'partial'); + await fs.writeFile( + path.join(tusDir, `${activeUploadId}.json`), + JSON.stringify({ + id: activeUploadId, + size: 1024, + metadata: { filename: 'active.bin' }, + creation_date: new Date().toISOString(), + }) + ); + + const { cleanupExpiredUploads } = envContext.requireFresh('src/services/tusUploadService'); + await cleanupExpiredUploads({ force: true }); + + await expect(fs.access(path.join(tusDir, staleUploadId))).rejects.toBeTruthy(); + await expect(fs.access(path.join(tusDir, `${staleUploadId}.json`))).rejects.toBeTruthy(); + await expect(fs.access(path.join(tusDir, activeUploadId))).resolves.toBeUndefined(); + await expect(fs.access(path.join(tusDir, `${activeUploadId}.json`))).resolves.toBeUndefined(); + }); + + /** + * Uppy stringifies every field named in `allowedMetaFields`, whether or not + * the file carries it — a field the file doesn't have arrives as the literal + * string "undefined". Only folder uploads get `resolvedRelativePath`, so a + * plain file sends "undefined" and used to be stored under that name. + */ + it('ignores metadata Uppy stringified from a missing value', async () => { + const settingsService = envContext.requireFresh('src/services/settingsService'); + await settingsService.setSystemSetting('system', 'uploads', { + chunkedEnabled: true, + chunkSizeBytes: 1024 * 1024, + }); + + await fs.mkdir(path.join(envContext.volumeDir, 'Nvm'), { recursive: true }); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const content = Buffer.from('dropped straight onto the file list'); + + try { + const create = await request(baseUrl) + .post('/api/upload/tus') + .set('Cookie', cookie) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Length', String(content.length)) + .set( + 'Upload-Metadata', + encodeMetadata({ + filename: 'report.txt', + relativePath: 'report.txt', + resolvedRelativePath: 'undefined', + uploadTo: 'Nvm', + }) + ); + + expect(create.status).toBe(201); + + const uploadPath = new URL(create.headers.location).pathname; + const patch = await request(baseUrl) + .patch(uploadPath) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Offset', '0') + .set('Content-Type', 'application/offset+octet-stream') + .send(content); + + expect(patch.status).toBe(204); + await expect( + fs.readFile(path.join(envContext.volumeDir, 'Nvm', 'report.txt'), 'utf8') + ).resolves.toBe('dropped straight onto the file list'); + await expect( + fs.access(path.join(envContext.volumeDir, 'Nvm', 'undefined')) + ).rejects.toBeTruthy(); + } finally { + await closeServer(server); + } + }); + + /** + * A zero-byte file finishes inside its own creation request: the server sees + * offset === size and calls onUploadFinish from the POST handler. It then + * reads the upload back to compute Upload-Expires, so anything the hook + * removes has to still be there — otherwise creation answers 404 and the + * whole folder the file belonged to fails. + */ + it('accepts an empty file, which completes during creation', async () => { + const settingsService = envContext.requireFresh('src/services/settingsService'); + await settingsService.setSystemSetting('system', 'uploads', { + chunkedEnabled: true, + chunkSizeBytes: 1024 * 1024, + }); + + await fs.mkdir(path.join(envContext.volumeDir, 'Nvm'), { recursive: true }); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + + try { + const create = await request(baseUrl) + .post('/api/upload/tus') + .set('Cookie', cookie) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Length', '0') + .set( + 'Upload-Metadata', + encodeMetadata({ + filename: 'empty.js', + relativePath: 'empty.js', + uploadTo: 'Nvm', + }) + ); + + expect(create.status).toBe(201); + await expect( + fs.readFile(path.join(envContext.volumeDir, 'Nvm', 'empty.js'), 'utf8') + ).resolves.toBe(''); + } finally { + await closeServer(server); + } + }); + + /** + * When the cache and the destination sit on different filesystems — the norm + * once a user has more than one volume mounted — the finished file is copied + * rather than renamed. The client has stopped sending by then, so without + * this its progress bar sits at 100% for the length of the copy. + * + * The copy is observed from inside `unlink`, which the service calls once the + * bytes are written and before it forgets the upload. That is the last moment + * the entry is still there, and it makes the assertion deterministic instead + * of a race against a copy that finishes in milliseconds. + */ + it('reports the final copy while it is still running', async () => { + const settingsService = envContext.requireFresh('src/services/settingsService'); + await settingsService.setSystemSetting('system', 'uploads', { + chunkedEnabled: true, + chunkSizeBytes: 1024 * 1024, + }); + + await fs.mkdir(path.join(envContext.volumeDir, 'Nvm'), { recursive: true }); + + const content = Buffer.alloc(256 * 1024, 'x'); + let seen = null; + + // Only a move out of the cache crosses devices: the partial copy is written + // beside its destination and linked under its name within that filesystem. + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const originalUnlink = fs.unlink; + const restoreMoves = interceptCacheMoves(tusDir, { code: 'EXDEV' }); + + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + + fs.unlink = async (...args) => { + if (!seen) { + const response = await request(baseUrl).get('/api/upload/finalizations'); + seen = response.body; + } + return originalUnlink(...args); + }; + + try { + const create = await request(baseUrl) + .post('/api/upload/tus') + .set('Cookie', cookie) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Length', String(content.length)) + .set( + 'Upload-Metadata', + encodeMetadata({ + filename: 'large.bin', + relativePath: 'large.bin', + uploadTo: 'Nvm', + }) + ); + + expect(create.status).toBe(201); + + const uploadPath = new URL(create.headers.location).pathname; + const patch = await request(baseUrl) + .patch(uploadPath) + .set('Tus-Resumable', '1.0.0') + .set('Upload-Offset', '0') + .set('Content-Type', 'application/offset+octet-stream') + .send(content); + + expect(patch.status).toBe(204); + + // The copy was visible, counted, and named after the file being written. + expect(seen?.items).toEqual([ + { name: 'large.bin', copiedBytes: content.length, totalBytes: content.length }, + ]); + + // Copied, not just reported: the file is whole at its destination. + const stored = await fs.stat(path.join(envContext.volumeDir, 'Nvm', 'large.bin')); + expect(stored.size).toBe(content.length); + // With nothing left beside it from the copy. + expect(await fs.readdir(path.join(envContext.volumeDir, 'Nvm'))).toEqual(['large.bin']); + + // And forgotten once it is done, so nothing lingers in the list. + const after = await request(baseUrl).get('/api/upload/finalizations'); + expect(after.body).toEqual({ items: [] }); + } finally { + restoreMoves(); + fs.unlink = originalUnlink; + await closeServer(server); + } + }); + + const enableChunkedUploads = async () => { + const settingsService = envContext.requireFresh('src/services/settingsService'); + await settingsService.setSystemSetting('system', 'uploads', { + chunkedEnabled: true, + chunkSizeBytes: 1024 * 1024, + }); + await fs.mkdir(path.join(envContext.volumeDir, 'Nvm'), { recursive: true }); + }; + + /** + * The upload's last byte arrived, and moving the file into its folder failed. + * The data and its metadata stay in the cache — complete, so the sweep used to + * pass over them every time, and nothing else ever removed them. + */ + it('sweeps a finished upload that was never moved into place once past the TTL, not before', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const content = Buffer.from('sent in full, never arrived'); + + const restoreMoves = interceptCacheMoves(tusDir, { code: 'EACCES' }); + + try { + const oldPath = await createUpload(baseUrl, cookie, 'old.txt', content.length); + const recentPath = await createUpload(baseUrl, cookie, 'recent.txt', content.length); + expect((await sendUpload(baseUrl, oldPath, content)).status).toBe(500); + expect((await sendUpload(baseUrl, recentPath, content)).status).toBe(500); + restoreMoves(); + + const oldId = path.basename(oldPath); + const recentId = path.basename(recentPath); + await expectUploadInCache(tusDir, oldId, content.length); + await expectUploadInCache(tusDir, recentId, content.length); + expect(await fs.readdir(path.join(envContext.volumeDir, 'Nvm'))).toEqual([]); + + await ageUpload(tusDir, oldId); + const tus = require(modulePath('src/services/tusUploadService')); + await tus.cleanupExpiredUploads({ force: true }); + + await expectUploadGone(tusDir, oldId); + await expectUploadInCache(tusDir, recentId, content.length); + } finally { + restoreMoves(); + await closeServer(server); + } + }); + + /** + * Age does not tell a stuck upload from one moving into place: a copy to + * another filesystem can outlast the TTL, and an empty PATCH at the final + * offset finishes an old upload again. The move is held here while the + * upload is older than the TTL, and the sweep runs in between. + */ + it('leaves an old upload alone while it is being moved into place', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const content = Buffer.from('moved while the sweep ran'); + + let markMoveReached; + const moveReached = new Promise((resolve) => { + markMoveReached = resolve; + }); + let releaseMove; + const moveReleased = new Promise((resolve) => { + releaseMove = resolve; + }); + + const restoreMoves = interceptCacheMoves(tusDir, { + code: 'EXDEV', + before: async () => { + markMoveReached(); + await moveReleased; + }, + }); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'late.txt', content.length); + const uploadId = path.basename(uploadPath); + const pending = sendUpload(baseUrl, uploadPath, content).then((response) => response); + + await moveReached; + await ageUpload(tusDir, uploadId); + const tus = require(modulePath('src/services/tusUploadService')); + await tus.cleanupExpiredUploads({ force: true }); + await expectUploadInCache(tusDir, uploadId, content.length); + + releaseMove(); + expect((await pending).status).toBe(204); + await expect( + fs.readFile(path.join(envContext.volumeDir, 'Nvm', 'late.txt'), 'utf8') + ).resolves.toBe('moved while the sweep ran'); + } finally { + releaseMove(); + restoreMoves(); + await closeServer(server); + } + }); + + /** + * Across filesystems the file is written again, and a copy stopped halfway — + * a full disk here, a killed process in production — used to leave a + * truncated file under the name the user asked for. The folder is read while + * part of the file is on disk, then the copy fails. + */ + it('never shows a partial copy under the final name, and removes it when the copy fails', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + const content = Buffer.alloc(256 * 1024, 'x'); + + const originalCreateWriteStream = fsSync.createWriteStream; + const restoreMoves = interceptCacheMoves(tusDir, { code: 'EXDEV' }); + + let partialBytes = 0; + let folderDuringCopy = null; + fsSync.createWriteStream = (target, options) => { + const real = originalCreateWriteStream(target, options); + if (!String(target).startsWith(nvmDir + path.sep)) return real; + let chunks = 0; + return new Writable({ + write(chunk, _encoding, callback) { + chunks += 1; + if (chunks === 1) { + real.write(chunk, callback); + return; + } + real.end(() => { + partialBytes = fsSync.statSync(target).size; + folderDuringCopy = fsSync.readdirSync(nvmDir); + callback(codedError('ENOSPC', 'no space left on device')); + }); + }, + final(callback) { + real.end(callback); + }, + }); + }; + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'large.bin', content.length); + const response = await sendUpload(baseUrl, uploadPath, content); + // A full volume, said as such: 507 and the reason, not a generic 500. + expect(response.status).toBe(507); + expect(finalizeError(response)).toBe( + 'The file was received, but it could not be put in its folder: there is not enough space left on the volume.' + ); + + // A partial file really was on disk when the folder was read. + expect(partialBytes).toBeGreaterThan(0); + expect(partialBytes).toBeLessThan(content.length); + expect(folderDuringCopy).not.toContain('large.bin'); + + // Nothing left in the folder, and the upload still whole in the cache. + expect(await fs.readdir(nvmDir)).toEqual([]); + await expectUploadInCache(tusDir, path.basename(uploadPath), content.length); + } finally { + restoreMoves(); + fsSync.createWriteStream = originalCreateWriteStream; + await closeServer(server); + } + }); + + /** + * A file written under its own name used to hold that name for the copy's + * whole length; a hidden partial copy does not, so whatever lands there + * meanwhile must not be overwritten when the copy takes the name. The move is + * held before the copy, and a file arrives under the name. + */ + it('does not overwrite a file that arrives under the same name during the copy', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + const content = Buffer.from('the upload'); + + let markMoveReached; + const moveReached = new Promise((resolve) => { + markMoveReached = resolve; + }); + let releaseMove; + const moveReleased = new Promise((resolve) => { + releaseMove = resolve; + }); + + const restoreMoves = interceptCacheMoves(tusDir, { + code: 'EXDEV', + before: async () => { + markMoveReached(); + await moveReleased; + }, + }); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'large.bin', content.length); + const pending = sendUpload(baseUrl, uploadPath, content).then((response) => response); + + await moveReached; + await fs.writeFile(path.join(nvmDir, 'large.bin'), 'arrived meanwhile'); + releaseMove(); + expect((await pending).status).toBe(204); + + expect((await fs.readdir(nvmDir)).sort()).toEqual(['large (1).bin', 'large.bin']); + await expect(fs.readFile(path.join(nvmDir, 'large.bin'), 'utf8')).resolves.toBe( + 'arrived meanwhile' + ); + await expect(fs.readFile(path.join(nvmDir, 'large (1).bin'), 'utf8')).resolves.toBe( + 'the upload' + ); + } finally { + releaseMove(); + restoreMoves(); + await closeServer(server); + } + }); + + /** + * A process killed during the copy leaves its hidden partial file beside the + * destination. The direct upload path sweeps such remains where it is about + * to write; a chunked upload to the same folder does too. + */ + it('removes what a killed copy left in the destination when the next upload is created', async () => { + await enableChunkedUploads(); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + const remnant = path.join(nvmDir, '.upload-0123456789abcdef.uploading'); + await fs.writeFile(remnant, 'half a file'); + const twoDaysAgo = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000); + await fs.utimes(remnant, twoDaysAgo, twoDaysAgo); + + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + + try { + await createUpload(baseUrl, cookie, 'next.txt', 5); + await expect(fs.access(remnant)).rejects.toBeTruthy(); + } finally { + await closeServer(server); + } + }); + + /** + * On one filesystem the finished file is linked into its folder. The name is + * free when the move begins; a file put under it at the last moment, just + * before the move, is kept, and the upload takes the next name. + */ + it('does not overwrite a file that arrives under the name as the upload is moved in', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + const content = Buffer.from('the upload'); + + let arrived = false; + const restoreMoves = interceptCacheMoves(tusDir, { + before: async (_source, destination) => { + if (arrived) return; + arrived = true; + await fs.writeFile(destination, 'arrived meanwhile'); + }, + }); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'notes.txt', content.length); + expect((await sendUpload(baseUrl, uploadPath, content)).status).toBe(204); + + expect(arrived).toBe(true); + expect((await fs.readdir(nvmDir)).sort()).toEqual(['notes (1).txt', 'notes.txt']); + await expect(fs.readFile(path.join(nvmDir, 'notes.txt'), 'utf8')).resolves.toBe( + 'arrived meanwhile' + ); + await expect(fs.readFile(path.join(nvmDir, 'notes (1).txt'), 'utf8')).resolves.toBe( + 'the upload' + ); + } finally { + restoreMoves(); + await closeServer(server); + } + }); + + /** + * Every byte arrived and the file could not be moved into its folder. Thrown, + * that was a 500 with a generic body, which the client retried into a false + * success (see the HEAD tests below). It is answered with the reason, in the + * body and in a header a cross-origin client is allowed to read, and without + * the server's own paths. + */ + it('answers a move into the folder that fails with the reason, in a header the client can read', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const content = Buffer.from('sent in full'); + const restoreMoves = interceptCacheMoves(tusDir, { code: 'EACCES' }); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'notes.txt', content.length); + const response = await sendUpload(baseUrl, uploadPath, content); + + expect(response.status).toBe(500); + expect(finalizeError(response)).toBe(NOT_ALLOWED); + expect(response.text.trim()).toBe(NOT_ALLOWED); + expect(response.text).not.toContain(envContext.tmpRoot); + expect(response.headers['access-control-expose-headers']).toMatch( + /\bUpload-Finalize-Error\b/ + ); + await expectUploadInCache(tusDir, path.basename(uploadPath), content.length); + } finally { + restoreMoves(); + await closeServer(server); + } + }); + + /** + * The client retries a failed PATCH by asking for the offset. @tus/server + * answered from the cache, where the upload is complete, and tus-js-client + * then reported the upload as done without another request: a file that + * never reached its folder, shown as uploaded. The move is tried again + * instead, and the offset is complete only once the file is in its folder. + */ + it('tries the move again when asked for the offset, and says complete only once the file is in place', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + const content = Buffer.from('placed on the second try'); + const restoreMoves = interceptCacheMoves(tusDir, { code: 'EACCES' }); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'notes.txt', content.length); + const uploadId = path.basename(uploadPath); + expect((await sendUpload(baseUrl, uploadPath, content)).status).toBe(500); + + // Still failing: an error with the reason. Never complete, and never the + // other refusals, which the client takes as an upload to create again. + const failing = await head(baseUrl, uploadPath); + expect(failing.status).toBe(423); + expect(finalizeError(failing)).toBe(NOT_ALLOWED); + expect(failing.headers['upload-offset']).toBeUndefined(); + expect(await fs.readdir(nvmDir)).toEqual([]); + await expectUploadInCache(tusDir, uploadId, content.length); + + restoreMoves(); + const placed = await head(baseUrl, uploadPath); + expect(placed.status).toBe(200); + expect(placed.headers['upload-offset']).toBe(String(content.length)); + expect(placed.headers['upload-length']).toBe(String(content.length)); + expect(placed.headers['cache-control']).toBe('no-store'); + expect(finalizeError(placed)).toBeUndefined(); + await expect(fs.readFile(path.join(nvmDir, 'notes.txt'), 'utf8')).resolves.toBe( + 'placed on the second try' + ); + await expectUploadGone(tusDir, uploadId); + + // Asked again: still complete, and placed once. + expect((await head(baseUrl, uploadPath)).status).toBe(200); + expect(await fs.readdir(nvmDir)).toEqual(['notes.txt']); + } finally { + restoreMoves(); + await closeServer(server); + } + }); + + /** + * Once placed, the upload leaves the cache, and @tus/server answers a HEAD for + * it with 404, which tus-js-client takes as an upload to start over: a PATCH + * whose response was lost during a long copy was retried that way, and the + * whole file sent again and stored twice. Said only to the person who sent it. + */ + it('says an upload placed a moment ago is complete, to the person who sent it', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const content = Buffer.from('already in its folder'); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'notes.txt', content.length); + expect((await sendUpload(baseUrl, uploadPath, content)).status).toBe(204); + + const response = await head(baseUrl, uploadPath); + expect(response.status).toBe(200); + expect(response.headers['upload-offset']).toBe(String(content.length)); + expect(response.headers['upload-length']).toBe(String(content.length)); + expect(response.headers['tus-resumable']).toBe('1.0.0'); + expect(response.headers['cache-control']).toBe('no-store'); + + expect((await head(baseUrl, uploadPath, 'someone-else')).status).toBe(404); + } finally { + await closeServer(server); + } + }); + + /** + * Asked for the offset while the file is being moved in, by a client whose + * PATCH response was lost during a long copy, the answer waits for that move + * rather than reading the cache, and the file is placed once. + */ + it('waits for a move in progress before saying the upload is complete', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + const content = Buffer.from('moved while someone asked'); + + let markMoveReached; + const moveReached = new Promise((resolve) => { + markMoveReached = resolve; + }); + let releaseMove; + const moveReleased = new Promise((resolve) => { + releaseMove = resolve; + }); + const restoreMoves = interceptCacheMoves(tusDir, { + before: async () => { + markMoveReached(); + await moveReleased; + }, + }); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'late.txt', content.length); + const pending = sendUpload(baseUrl, uploadPath, content).then((response) => response); + await moveReached; + + let placedWhenAnswered = null; + const asked = head(baseUrl, uploadPath).then((response) => { + placedWhenAnswered = fsSync.existsSync(path.join(nvmDir, 'late.txt')); + return response; + }); + await new Promise((resolve) => setTimeout(resolve, 150)); + expect(placedWhenAnswered).toBeNull(); + + releaseMove(); + expect((await pending).status).toBe(204); + const answer = await asked; + expect(answer.status).toBe(200); + expect(answer.headers['upload-offset']).toBe(String(content.length)); + expect(placedWhenAnswered).toBe(true); + expect(await fs.readdir(nvmDir)).toEqual(['late.txt']); + } finally { + releaseMove(); + restoreMoves(); + await closeServer(server); + } + }); + + /** + * The move tried again on a HEAD is authorised with that request's user, as + * the PATCH was: someone who may not upload to the folder cannot finish an + * upload into it. + */ + it('does not move a stuck upload into place for someone who may not upload there', async () => { + await enableChunkedUploads(); + await envContext + .requireFresh('src/services/accessControlService') + .setRules([{ path: 'Nvm', recursive: true, permissions: 'ro' }]); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + const content = Buffer.from('an administrator sent this'); + const restoreMoves = interceptCacheMoves(tusDir, { code: 'EACCES' }); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'notes.txt', content.length); + const uploadId = path.basename(uploadPath); + expect((await sendUpload(baseUrl, uploadPath, content)).status).toBe(500); + restoreMoves(); + + const refused = await head(baseUrl, uploadPath, 'reader'); + expect(refused.status).toBe(403); + expect(await fs.readdir(nvmDir)).toEqual([]); + await expectUploadInCache(tusDir, uploadId, content.length); + + // The administrator who sent it still can. + expect((await head(baseUrl, uploadPath)).status).toBe(200); + expect(await fs.readdir(nvmDir)).toEqual(['notes.txt']); + } finally { + restoreMoves(); + await closeServer(server); + } + }); + + /** + * Send `content` with tus-js-client, the library behind the browser's + * uploads, and answer how it ended. Its own decision on what to retry, with + * no wait in between. + */ + const uploadWithClient = (baseUrl, name, content) => + new Promise((resolve) => { + const { Upload } = require('tus-js-client'); + const upload = new Upload(content, { + endpoint: `${baseUrl}/api/upload/tus`, + metadata: { filename: name, relativePath: name, uploadTo: 'Nvm' }, + retryDelays: [0, 0, 0], + onSuccess: () => resolve({ succeeded: true }), + onError: (error) => resolve({ succeeded: false, error }), + }); + upload.start(); + }); + + /** + * What the person finally sees is the client's reading of the exchange, so + * the exchange is played with the client itself. A move that keeps failing + * used to end in a success: the retry's HEAD was answered complete from the + * cache. It ends in an error carrying the reason, and the file is not created + * and sent a second time, which tus-js-client does for any refusal of that + * HEAD other than 423. + */ + it('ends, for tus-js-client, in an error with the reason when the move keeps failing', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + const restoreMoves = interceptCacheMoves(tusDir, { code: 'EACCES' }); + + try { + const outcome = await uploadWithClient(baseUrl, 'notes.txt', Buffer.from('never placed')); + + expect(outcome.succeeded).toBe(false); + const header = outcome.error?.originalResponse?.getHeader('Upload-Finalize-Error'); + expect(decodeURIComponent(header)).toBe(NOT_ALLOWED); + expect(server.requestLog.filter((method) => method === 'POST')).toHaveLength(1); + expect(await fs.readdir(nvmDir)).toEqual([]); + } finally { + restoreMoves(); + await closeServer(server); + } + }); + + it('ends, for tus-js-client, in a success only once a retried move put the file in its folder', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + let failures = 0; + const restoreMoves = interceptCacheMoves(tusDir, { + before: async () => { + if (failures > 0) return; + failures += 1; + throw codedError('EACCES', 'permission denied'); + }, + }); + + try { + const outcome = await uploadWithClient( + baseUrl, + 'notes.txt', + Buffer.from('placed on the retry') + ); + + expect(outcome.error).toBeUndefined(); + expect(outcome.succeeded).toBe(true); + expect(failures).toBe(1); + await expect(fs.readFile(path.join(nvmDir, 'notes.txt'), 'utf8')).resolves.toBe( + 'placed on the retry' + ); + expect(await fs.readdir(nvmDir)).toEqual(['notes.txt']); + expect(server.requestLog.filter((method) => method === 'POST')).toHaveLength(1); + } finally { + restoreMoves(); + await closeServer(server); + } + }); + + /** + * Across filesystems the cache copy is removed once the file is in its + * folder. A failure to remove it used to fail the upload, though the file + * had arrived, and a retry would then have placed it a second time. What + * stays in the cache is the sweep's to remove. + */ + it('reports a copied upload as arrived even when its cache copy cannot be removed', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + const content = Buffer.from('copied, then stuck in the cache'); + const restoreMoves = interceptCacheMoves(tusDir, { code: 'EXDEV' }); + const originalUnlink = fs.unlink; + let refusedRemoval = false; + fs.unlink = async (target) => { + const inCache = String(target).startsWith(tusDir + path.sep); + if (inCache && !String(target).endsWith('.json')) { + refusedRemoval = true; + throw codedError('EBUSY', 'resource busy or locked'); + } + return originalUnlink(target); + }; + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'notes.txt', content.length); + const response = await sendUpload(baseUrl, uploadPath, content); + + expect(refusedRemoval).toBe(true); + expect(response.status).toBe(204); + expect(finalizeError(response)).toBeUndefined(); + await expect(fs.readFile(path.join(nvmDir, 'notes.txt'), 'utf8')).resolves.toBe( + 'copied, then stuck in the cache' + ); + + // Asked again, it is complete, and still in its folder once. + expect((await head(baseUrl, uploadPath)).status).toBe(200); + expect(await fs.readdir(nvmDir)).toEqual(['notes.txt']); + } finally { + fs.unlink = originalUnlink; + restoreMoves(); + await closeServer(server); + } + }); + + /** + * Only an upload whose every byte arrived is answered here. One still being + * sent is answered as before, with its real offset, and nothing is moved + * into the folder before its last byte. + */ + /** + * A chunked upload told the folder sizes nothing: only the periodic + * reconciliation ever counted its file. It is announced as a direct upload is. + */ + it('tells the folder sizes about the file it placed', async () => { + await enableChunkedUploads(); + const hooks = require(modulePath('src/services/folderSizeHooks')); + const written = vi.spyOn(hooks, 'onFileWritten'); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const content = Buffer.from('counted in its folder'); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'notes.txt', content.length); + expect((await sendUpload(baseUrl, uploadPath, content)).status).toBe(204); + + expect(written).toHaveBeenCalledTimes(1); + const [writtenPath, writtenSize] = written.mock.calls[0]; + expect(await fs.realpath(writtenPath)).toBe( + await fs.realpath(path.join(envContext.volumeDir, 'Nvm', 'notes.txt')) + ); + expect(writtenSize).toBe(content.length); + } finally { + written.mockRestore(); + await closeServer(server); + } + }); + + /** + * The memory of uploads placed a moment ago goes with the process. After a + * restart, a client asking for the offset of one got 404 and sent the whole + * file again, into "name (1)". A record on disk answers it instead, to the + * person who sent it, until the cache sweep removes it. + */ + it('says an upload placed just before a restart is complete, from its record on disk', async () => { + await enableChunkedUploads(); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + const content = Buffer.from('placed, then the server restarted'); + let uploadPath; + + const before = buildApp(); + const beforeUrl = await startServer(before); + try { + const cookie = await establishSession(beforeUrl); + uploadPath = await createUpload(beforeUrl, cookie, 'notes.txt', content.length); + expect((await sendUpload(beforeUrl, uploadPath, content)).status).toBe(204); + } finally { + await closeServer(before); + } + + // A new process: nothing of the old one's memory. + const tus = envContext.requireFresh('src/services/tusUploadService'); + const after = buildApp(); + const afterUrl = await startServer(after); + try { + const response = await head(afterUrl, uploadPath); + expect(response.status).toBe(200); + expect(response.headers['upload-offset']).toBe(String(content.length)); + expect(response.headers['upload-length']).toBe(String(content.length)); + expect((await head(afterUrl, uploadPath, 'someone-else')).status).toBe(404); + expect(await fs.readdir(nvmDir)).toEqual(['notes.txt']); + + // Past the time an unfinished upload is kept, the sweep removes the record. + const record = path.join(tusDir, '.finished', `${path.basename(uploadPath)}.json`); + const aged = TWO_HOURS_AGO(); + const data = JSON.parse(await fs.readFile(record, 'utf8')); + await fs.writeFile(record, JSON.stringify({ ...data, at: aged.getTime() })); + await fs.utimes(record, aged, aged); + // An aged record answers nothing any more, even before the sweep removes it. + expect((await head(afterUrl, uploadPath)).status).toBe(404); + await tus.cleanupInactiveUploads(); + await expect(fs.access(record)).rejects.toBeTruthy(); + expect((await head(afterUrl, uploadPath)).status).toBe(404); + } finally { + await closeServer(after); + } + }); + + /** + * Every HEAD retrying a move that keeps failing wrote the same error line. + * The error is logged once for each reason, and the repeats at debug. + */ + it('logs a move that keeps failing once for each reason', async () => { + await enableChunkedUploads(); + const logger = require(modulePath('src/utils/logger')); + const errors = vi.spyOn(logger, 'error'); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + const content = Buffer.from('never placed'); + const failures = () => + errors.mock.calls.filter( + ([, message]) => message === 'A finished TUS upload could not be moved into its folder' + ).length; + let restoreMoves = interceptCacheMoves(tusDir, { code: 'EACCES' }); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'notes.txt', content.length); + expect((await sendUpload(baseUrl, uploadPath, content)).status).toBe(500); + expect((await head(baseUrl, uploadPath)).status).toBe(423); + expect((await head(baseUrl, uploadPath)).status).toBe(423); + expect(failures()).toBe(1); + + restoreMoves(); + restoreMoves = interceptCacheMoves(tusDir, { code: 'EROFS' }); + expect((await head(baseUrl, uploadPath)).status).toBe(423); + expect(failures()).toBe(2); + } finally { + restoreMoves(); + errors.mockRestore(); + await closeServer(server); + } + }); + + it('answers an unfinished upload with its offset and moves nothing', async () => { + await enableChunkedUploads(); + const server = buildApp(); + const baseUrl = await startServer(server); + const cookie = await establishSession(baseUrl); + const nvmDir = path.join(envContext.volumeDir, 'Nvm'); + + try { + const uploadPath = await createUpload(baseUrl, cookie, 'half.txt', 20); + const first = await sendUpload(baseUrl, uploadPath, Buffer.from('0123456789')); + expect(first.status).toBe(204); + + const response = await head(baseUrl, uploadPath); + expect(response.status).toBe(200); + expect(response.headers['upload-offset']).toBe('10'); + expect(response.headers['upload-length']).toBe('20'); + expect(await fs.readdir(nvmDir)).toEqual([]); + } finally { + await closeServer(server); + } + }); +}); + +describe('TUS upload cache sweep timer', () => { + let envContext; + + beforeEach(async () => { + envContext = await setupTestEnv({ + tag: 'tus-sweep-timer-test-', + env: { TUS_CLEANUP_INTERVAL_MS: '40' }, + }); + }); + + afterEach(async () => { + await envContext.cleanup(); + }); + + const writeAbandonedUpload = async (tusDir, uploadId) => { + await fs.writeFile(path.join(tusDir, uploadId), 'partial'); + await fs.writeFile( + path.join(tusDir, `${uploadId}.json`), + JSON.stringify({ id: uploadId, size: 1024, metadata: { filename: `${uploadId}.bin` } }) + ); + await ageUpload(tusDir, uploadId); + }; + + const waitUntilGone = async (filePath, timeoutMs = 3000) => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + await fs.access(filePath); + } catch { + return true; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + return false; + }; + + /** + * Sweeping used to happen at load and when an upload was created, so a + * server nobody uploaded to kept whatever its cache held. Nothing here + * creates an upload: only the timer can remove the second file, written + * after the sweep that removed the first had already read the directory. + */ + it('sweeps the cache on a timer until stopped', async () => { + const tusDir = path.join(envContext.cacheDir, 'tus-uploads'); + await fs.mkdir(tusDir, { recursive: true }); + const tus = envContext.requireFresh('src/services/tusUploadService'); + + await writeAbandonedUpload(tusDir, 'before-start'); + tus.startCacheSweep(); + try { + expect(await waitUntilGone(path.join(tusDir, 'before-start'))).toBe(true); + await writeAbandonedUpload(tusDir, 'while-running'); + expect(await waitUntilGone(path.join(tusDir, 'while-running'))).toBe(true); + } finally { + await tus.stopCacheSweep(); + } + + await writeAbandonedUpload(tusDir, 'after-stop'); + // Ten intervals. + await new Promise((resolve) => setTimeout(resolve, 400)); + await expect(fs.access(path.join(tusDir, 'after-stop'))).resolves.toBeUndefined(); + }); +}); diff --git a/backend/tests/routes/two-factor-auth.test.js b/backend/tests/routes/two-factor-auth.test.js new file mode 100644 index 000000000..71a80185b --- /dev/null +++ b/backend/tests/routes/two-factor-auth.test.js @@ -0,0 +1,392 @@ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import express from 'express'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Signing in with a second factor. + * + * Wired the way the application wires it — the SQLite session store, then the + * auth middleware — because what is being pinned is what a half-finished + * sign-in can reach. A password that is right and a code that is missing must + * open nothing at all, and `/api/users/shareable` is the plainest route that + * says whether anything is open. + * + * Passwords are hashed with bcrypt at cost 12 throughout, hence the timeout. + */ + +const PASSWORD = 'secret123'; + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const build = async () => { + currentEnv = await setupTestEnv({ + tag: 'two-factor-auth-', + env: { AUTH_ENABLED: 'true', AUTH_MODE: 'local' }, + }); + const { configureSession } = currentEnv.requireFresh('src/middleware/session'); + const authMiddleware = currentEnv.requireFresh('src/middleware/authMiddleware'); + const authRoutes = currentEnv.requireFresh('src/routes/auth'); + const userRoutes = currentEnv.requireFresh('src/routes/users'); + const { errorHandler, notFoundHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const { totpCode } = currentEnv.requireFresh('src/utils/totp'); + + const app = express(); + app.use(express.json()); + app.use(cookieParser()); + configureSession(app); + app.use(authMiddleware); + app.use('/api/auth', authRoutes); + app.use('/api', userRoutes); + app.use(notFoundHandler); + app.use(errorHandler); + + return { app, totpCode }; +}; + +/** The first administrator, signed in through the setup. */ +const setUpOwner = async (app) => { + const browser = request.agent(app); + const response = await browser + .post('/api/auth/setup') + .send({ email: 'owner@example.com', username: 'owner', password: PASSWORD }); + expect(response.status).toBe(201); + return browser; +}; + +/** Whether this browser can reach something that needs an account. */ +const signedIn = async (browser) => (await browser.get('/api/users/shareable')).status === 200; + +/** Set up an authenticator on the signed-in account, and keep its secret. */ +const turnOn = async (browser, totpCode) => { + const started = await browser.post('/api/auth/totp/start').send({}); + expect(started.status).toBe(200); + const confirmed = await browser + .post('/api/auth/totp/confirm') + .send({ code: totpCode(started.body.secret) }); + expect(confirmed.status).toBe(200); + return { secret: started.body.secret, recoveryCodes: confirmed.body.recoveryCodes }; +}; + +describe('turning it on', { timeout: 30_000 }, () => { + it('shows a secret, and asks for a code before it counts', async () => { + const { app, totpCode } = await build(); + const browser = await setUpOwner(app); + + const started = await browser.post('/api/auth/totp/start').send({}); + + expect(started.body.secret).toMatch(/^[A-Z2-7]{32}$/); + expect(started.body.uri).toContain('otpauth://totp/'); + expect((await browser.get('/api/auth/totp')).body).toMatchObject({ + enabled: false, + pending: true, + }); + + const wrong = await browser.post('/api/auth/totp/confirm').send({ code: '000000' }); + expect(wrong.status).toBe(401); + expect(wrong.body.error.code).toBe('AUTH_INVALID_TOTP_CODE'); + expect((await browser.get('/api/auth/totp')).body.enabled).toBe(false); + + const right = await browser + .post('/api/auth/totp/confirm') + .send({ code: totpCode(started.body.secret) }); + + expect(right.status).toBe(200); + expect(right.body.recoveryCodes).toHaveLength(10); + expect((await browser.get('/api/auth/totp')).body).toMatchObject({ + enabled: true, + recoveryCodesLeft: 10, + }); + }); + + it('refuses to set one up for somebody who is not signed in', async () => { + const { app } = await build(); + await setUpOwner(app); + + const stranger = request.agent(app); + expect((await stranger.post('/api/auth/totp/start').send({})).status).toBe(401); + expect((await stranger.get('/api/auth/totp')).status).toBe(401); + }); +}); + +describe('signing in with it', { timeout: 30_000 }, () => { + it('opens nothing on the password alone', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + await turnOn(owner, totpCode); + + const browser = request.agent(app); + const first = await browser + .post('/api/auth/login') + .send({ identifier: 'owner', password: PASSWORD }); + + expect(first.status).toBe(200); + expect(first.body).toEqual({ totpRequired: true }); + expect(first.body.user).toBeUndefined(); + expect(await signedIn(browser)).toBe(false); + // A reload lands back on the code rather than on the password. + expect((await browser.get('/api/auth/status')).body.totpPending).toBe(true); + }); + + it('finishes on the code the phone shows', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + const { secret } = await turnOn(owner, totpCode); + + const browser = request.agent(app); + await browser.post('/api/auth/login').send({ identifier: 'owner', password: PASSWORD }); + const second = await browser + .post('/api/auth/login/totp') + .send({ code: totpCode(secret, { at: Date.now() + 30_000 }) }); + + expect(second.status).toBe(200); + expect(second.body.user.username).toBe('owner'); + expect(second.body.usedRecoveryCode).toBe(false); + expect(await signedIn(browser)).toBe(true); + expect((await browser.get('/api/auth/status')).body.totpPending).toBe(false); + }); + + it('refuses a wrong code and leaves the sign-in waiting', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + const { secret } = await turnOn(owner, totpCode); + + const browser = request.agent(app); + await browser.post('/api/auth/login').send({ identifier: 'owner', password: PASSWORD }); + + const refused = await browser.post('/api/auth/login/totp').send({ code: '000000' }); + expect(refused.status).toBe(401); + // Its own code, so a screen can say "that code" rather than "those + // credentials": the password was right, and saying otherwise sends + // somebody looking for the wrong mistake. + expect(refused.body.error.code).toBe('AUTH_INVALID_TOTP_CODE'); + expect(await signedIn(browser)).toBe(false); + + const second = await browser + .post('/api/auth/login/totp') + .send({ code: totpCode(secret, { at: Date.now() + 30_000 }) }); + expect(second.status).toBe(200); + }); + + /** Six digits are worth one sign-in, through the API as much as anywhere. */ + it('refuses the same code for a second sign-in', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + const { secret } = await turnOn(owner, totpCode); + const code = totpCode(secret, { at: Date.now() + 30_000 }); + + const first = request.agent(app); + await first.post('/api/auth/login').send({ identifier: 'owner', password: PASSWORD }); + expect((await first.post('/api/auth/login/totp').send({ code })).status).toBe(200); + + const second = request.agent(app); + await second.post('/api/auth/login').send({ identifier: 'owner', password: PASSWORD }); + expect((await second.post('/api/auth/login/totp').send({ code })).status).toBe(401); + expect(await signedIn(second)).toBe(false); + }); + + it('takes a recovery code, and says how many are left', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + const { recoveryCodes } = await turnOn(owner, totpCode); + + const browser = request.agent(app); + await browser.post('/api/auth/login').send({ identifier: 'owner', password: PASSWORD }); + const second = await browser.post('/api/auth/login/totp').send({ code: recoveryCodes[0] }); + + expect(second.status).toBe(200); + expect(second.body).toMatchObject({ usedRecoveryCode: true, recoveryCodesLeft: 9 }); + expect(await signedIn(browser)).toBe(true); + }); + + /** + * Nothing to finish: a code on its own is not a sign-in, and whose sign-in it + * would be is the server's to know. The body here tries to say — which is the + * whole of the attack, a correct code and a chosen account — and is ignored. + */ + it('refuses a code with no sign-in waiting for it, whoever the body names', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + const { secret } = await turnOn(owner, totpCode); + const me = (await owner.get('/api/auth/status')).body.user; + + const stranger = request.agent(app); + const response = await stranger.post('/api/auth/login/totp').send({ + // A code that would be accepted: a step later than the one the setup + // spent, so what refuses this is the missing sign-in and nothing else. + code: totpCode(secret, { at: Date.now() + 30_000 }), + userId: me.id, + identifier: 'owner', + email: 'owner@example.com', + }); + + expect(response.status).toBe(401); + expect(await signedIn(stranger)).toBe(false); + }); + + /** + * A machine walked away from, halfway through signing in, is not a sign-in + * waiting to be finished by whoever sits down next. + */ + it('lets the second step go stale, and asks for the password again', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + const { secret } = await turnOn(owner, totpCode); + + const browser = request.agent(app); + await browser.post('/api/auth/login').send({ identifier: 'owner', password: PASSWORD }); + + // Six minutes later, on a step of its own so the code itself is still good. + vi.useFakeTimers({ shouldAdvanceTime: true }); + try { + const later = Date.now() + 6 * 60 * 1000; + vi.setSystemTime(later); + const response = await browser + .post('/api/auth/login/totp') + .send({ code: totpCode(secret, { at: later }) }); + + expect(response.status).toBe(401); + expect(await signedIn(browser)).toBe(false); + expect((await browser.get('/api/auth/status')).body.totpPending).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + + it('leaves an account without one signing in as it always did', async () => { + const { app } = await build(); + await setUpOwner(app); + + const browser = request.agent(app); + const response = await browser + .post('/api/auth/login') + .send({ identifier: 'owner', password: PASSWORD }); + + expect(response.body.user.username).toBe('owner'); + expect(response.body.totpRequired).toBeUndefined(); + expect(await signedIn(browser)).toBe(true); + }); +}); + +describe('turning it off, and drawing new codes', { timeout: 30_000 }, () => { + it('asks for the password before taking it off', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + await turnOn(owner, totpCode); + + const refused = await owner.delete('/api/auth/totp').send({ password: 'not-the-password' }); + + expect(refused.status).toBe(401); + expect(refused.body.error.code).toBe('AUTH_PASSWORD_INCORRECT'); + expect((await owner.get('/api/auth/totp')).body.enabled).toBe(true); + + const accepted = await owner.delete('/api/auth/totp').send({ password: PASSWORD }); + + expect(accepted.status).toBe(204); + expect((await owner.get('/api/auth/totp')).body).toMatchObject({ + enabled: false, + recoveryCodesLeft: 0, + }); + }); + + it('asks for the password before drawing new recovery codes, and retires the old ones', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + const { recoveryCodes } = await turnOn(owner, totpCode); + + const refused = await owner.post('/api/auth/totp/recovery-codes').send({ password: 'wrong' }); + expect(refused.status).toBe(401); + + const fresh = await owner.post('/api/auth/totp/recovery-codes').send({ password: PASSWORD }); + expect(fresh.body.recoveryCodes).toHaveLength(10); + expect(fresh.body.recoveryCodes).not.toContain(recoveryCodes[0]); + + const browser = request.agent(app); + await browser.post('/api/auth/login').send({ identifier: 'owner', password: PASSWORD }); + expect( + (await browser.post('/api/auth/login/totp').send({ code: recoveryCodes[0] })).status + ).toBe(401); + }); + + /** Signing in again after it is off asks for nothing but the password. */ + it('goes back to one step once it is off', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + await turnOn(owner, totpCode); + await owner.delete('/api/auth/totp').send({ password: PASSWORD }); + + const browser = request.agent(app); + const response = await browser + .post('/api/auth/login') + .send({ identifier: 'owner', password: PASSWORD }); + + expect(response.body.user.username).toBe('owner'); + expect(await signedIn(browser)).toBe(true); + }); +}); + +/** + * The phone in the bag that was stolen with the printout. + * + * Somebody has to be able to take it off, and that somebody is whoever can + * already reset the account's password: an administrator. Anyone else asking + * is told no, and the account keeps asking for its code. + */ +describe('an administrator taking it off', { timeout: 30_000 }, () => { + const createRegular = async (owner) => + ( + await owner.post('/api/users').send({ + email: 'regular@example.com', + username: 'regular', + password: PASSWORD, + roles: ['user'], + }) + ).body.user; + + it('takes it off, and the account signs in with its password alone', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + const regular = await createRegular(owner); + + const theirs = request.agent(app); + await theirs.post('/api/auth/login').send({ identifier: 'regular', password: PASSWORD }); + await turnOn(theirs, totpCode); + + expect( + (await owner.get('/api/users')).body.users.find((u) => u.id === regular.id) + ).toMatchObject({ twoFactorEnabled: true }); + + expect((await owner.delete(`/api/users/${regular.id}/two-factor`)).status).toBe(204); + + const again = request.agent(app); + const response = await again + .post('/api/auth/login') + .send({ identifier: 'regular', password: PASSWORD }); + + expect(response.body.user.username).toBe('regular'); + expect(await signedIn(again)).toBe(true); + }); + + it('refuses anybody who is not an administrator', async () => { + const { app, totpCode } = await build(); + const owner = await setUpOwner(app); + const regular = await createRegular(owner); + + const theirs = request.agent(app); + await theirs.post('/api/auth/login').send({ identifier: 'regular', password: PASSWORD }); + await turnOn(theirs, totpCode); + + const response = await theirs.delete(`/api/users/${regular.id}/two-factor`); + + expect(response.status).toBe(403); + expect((await theirs.get('/api/auth/totp')).body.enabled).toBe(true); + }); +}); diff --git a/backend/tests/routes/upload-authorization.test.js b/backend/tests/routes/upload-authorization.test.js new file mode 100644 index 000000000..a416e2cad --- /dev/null +++ b/backend/tests/routes/upload-authorization.test.js @@ -0,0 +1,135 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Where an upload is allowed to land. The transfer itself is covered by the + * direct and tus suites; what had no test is the check that runs before either + * of them — a destination the caller may not write to, and a request that + * carries no file at all. + * + * The route reads `if (!allowed || !resolved)`, and no test can tell those two + * halves apart: `authorizeAndResolve` returns no resolved path whenever it + * refuses, so the second half never fires on its own. It is a guard against the + * service changing that contract, not a branch with a case behind it — worth + * knowing before spending an hour trying to reach it. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async (env = {}) => { + currentEnv = await setupTestEnv({ tag: 'upload-auth-', env }); + const dbService = currentEnv.requireFresh('src/services/db'); + const db = await dbService.getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('user-1','regular@example.com',1,'regular','Regular','["user"]', ?, ?)` + ).run(now, now); + await fs.mkdir(path.join(currentEnv.volumeDir, 'Inbox'), { recursive: true }); +}; + +const buildApp = (user) => { + const routes = currentEnv.requireFresh('src/routes/upload'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + if (user) req.user = user; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +const ADMIN = { id: 'admin-1', email: 'a@example.com', roles: ['admin'] }; +const RESTRICTED = { id: 'user-1', roles: [] }; + +describe('where a folder upload may start', () => { + it('reserves a destination the caller may write to', async () => { + await seed(); + + const response = await request(buildApp(ADMIN)) + .post('/api/upload/folder-session') + .send({ uploadTo: 'Inbox', sourceRoot: 'photos' }); + + expect(response.status).toBe(201); + expect(typeof response.body.targetRoot).toBe('string'); + }); + + it('refuses a destination the caller has no access to at all', async () => { + await seed({ USER_VOLUMES: 'true' }); + + const response = await request(buildApp(RESTRICTED)) + .post('/api/upload/folder-session') + .send({ uploadTo: 'Inbox', sourceRoot: 'photos' }); + + expect(response.status).toBe(403); + }); + + /** + * A read-only assignment is the case that separates the two halves of the + * check: the path resolves perfectly well, and only the permission says no. + * Without it, a test passes whether `allowed` is consulted or not. + */ + it('refuses a destination the caller may only read', async () => { + await seed({ USER_VOLUMES: 'true' }); + const dbService = currentEnv.requireFresh('src/services/db'); + const db = await dbService.getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO user_volumes (id, user_id, label, path, access_mode, created_at, updated_at) + VALUES ('uv-ro','user-1','Inbox', ?, 'readonly', ?, ?)` + ).run(path.join(currentEnv.volumeDir, 'Inbox'), now, now); + + const response = await request(buildApp(RESTRICTED)) + .post('/api/upload/folder-session') + .send({ uploadTo: 'Inbox', sourceRoot: 'photos' }); + + expect(response.status).toBe(403); + }); + + it('refuses a destination outside the volume', async () => { + await seed(); + + const response = await request(buildApp(ADMIN)) + .post('/api/upload/folder-session') + .send({ uploadTo: '../../etc', sourceRoot: 'photos' }); + + expect(response.status).not.toBe(201); + }); +}); + +describe('an upload with nothing in it', () => { + it('says so rather than answering as though it worked', async () => { + await seed(); + + const response = await request(buildApp(ADMIN)) + .post('/api/upload') + .query({ uploadTo: 'Inbox' }); + + expect(response.status).toBe(400); + }); +}); + +describe('what is still being written', () => { + it('answers with the caller’s own list, empty when there is nothing', async () => { + await seed(); + + const response = await request(buildApp(ADMIN)).get('/api/upload/finalizations'); + + expect(response.status).toBe(200); + expect(response.body.items).toEqual([]); + }); +}); diff --git a/backend/tests/routes/upload-landing.test.js b/backend/tests/routes/upload-landing.test.js new file mode 100644 index 000000000..aebaf2f45 --- /dev/null +++ b/backend/tests/routes/upload-landing.test.js @@ -0,0 +1,675 @@ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import http from 'node:http'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Where a direct upload ends up, and what it leaves when it does not finish. + * + * The destination a person picks is authorized once. Every file then carries a + * relative path the browser made up from the folder that was dropped, and that + * path decides where the bytes really go: into a subfolder an administrator + * hid or made read-only, into the zone that holds what people deleted, beside + * a file that already has the name, or up and out of the destination + * altogether. And an upload that dies half way — too large, cancelled, or a + * client that stopped sending — must not leave a truncated file presented as + * the real one, nor a `.uploading` remnant filling the disk. + * + * `direct-upload` covers the upload that works, the full volume and the sweep + * of old remnants; `upload-authorization` covers the chosen destination. + */ + +const fsp = require('fs/promises'); + +const ADMIN = { id: 'admin-1', roles: ['admin'] }; +const REGULAR = { id: 'user-1', roles: ['user'] }; +const BOUNDARY = 'upload-landing-boundary'; + +let envContext; +let server; + +afterEach(async () => { + vi.restoreAllMocks(); + if (server) { + server.closeAllConnections?.(); + await new Promise((resolve) => server.close(resolve)); + server = null; + } + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +const seed = async ({ env = {}, rules = [] } = {}) => { + envContext = await setupTestEnv({ tag: 'upload-landing-', env }); + if (rules.length) { + await envContext.requireFresh('src/services/accessControlService').setRules(rules); + } + const destination = path.join(envContext.volumeDir, 'Nvm'); + await fs.mkdir(destination, { recursive: true }); + return destination; +}; + +const buildApp = (user = ADMIN) => { + const routes = envContext.requireFresh('src/routes/upload'); + const { errorHandler } = envContext.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = user; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +const upload = (app, query, { name = 'file.txt', content = 'content' } = {}) => + request(app).post('/api/upload').query(query).attach('filedata', Buffer.from(content), name); + +const reason = (response) => response.body?.error?.message || response.text; + +const exists = (target) => + fs + .access(target) + .then(() => true) + .catch(() => false); + +/** Every entry under `dir`, relative to it. */ +const tree = async (dir) => { + const entries = await fs.readdir(dir, { recursive: true }); + return entries.map((entry) => entry.split(path.sep).join('/')).sort(); +}; + +/** The hidden files uploads are written through, in `dir`. */ +const temporaries = async (dir) => + (await fs.readdir(dir)).filter((name) => /^\.upload-[0-9a-f]{16}\.uploading$/.test(name)); + +const waitFor = async (predicate, timeoutMs = 3000) => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await predicate()) return true; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return false; +}; + +const partHead = (name) => + Buffer.from( + `--${BOUNDARY}\r\nContent-Disposition: form-data; name="filedata"; filename="${name}"\r\n` + + 'Content-Type: application/octet-stream\r\n\r\n' + ); + +const listen = async (app) => { + server = http.createServer(app); + await new Promise((resolve) => server.listen(0, resolve)); + return server.address().port; +}; + +/** + * Start an upload that announces a large file and sends only its beginning, + * leaving the connection open: what a browser tab looks like mid-transfer. + */ +const startPartialUpload = (port, name) => { + const req = http.request({ + host: '127.0.0.1', + port, + method: 'POST', + path: `/api/upload?uploadTo=Nvm&relativePath=${encodeURIComponent(name)}`, + headers: { + 'Content-Type': `multipart/form-data; boundary=${BOUNDARY}`, + 'Content-Length': String(64 * 1024 * 1024), + }, + }); + // The connection is cut on purpose; the error that follows is expected. + req.on('error', () => {}); + req.write(partHead(name)); + req.write(Buffer.alloc(256 * 1024, 7)); + return req; +}; + +describe('a name already taken', () => { + it('keeps the file that was there and gives the upload a numbered name', async () => { + const destination = await seed(); + await fs.writeFile(path.join(destination, 'report.txt'), 'the original'); + + const response = await upload( + buildApp(), + { uploadTo: 'Nvm', relativePath: 'report.txt' }, + { name: 'report.txt', content: 'the upload' } + ); + + expect(response.status).toBe(200); + expect(await fs.readFile(path.join(destination, 'report.txt'), 'utf8')).toBe('the original'); + expect(await fs.readFile(path.join(destination, 'report (1).txt'), 'utf8')).toBe('the upload'); + // What the listing and the person are told is the name it really took. + expect(response.body).toEqual([ + expect.objectContaining({ name: 'report (1).txt', path: 'Nvm' }), + ]); + }); + + /** + * The name used to be chosen before the transfer and taken by a rename after + * it, which replaces a file silently: whatever arrived under it in between — + * another upload, a copy, a file saved over SMB — was lost. A file is put + * under the name at the last moment, after every byte arrived and just + * before the upload takes the name. + */ + it('keeps a file that arrives under the name at the last moment', async () => { + const destination = await seed(); + const target = path.join(destination, 'report.txt'); + + let arrived = false; + const arriveFirst = (original) => + async function arriving(from, to) { + if (!arrived && String(from).endsWith('.uploading') && to === target) { + arrived = true; + await fs.writeFile(target, 'arrived meanwhile'); + } + return original.call(this, from, to); + }; + const { link, rename } = fsp; + vi.spyOn(fsp, 'link').mockImplementation(arriveFirst(link)); + vi.spyOn(fsp, 'rename').mockImplementation(arriveFirst(rename)); + + const response = await upload( + buildApp(), + { uploadTo: 'Nvm', relativePath: 'report.txt' }, + { name: 'report.txt', content: 'the upload' } + ); + + expect(arrived).toBe(true); + expect(response.status).toBe(200); + expect(await fs.readFile(target, 'utf8')).toBe('arrived meanwhile'); + expect(await fs.readFile(path.join(destination, 'report (1).txt'), 'utf8')).toBe('the upload'); + // Named and measured after the file it became, not the one that arrived. + expect(response.body).toEqual([ + expect.objectContaining({ name: 'report (1).txt', size: 'the upload'.length }), + ]); + expect(await tree(destination)).toEqual(['report (1).txt', 'report.txt']); + }); + + /** + * Two uploads of one name at once used to pick the same name, and so the + * same `name.uploading` temporary: each wrote into the other's file, and the + * rename of the first left the second nothing to rename. Both are sent half + * way before either finishes. + */ + it('gives two uploads of the same name at once a name each, with their own content', async () => { + const destination = await seed(); + const port = await listen(buildApp(ADMIN)); + + const send = (content) => { + const tail = Buffer.from(`\r\n--${BOUNDARY}--\r\n`); + const head = partHead('notes.txt'); + const body = Buffer.from(content); + const half = Math.floor(body.length / 2); + let resolveResponse; + const response = new Promise((resolve, reject) => { + resolveResponse = { resolve, reject }; + }); + const req = http.request( + { + host: '127.0.0.1', + port, + method: 'POST', + path: '/api/upload?uploadTo=Nvm&relativePath=notes.txt', + headers: { + 'Content-Type': `multipart/form-data; boundary=${BOUNDARY}`, + 'Content-Length': String(head.length + body.length + tail.length), + }, + }, + (res) => { + let text = ''; + res.setEncoding('utf8'); + res.on('data', (chunk) => { + text += chunk; + }); + res.on('end', () => resolveResponse.resolve({ status: res.statusCode, text })); + } + ); + req.on('error', (err) => resolveResponse.reject(err)); + req.write(Buffer.concat([head, body.subarray(0, half)])); + return { finish: () => req.end(Buffer.concat([body.subarray(half), tail])), response }; + }; + + const first = send('a'.repeat(128 * 1024)); + const second = send('b'.repeat(128 * 1024)); + // Both are writing before either finishes. The old code shared one + // temporary between them, so this only waits as long as it has to. + await waitFor(async () => (await temporaries(destination)).length === 2, 1500); + first.finish(); + second.finish(); + const answers = await Promise.all([first.response, second.response]); + + expect(answers.map((answer) => answer.status)).toEqual([200, 200]); + const told = answers.map((answer) => JSON.parse(answer.text)[0].name).sort(); + expect(told).toEqual(['notes (1).txt', 'notes.txt']); + expect(await tree(destination)).toEqual(['notes (1).txt', 'notes.txt']); + const contents = await Promise.all( + told.map((name) => fs.readFile(path.join(destination, name), 'utf8')) + ); + expect(contents.sort()).toEqual(['a'.repeat(128 * 1024), 'b'.repeat(128 * 1024)]); + }); +}); + +describe('the folder a relative path lands in', () => { + it('is refused when an administrator hid it, though the chosen destination is open', async () => { + const destination = await seed({ + rules: [{ path: 'Nvm/Secret', recursive: true, permissions: 'hidden' }], + }); + await fs.mkdir(path.join(destination, 'Secret')); + const app = buildApp(ADMIN); + + // Both ways a client names it: the path from the folder picker, and the + // one a folder session already resolved. + for (const query of [ + { uploadTo: 'Nvm', relativePath: 'Secret/planted.txt' }, + { uploadTo: 'Nvm', resolvedRelativePath: 'Secret/planted.txt' }, + ]) { + const response = await upload(app, query, { name: 'planted.txt' }); + + expect(response.status).toBe(403); + expect(reason(response)).toBe('Path is hidden'); + } + expect(await tree(path.join(destination, 'Secret'))).toEqual([]); + }); + + /** + * Read-only answers with the generic refusal, the same words the chosen + * destination would use. The upload into `Nvm` itself succeeding is what + * shows it was the subfolder that said no. + */ + it('is refused when it is read-only for someone who may write the destination', async () => { + const destination = await seed({ + rules: [{ path: 'Nvm/Archive', recursive: true, permissions: 'ro' }], + }); + await fs.mkdir(path.join(destination, 'Archive')); + const app = buildApp(REGULAR); + + const beside = await upload(app, { uploadTo: 'Nvm', relativePath: 'beside.txt' }); + const inside = await upload(app, { uploadTo: 'Nvm', relativePath: 'Archive/inside.txt' }); + + expect(beside.status).toBe(200); + expect(inside.status).toBe(403); + expect(reason(inside)).toBe('Cannot upload files to this path.'); + expect(await tree(path.join(destination, 'Archive'))).toEqual([]); + }); + + /** + * The zone sits inside every volume, so any folder upload into a volume's + * root is one crafted relative path away from it. A file planted there could + * pose as a deleted item, or as an earlier version of someone's document. + */ + it('is refused when it is the zone that holds deleted files', async () => { + const destination = await seed(); + await fs.writeFile(path.join(destination, 'deleted.txt'), 'deleted'); + await envContext + .requireFresh('src/services/trash/operations') + .moveToTrash({ absolutePath: path.join(destination, 'deleted.txt') }); + const zone = path.join(destination, '.nextexplorer'); + const before = await tree(zone); + expect(before).toContain('trash'); + const app = buildApp(ADMIN); + + for (const query of [ + { uploadTo: 'Nvm', relativePath: '.nextexplorer/trash/planted.txt' }, + { uploadTo: 'Nvm', resolvedRelativePath: '.nextexplorer/versions/planted.txt' }, + ]) { + const response = await upload(app, query, { name: 'planted.txt' }); + + expect(response.status).toBe(403); + expect(reason(response)).toMatch(/reserved by the application/); + } + expect(await tree(zone)).toEqual(before); + }); +}); + +/** + * The folder a picked folder is poured into is created by the server, before + * any file of the batch arrives: a folder session reserves it, and so does the + * first file of a batch that has no session. That reservation used to come + * before the folder was authorized, so a refusal left it behind, empty — and + * `.nextexplorer`, the zone's own name, left one that no path can reach again. + */ +describe('what a refused folder upload leaves', () => { + const startSession = (app, body) => request(app).post('/api/upload/folder-session').send(body); + + it('does not take the zone name for a folder session', async () => { + const destination = await seed(); + + const response = await startSession(buildApp(ADMIN), { + uploadTo: 'Nvm', + sourceRoot: '.nextexplorer', + }); + + expect(response.status).toBe(403); + expect(reason(response)).toMatch(/reserved by the application/); + expect(await tree(destination)).toEqual([]); + }); + + it('does not create the folder a session asks for where an administrator hid it', async () => { + const destination = await seed({ + rules: [{ path: 'Nvm/Secret', recursive: true, permissions: 'hidden' }], + }); + + const response = await startSession(buildApp(ADMIN), { uploadTo: 'Nvm', sourceRoot: 'Secret' }); + + expect(response.status).toBe(403); + expect(reason(response)).toBe('Path is hidden'); + expect(await tree(destination)).toEqual([]); + }); + + it('does not create the folder a session asks for where it is read-only', async () => { + const destination = await seed({ + rules: [{ path: 'Nvm/Archive', recursive: true, permissions: 'ro' }], + }); + + const response = await startSession(buildApp(REGULAR), { + uploadTo: 'Nvm', + sourceRoot: 'Archive', + }); + + expect(response.status).toBe(403); + expect(reason(response)).toBe('Cannot upload files to this path.'); + expect(await tree(destination)).toEqual([]); + }); + + /** + * A batch without a session reserves its folder on the first file. The batch + * id is what turns that reservation on, so it is the form that leaves + * something behind — the same request without one is refused with nothing + * created, and is covered above. + */ + it.each([ + ['the zone name', '.nextexplorer/trash/planted.txt', [], /reserved by the application/], + ['a folder an administrator hid', 'Secret/planted.txt', ['Nvm/Secret'], /hidden/], + ])('does not create %s for a batch of files', async (_label, relativePath, paths, refusal) => { + const destination = await seed({ + rules: paths.map((rulePath) => ({ + path: rulePath, + recursive: true, + permissions: 'hidden', + })), + }); + + const response = await upload( + buildApp(ADMIN), + { uploadTo: 'Nvm', relativePath, uploadBatchId: 'batch-refused-0001' }, + { name: 'planted.txt' } + ); + + expect(response.status).toBe(403); + expect(reason(response)).toMatch(refusal); + expect(await tree(destination)).toEqual([]); + }); +}); + +describe('a relative path that points elsewhere', () => { + it('cannot climb out of the chosen destination into a sibling folder', async () => { + const destination = await seed(); + await fs.mkdir(path.join(destination, 'Inbox')); + await fs.mkdir(path.join(destination, 'Other')); + + const app = buildApp(ADMIN); + + // The resolved form matters most: it skips the folder reservation, which + // would otherwise normalize the path a second time on the way through. + for (const query of [ + { uploadTo: 'Nvm/Inbox', relativePath: '../Other/moved.txt' }, + { uploadTo: 'Nvm/Inbox', resolvedRelativePath: '../Other/moved.txt' }, + ]) { + const response = await upload(app, query); + + expect(response.status).toBe(400); + expect(reason(response)).toMatch(/traversal/i); + } + expect(await tree(path.join(destination, 'Other'))).toEqual([]); + }); + + it('cannot climb out of the volume, whichever separator it is written with', async () => { + await seed(); + // What the application keeps in its own directories comes and goes as + // modules load; the question is only whether a file appeared anywhere else. + const foreign = async () => + (await tree(envContext.tmpRoot)).filter((entry) => !/^(config|cache)(\/|$)/.test(entry)); + const outside = await foreign(); + const app = buildApp(ADMIN); + + for (const climb of ['../../escaped.txt', '..\\..\\..\\escaped.txt']) { + for (const key of ['relativePath', 'resolvedRelativePath']) { + const response = await upload(app, { uploadTo: 'Nvm', [key]: climb }); + + expect(response.status).toBe(400); + expect(reason(response)).toMatch(/traversal/i); + } + } + expect(await foreign()).toEqual(outside); + }); + + it('takes an absolute path as one under the destination', async () => { + const destination = await seed(); + + const response = await upload(buildApp(ADMIN), { + uploadTo: 'Nvm', + relativePath: '/etc/hosts-copy.txt', + }); + + expect(response.status).toBe(200); + expect(await exists(path.join(destination, 'etc', 'hosts-copy.txt'))).toBe(true); + }); +}); + +describe('an upload that does not finish', () => { + /** + * The parser stops a file at the size limit by ending it early, so to the + * storage the truncated file looks complete: it was moved into place and + * removed a moment later, and nothing is left either way. What is watched + * here is that moment in between — the name a refused upload must never + * hold, since it is the name another upload may be asking for right then. + * + * A name is taken by the link or the rename that puts a file under it, so + * every one of those is recorded and the refused name must not be among + * them. `fs.open(target, 'wx')` on a filesystem without hard links is + * followed by the rename that is recorded here. + */ + it('never takes the name it asked for when it is larger than the limit', async () => { + const destination = await seed({ env: { MAX_DIRECT_UPLOAD_SIZE: '1K' } }); + const taken = []; + const record = (original, target) => + function taking(...args) { + taken.push(String(args[target])); + return original.apply(this, args); + }; + const { link, rename } = fsp; + vi.spyOn(fsp, 'link').mockImplementation(record(link, 1)); + vi.spyOn(fsp, 'rename').mockImplementation(record(rename, 1)); + + const response = await upload( + buildApp(ADMIN), + { uploadTo: 'Nvm', relativePath: 'large.bin' }, + { name: 'large.bin', content: 'x'.repeat(64 * 1024) } + ); + + // 413 naming the limit and the setting, not multer's "File too large" as a 500. + expect(response.status).toBe(413); + expect(reason(response)).toMatch(/larger than the 1 KB a direct upload accepts/); + expect(taken).not.toContain(path.join(destination, 'large.bin')); + expect(await tree(destination)).toEqual([]); + }); + + it('leaves nothing behind when the client goes away half way', async () => { + const destination = await seed(); + const port = await listen(buildApp(ADMIN)); + + const req = startPartialUpload(port, 'film.mkv'); + expect(await waitFor(async () => (await temporaries(destination)).length === 1)).toBe(true); + + req.destroy(); + + expect(await waitFor(async () => (await tree(destination)).length === 0)).toBe(true); + }); + + /** + * A client that stops sending without closing — a laptop lid, a network that + * drops silently — would otherwise hold its half-written file for as long + * as the connection lingers. The connection stays open for the whole of this + * test, so the cleanup cannot be the disconnect's doing. + */ + it('leaves nothing behind when the client stops sending and never hangs up', async () => { + const destination = await seed({ env: { UPLOAD_INACTIVITY_TIMEOUT: '300' } }); + const port = await listen(buildApp(ADMIN)); + + const req = startPartialUpload(port, 'stalled.bin'); + try { + expect(await waitFor(async () => (await temporaries(destination)).length === 1)).toBe(true); + + expect(await waitFor(async () => (await tree(destination)).length === 0)).toBe(true); + expect(req.destroyed).toBe(false); + } finally { + req.destroy(); + } + }); +}); + +describe('the room left for a request of several files', () => { + /** + * Room is measured once, against the whole request, before its first file. + * Measured again for the second file, the whole request would be weighed + * against the space left after the first had landed, and an upload that fits + * would be refused half way through. + */ + it('is not measured again for each file, which would refuse what fits', async () => { + const destination = await seed({ env: { UPLOAD_STORAGE_RESERVE: '0' } }); + // The parser only knows a file has ended once it sees the next boundary, so + // the first chunk carries that boundary and the second chunk the rest. + const secondHead = partHead('second.bin'); + const nextBoundary = Buffer.from(`--${BOUNDARY}\r\n`); + const firstChunk = Buffer.concat([ + partHead('first.bin'), + Buffer.alloc(4000, 1), + Buffer.from('\r\n'), + nextBoundary, + ]); + const rest = Buffer.concat([ + secondHead.subarray(nextBoundary.length), + Buffer.alloc(4000, 2), + Buffer.from(`\r\n--${BOUNDARY}--\r\n`), + ]); + const declared = firstChunk.length + rest.length; + + // A volume with just over the request's size free, which shrinks as files land. + const capacity = declared + 1000; + vi.spyOn(fsp, 'statfs').mockImplementation(async () => { + const names = await fs.readdir(destination); + const sizes = await Promise.all( + names.map((name) => fs.stat(path.join(destination, name)).then((stats) => stats.size)) + ); + const used = sizes.reduce((total, size) => total + size, 0); + return { bavail: Math.max(0, capacity - used), bsize: 1, blocks: 1_000_000 }; + }); + + const port = await listen(buildApp(ADMIN)); + const response = new Promise((resolve, reject) => { + const req = http.request( + { + host: '127.0.0.1', + port, + method: 'POST', + path: '/api/upload?uploadTo=Nvm', + headers: { + 'Content-Type': `multipart/form-data; boundary=${BOUNDARY}`, + 'Content-Length': String(declared), + }, + }, + (res) => { + res.resume(); + res.on('end', () => resolve(res.statusCode)); + } + ); + req.on('error', reject); + req.write(firstChunk); + // The second file is sent only once the first has landed, so the space + // it takes is already gone when the second one starts. + waitFor(() => exists(path.join(destination, 'first.bin'))).then((landed) => { + if (!landed) req.destroy(new Error('the first file never landed')); + else req.end(rest); + }); + }); + + expect(await response).toBe(200); + expect(await tree(destination)).toEqual(['first.bin', 'second.bin']); + }); +}); + +/** + * The folders an upload had to create, when it is then refused. + * + * The destination is authorized before anything is made, but a file carries a + * relative path of its own and the folders of that path are created after — + * so a refusal past that point left them behind: empty folders an upload + * invented, in somebody's tree, with nothing to say where they came from. The + * two refusals that reach that point are a volume that cannot hold what is + * coming, and a file over the size limit. + */ +describe('folders an upload created before it was refused', () => { + it('are taken back when the volume cannot hold the upload', async () => { + const destination = await seed({ env: { UPLOAD_STORAGE_RESERVE: '900T' } }); + + const response = await upload( + buildApp(), + { uploadTo: 'Nvm', relativePath: 'Invented/Deeper/report.txt' }, + { name: 'report.txt' } + ); + + expect(response.status).toBe(507); + expect(await tree(destination)).toEqual([]); + }); + + it('are taken back when the file is over the size limit', async () => { + const destination = await seed({ env: { MAX_DIRECT_UPLOAD_SIZE: '16' } }); + + const response = await upload( + buildApp(), + { uploadTo: 'Nvm', relativePath: 'Invented/Deeper/report.txt' }, + { name: 'report.txt', content: 'x'.repeat(4096) } + ); + + expect(response.status).toBe(413); + expect(await tree(destination)).toEqual([]); + }); + + /** + * Only what this upload created, and only while it is empty. A folder that + * was already there is not the upload's to remove, and one that has since + * been written into is somebody's — which is why `rmdir` refusing a folder + * that is not empty is the guard, rather than a look of our own that could + * be out of date by the time it is acted on. + */ + it('leaves a folder that was already there, and one that is not empty', async () => { + const destination = await seed({ env: { MAX_DIRECT_UPLOAD_SIZE: '4096' } }); + await fs.mkdir(path.join(destination, 'Existing'), { recursive: true }); + + const landed = await upload( + buildApp(), + { uploadTo: 'Nvm', relativePath: 'Existing/Kept/small.txt' }, + { name: 'small.txt', content: 'small enough' } + ); + expect(landed.status).toBe(200); + + const refused = await upload( + buildApp(), + { uploadTo: 'Nvm', relativePath: 'Existing/Kept/Invented/big.txt' }, + { name: 'big.txt', content: 'x'.repeat(8192) } + ); + + expect(refused.status).toBe(413); + expect(await tree(destination)).toEqual([ + 'Existing', + 'Existing/Kept', + 'Existing/Kept/small.txt', + ]); + }); +}); diff --git a/backend/tests/routes/upload-never-overwrite.test.js b/backend/tests/routes/upload-never-overwrite.test.js index a5e9bbe6b..db9030679 100644 --- a/backend/tests/routes/upload-never-overwrite.test.js +++ b/backend/tests/routes/upload-never-overwrite.test.js @@ -72,9 +72,8 @@ const temporaries = async (dir) => const waitFor = async (predicate, timeoutMs = 3000) => { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { - // eslint-disable-next-line no-await-in-loop if (await predicate()) return true; - // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, 20)); } return false; diff --git a/backend/tests/routes/usage-no-shell.test.js b/backend/tests/routes/usage-no-shell.test.js new file mode 100644 index 000000000..fc2628289 --- /dev/null +++ b/backend/tests/routes/usage-no-shell.test.js @@ -0,0 +1,83 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import request from 'supertest'; + +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * How full a volume is, asked of the kernel rather than of a shell. + * + * It used to be two commands with the path pasted into them — `du -sb ""` + * and `df -Pk ""` — and a folder name is not a shell string. A folder + * called `x";;echo "` closed the quote and left whatever followed for + * the shell to run, as the server's user, the moment somebody opened it. Any + * account that can make a folder could do it, and with sign-in switched off + * that is anybody at all. + * + * `fs.statfs` answers the same question with no shell and no subprocess, which + * is also why it is instant: `du` walked the whole tree to report a number the + * filesystem already had. + * + * The test makes such a folder, asks for its usage, and checks that what the + * name said to run did not run. It is written against the route because the + * route is where the path arrived. + */ + +describe('the usage of a folder whose name is shell syntax', () => { + let env; + let app; + let witness; + + const NAME = 'wedge";touch $WITNESS;echo "'; + + beforeEach(async () => { + env = await setupTestEnv({ tag: 'usage-no-shell-', modules: ['src/routes/usage'] }); + witness = path.join(env.cacheDir, 'ran'); + process.env.WITNESS = witness; + await fs.mkdir(path.join(env.volumeDir, NAME), { recursive: true }); + app = createTestApp({ + router: env.requireFresh('src/routes/usage'), + mountPath: '/api', + user: { id: 'u-1', roles: ['admin'] }, + }); + }); + + afterEach(async () => { + delete process.env.WITNESS; + await env.cleanup(); + }); + + const exists = async (file) => + fs + .access(file) + .then(() => true) + .catch(() => false); + + it('does not run what the name says to run', async () => { + expect(await exists(witness)).toBe(false); + + const response = await request(app).get(`/api/usage/${encodeURIComponent(NAME)}`); + + expect(response.status).toBe(200); + expect(await exists(witness)).toBe(false); + }); + + it('answers with the numbers the filesystem holds', async () => { + const response = await request(app).get('/api/usage/'); + + expect(response.status).toBe(200); + expect(response.body.total).toBeGreaterThan(0); + expect(response.body.free).toBeGreaterThan(0); + expect(response.body.used).toBe(response.body.total - response.body.free); + expect(response.body.percentUsed).toBeGreaterThanOrEqual(0); + expect(response.body.percentUsed).toBeLessThanOrEqual(100); + }); + + it('answers zeroes rather than failing when the path cannot be read', async () => { + const response = await request(app).get('/api/usage/nothing-here'); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ size: 0, free: 0, total: 0, percentUsed: 0 }); + }); +}); diff --git a/backend/tests/routes/usage.test.js b/backend/tests/routes/usage.test.js new file mode 100644 index 000000000..dff31494a --- /dev/null +++ b/backend/tests/routes/usage.test.js @@ -0,0 +1,42 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import request from 'supertest'; +import { setupTestEnv, createTestApp } from '../helpers/env-test-utils.js'; + +describe('Usage Routes', () => { + let env; + + afterEach(async () => { + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('returns filesystem usage for an accessible volume path', async () => { + env = await setupTestEnv({ + tag: 'usage-route-', + modules: ['src/routes/usage', 'src/services/accessManager', 'src/utils/pathUtils'], + }); + + await fs.mkdir(`${env.volumeDir}/TestVol`); + + const usageRoutes = env.requireFresh('src/routes/usage'); + const app = createTestApp({ + router: usageRoutes, + mountPath: '/api', + user: { id: 'admin-user', roles: ['admin'] }, + }); + + const response = await request(app).get('/api/usage/TestVol'); + + expect(response.status).toBe(200); + expect(response.body.path).toBe('TestVol'); + expect(response.body.total).toBeGreaterThan(0); + expect(response.body.free).toBeGreaterThanOrEqual(0); + expect(response.body.used).toBeGreaterThanOrEqual(0); + expect(response.body.size).toBe(response.body.used); + expect(response.body.percentUsed).toBeGreaterThanOrEqual(0); + expect(response.body.percentUsed).toBeLessThanOrEqual(100); + }); +}); diff --git a/backend/tests/routes/user-volumes-ownership.test.js b/backend/tests/routes/user-volumes-ownership.test.js new file mode 100644 index 000000000..9a5e815e6 --- /dev/null +++ b/backend/tests/routes/user-volumes-ownership.test.js @@ -0,0 +1,123 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Which account a volume belongs to, on the routes that change one. + * + * A volume is addressed twice in these URLs: by the account in the path and by + * its own id. Only the id is needed to find it, so a route that looked it up by + * id alone would let `/users/alice/volumes/` rename, re-mode or + * remove Bob's folder while the screen said it was editing Alice. The routes + * refuse that pairing as "not found", and these pin it — with the same calls + * made through the right account as the control, so a 404 cannot come from a + * wrong id instead. + * + * The admin and feature gates in front of all of this are pinned in + * `user-volumes.test.js`. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const ADMIN = { id: 'admin-1', roles: ['admin'] }; + +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'user-volumes-owner-', env: { USER_VOLUMES: 'true' } }); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + for (const id of ['alice', 'bob']) { + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, '["user"]', ?, ?)` + ).run(id, `${id}@example.com`, id, id, now, now); + } + const media = path.join(currentEnv.volumeDir, 'Media'); + await fs.mkdir(media, { recursive: true }); + + const routes = currentEnv.requireFresh('src/routes/userVolumes'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = ADMIN; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + + const created = await request(app) + .post('/api/users/bob/volumes') + .send({ label: 'Media', path: media, accessMode: 'readonly' }); + expect(created.status).toBe(201); + + return { app, db, media, bobVolume: created.body.volume }; +}; + +const volumesOf = async (app, userId) => + (await request(app).get(`/api/users/${userId}/volumes`)).body.volumes; + +describe('a volume reached through an account it does not belong to', () => { + it('is not changed', async () => { + const { app, bobVolume } = await seed(); + + const response = await request(app) + .patch(`/api/users/alice/volumes/${bobVolume.id}`) + .send({ label: 'Taken', accessMode: 'readwrite' }); + + expect(response.status).toBe(404); + expect(response.body.error.message).toBe('Volume not found.'); + expect(await volumesOf(app, 'bob')).toMatchObject([{ label: 'Media', accessMode: 'readonly' }]); + }); + + it('is not removed', async () => { + const { app, bobVolume } = await seed(); + + const response = await request(app).delete(`/api/users/alice/volumes/${bobVolume.id}`); + + expect(response.status).toBe(404); + expect(response.body.error.message).toBe('Volume not found.'); + expect((await volumesOf(app, 'bob')).map((v) => v.id)).toEqual([bobVolume.id]); + }); +}); + +describe('a volume reached through the account it belongs to', () => { + it('is changed, and then removed', async () => { + const { app, bobVolume } = await seed(); + + const patched = await request(app) + .patch(`/api/users/bob/volumes/${bobVolume.id}`) + .send({ label: 'Films', accessMode: 'readwrite' }); + expect(patched.status).toBe(200); + expect(patched.body.volume).toMatchObject({ label: 'Films', accessMode: 'readwrite' }); + + const removed = await request(app).delete(`/api/users/bob/volumes/${bobVolume.id}`); + expect(removed.status).toBe(204); + expect(await volumesOf(app, 'bob')).toEqual([]); + }); +}); + +describe('assigning a volume to an account that does not exist', () => { + it('is refused, and stores nothing', async () => { + const { app, db, media } = await seed(); + + const response = await request(app) + .post('/api/users/nobody/volumes') + .send({ label: 'Elsewhere', path: media }); + + expect(response.status).toBe(404); + expect(response.body.error.message).toBe('User not found.'); + expect( + db.prepare("SELECT COUNT(*) AS n FROM user_volumes WHERE user_id = 'nobody'").get().n + ).toBe(0); + }); +}); diff --git a/backend/tests/routes/user-volumes.test.js b/backend/tests/routes/user-volumes.test.js new file mode 100644 index 000000000..ac198a272 --- /dev/null +++ b/backend/tests/routes/user-volumes.test.js @@ -0,0 +1,174 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Assigning volumes to accounts. Every endpoint sits behind two gates — an + * administrator, and the feature actually being switched on — and the order + * matters: a regular account must be told no before it learns whether the + * feature exists. + * + * `/admin/browse-directories` reads the container's filesystem outside the + * volume root on purpose, because a volume may point anywhere the container can + * see. That is exactly why both gates are pinned here. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async (env = { USER_VOLUMES: 'true' }) => { + currentEnv = await setupTestEnv({ tag: 'user-volumes-', env }); + const dbService = currentEnv.requireFresh('src/services/db'); + const db = await dbService.getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('user-1','regular@example.com',1,'regular','Regular','["user"]', ?, ?)` + ).run(now, now); + await fs.mkdir(path.join(currentEnv.volumeDir, 'Media'), { recursive: true }); + return db; +}; + +const buildApp = (user) => { + const routes = currentEnv.requireFresh('src/routes/userVolumes'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + if (user) req.user = user; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +const ADMIN = { id: 'admin-1', roles: ['admin'] }; +const REGULAR = { id: 'user-1', roles: ['user'] }; + +const ENDPOINTS = [ + ['get', '/api/users/user-1/volumes', undefined], + ['post', '/api/users/user-1/volumes', { label: 'Media', path: '/tmp' }], + ['patch', '/api/users/user-1/volumes/anything', { accessMode: 'readonly' }], + ['delete', '/api/users/user-1/volumes/anything', undefined], + ['get', '/api/admin/browse-directories', undefined], +]; + +const call = (app, method, url, body) => { + const pending = request(app)[method](url); + return body ? pending.send(body) : pending; +}; + +describe('the two gates on assigning volumes', () => { + it.each(ENDPOINTS)('refuses a regular account on %s %s', async (method, url, body) => { + await seed(); + + const response = await call(buildApp(REGULAR), method, url, body); + + expect(response.status).toBe(403); + expect(response.body.error.message).toBe('Admin access required.'); + }); + + it.each(ENDPOINTS)( + 'refuses even an administrator when the feature is off, on %s %s', + async (method, url, body) => { + await seed({ USER_VOLUMES: 'false' }); + + const response = await call(buildApp(ADMIN), method, url, body); + + expect(response.status).toBe(403); + expect(response.body.error.message).toBe('User volumes feature is not enabled.'); + } + ); + + /** + * The role is checked first, so someone who is not an administrator cannot + * learn from the answer whether the feature is configured. + */ + it('tells a regular account about the role, never about the feature', async () => { + await seed({ USER_VOLUMES: 'false' }); + + const response = await request(buildApp(REGULAR)).get('/api/users/user-1/volumes'); + + expect(response.body.error.message).toBe('Admin access required.'); + }); +}); + +describe('assigning a volume', () => { + it('says not found for an account that does not exist', async () => { + await seed(); + + const response = await request(buildApp(ADMIN)).get('/api/users/nobody/volumes'); + + expect(response.status).toBe(404); + }); + + it('lists nothing for an account with no assignment', async () => { + await seed(); + + const response = await request(buildApp(ADMIN)).get('/api/users/user-1/volumes'); + + expect(response.status).toBe(200); + expect(response.body.volumes).toEqual([]); + }); + + it('gives back what it assigned', async () => { + await seed(); + const target = path.join(currentEnv.volumeDir, 'Media'); + + const created = await request(buildApp(ADMIN)) + .post('/api/users/user-1/volumes') + .send({ label: 'Media', path: target, accessMode: 'readonly' }); + + expect([200, 201]).toContain(created.status); + + const listed = await request(buildApp(ADMIN)).get('/api/users/user-1/volumes'); + expect(listed.body.volumes.map((v) => v.label)).toEqual(['Media']); + }); +}); + +describe('browsing for a folder to assign', () => { + it('lists only the directories it finds', async () => { + await seed(); + await fs.mkdir(path.join(currentEnv.volumeDir, 'Media', 'inner'), { recursive: true }); + await fs.writeFile(path.join(currentEnv.volumeDir, 'Media', 'file.txt'), 'x'); + + const response = await request(buildApp(ADMIN)) + .get('/api/admin/browse-directories') + .query({ path: path.join(currentEnv.volumeDir, 'Media') }); + + expect(response.status).toBe(200); + expect(response.body.directories.map((d) => d.name)).toEqual(['inner']); + }); + + it('says not found for a path that is not there', async () => { + await seed(); + + const response = await request(buildApp(ADMIN)) + .get('/api/admin/browse-directories') + .query({ path: '/definitely/not/here' }); + + expect(response.status).toBe(404); + }); + + it('refuses a file', async () => { + await seed(); + const file = path.join(currentEnv.volumeDir, 'Media', 'file.txt'); + await fs.writeFile(file, 'x'); + + const response = await request(buildApp(ADMIN)) + .get('/api/admin/browse-directories') + .query({ path: file }); + + expect(response.status).toBe(400); + }); +}); diff --git a/backend/tests/routes/users-admin-changes.test.js b/backend/tests/routes/users-admin-changes.test.js new file mode 100644 index 000000000..a076952e4 --- /dev/null +++ b/backend/tests/routes/users-admin-changes.test.js @@ -0,0 +1,140 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What an administrator changes on somebody else's account: the password, the + * email address, and the roles. + * + * Resetting a password is how a locked-out person gets back in, so a reset has + * to take effect — asserted by signing in, since a 204 alone says nothing + * about what was stored — and has to hold the same length rule as every other + * way a password is set. + * + * The email address is what somebody signs in with and what an identity + * provider's account is linked to by, so no two accounts may share one and no + * account may be left without. + * + * Who may reach these routes, and the rules that keep the last administrator, + * are pinned in `users.test.js`. + */ + +const PASSWORD = 'correct horse battery staple'; + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'users-admin-changes-' }); + const users = currentEnv.requireFresh('src/services/users'); + const admin = await users.createLocalUser({ + email: 'admin@example.com', + username: 'admin', + displayName: 'Admin', + password: PASSWORD, + roles: ['admin'], + }); + const regular = await users.createLocalUser({ + email: 'regular@example.com', + username: 'regular', + displayName: 'Regular', + password: PASSWORD, + roles: ['user'], + }); + + const routes = currentEnv.requireFresh('src/routes/users'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: admin.id, username: admin.username, roles: ['admin'] }; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + + return { app, users, admin, regular }; +}; + +const signsIn = async (users, identifier, password) => + Boolean(await users.attemptLocalLogin({ identifier, password })); + +describe("resetting somebody's password", { timeout: 30_000 }, () => { + it('replaces it: the old one stops signing in and the new one starts', async () => { + const { app, users, regular } = await seed(); + + const response = await request(app) + .post(`/api/users/${regular.id}/password`) + .send({ newPassword: 'another456' }); + + expect(response.status).toBe(204); + expect(await signsIn(users, 'regular', PASSWORD)).toBe(false); + expect(await signsIn(users, 'regular', 'another456')).toBe(true); + }); + + it('refuses a password under six characters, and the old one keeps working', async () => { + const { app, users, regular } = await seed(); + + const response = await request(app) + .post(`/api/users/${regular.id}/password`) + .send({ newPassword: '12345' }); + + expect(response.status).toBe(400); + expect(response.body.error.message).toBe('Password must be at least 6 characters long.'); + expect(await signsIn(users, 'regular', PASSWORD)).toBe(true); + expect(await signsIn(users, 'regular', '12345')).toBe(false); + }); +}); + +describe("changing somebody's email address", () => { + const emailOf = async (app, id) => + (await request(app).get('/api/users')).body.users.find((u) => u.id === id).email; + + /** Written differently, since addresses are compared the way they are stored. */ + it('refuses the address of another account, and keeps the one it had', async () => { + const { app, regular } = await seed(); + + const response = await request(app) + .patch(`/api/users/${regular.id}`) + .send({ email: ' ADMIN@example.com ' }); + + expect(response.status).toBe(409); + expect(response.body.error.message).toBe('Email already in use.'); + expect(await emailOf(app, regular.id)).toBe('regular@example.com'); + }); + + it('refuses to leave an account without one', async () => { + const { app, regular } = await seed(); + + const response = await request(app).patch(`/api/users/${regular.id}`).send({ email: ' ' }); + + expect(response.status).toBe(400); + expect(response.body.error.message).toBe('Email is required.'); + expect(await emailOf(app, regular.id)).toBe('regular@example.com'); + }); +}); + +describe("changing somebody's roles", () => { + /** + * The admin check everywhere else is an exact `includes('admin')`, so a role + * stored with its spaces, or a number in the list, would be a promotion that + * grants nothing — or a list another reader chokes on. + */ + it('promotes an account to administrator, keeping only real role names', async () => { + const { app, regular } = await seed(); + + const response = await request(app) + .patch(`/api/users/${regular.id}`) + .send({ roles: [' admin ', 42, '', 'user'] }); + + expect(response.status).toBe(200); + expect(response.body.user.roles).toEqual(['admin', 'user']); + }); +}); diff --git a/backend/tests/routes/users.test.js b/backend/tests/routes/users.test.js new file mode 100644 index 000000000..108b06e19 --- /dev/null +++ b/backend/tests/routes/users.test.js @@ -0,0 +1,320 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Account administration, and the three rules that keep an installation from + * locking its owner out: an administrator cannot be demoted, cannot delete + * themselves, and cannot be removed while they are the last one. Each is a + * single branch, and the route had no test of its own until now. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'users-route-' }); + const users = currentEnv.requireFresh('src/services/users'); + + const admin = await users.createLocalUser({ + email: 'admin@example.com', + username: 'admin', + displayName: 'Admin', + password: 'correct horse battery staple', + roles: ['admin'], + }); + const regular = await users.createLocalUser({ + email: 'regular@example.com', + username: 'regular', + displayName: 'Regular', + password: 'correct horse battery staple', + roles: ['user'], + }); + + return { users, admin, regular }; +}; + +const buildApp = (user) => { + const routes = currentEnv.requireFresh('src/routes/users'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + if (user) req.user = user; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +const asAdmin = (admin) => buildApp({ id: admin.id, username: admin.username, roles: ['admin'] }); +const asRegular = (regular) => + buildApp({ id: regular.id, username: regular.username, roles: ['user'] }); + +describe('who may administer accounts', () => { + it.each([ + ['get', '/api/users', undefined], + ['patch', '/api/users/someone', { roles: ['admin'] }], + ['post', '/api/users', { email: 'new@example.com', password: 'x' }], + ['post', '/api/users/someone/password', { newPassword: 'x' }], + ['delete', '/api/users/someone', undefined], + ])('refuses a regular account on %s %s', async (method, path, body) => { + const { regular } = await seed(); + + const call = request(asRegular(regular))[method](path); + const response = body ? await call.send(body) : await call; + + expect(response.status).toBe(403); + }); + + it.each([['/api/users/shareable'], ['/api/users/search?q=reg']])( + 'requires a signed-in account on %s', + async (path) => { + await seed(); + + const response = await request(buildApp(null)).get(path); + + expect(response.status).toBe(401); + } + ); +}); + +describe('the rules that keep an owner from locking themselves out', () => { + it('refuses to take the admin role away', async () => { + const { admin } = await seed(); + + const response = await request(asAdmin(admin)) + .patch(`/api/users/${admin.id}`) + .send({ roles: ['user'] }); + + expect(response.status).toBe(400); + expect(response.body.error.message).toBe('Demotion of admin is not allowed.'); + }); + + it('refuses to delete the account making the request', async () => { + const { admin } = await seed(); + + const response = await request(asAdmin(admin)).delete(`/api/users/${admin.id}`); + + expect(response.status).toBe(400); + expect(response.body.error.message).toBe('You cannot delete your own account.'); + }); + + /** + * The caller carries the admin role without being the local admin account — + * which is what an administrator elevated by their identity provider looks + * like. It is the only way to reach this rule: any caller who *is* the last + * local admin is stopped by the self-deletion rule first, so a test written + * that way passes whether this rule exists or not. + */ + it('refuses to remove the last local administrator', async () => { + const { admin } = await seed(); + const fromTheIdentityProvider = buildApp({ + id: 'oidc|someone-else', + username: 'federated', + roles: ['admin'], + }); + + const response = await request(fromTheIdentityProvider).delete(`/api/users/${admin.id}`); + + expect(response.status).toBe(400); + expect(response.body.error.message).toBe('Cannot remove the last admin.'); + }); + + it('allows removing an administrator while another one remains', async () => { + const { users, admin } = await seed(); + const secondAdmin = await users.createLocalUser({ + email: 'second@example.com', + username: 'second', + displayName: 'Second', + password: 'correct horse battery staple', + roles: ['admin'], + }); + + const response = await request(asAdmin(secondAdmin)).delete(`/api/users/${admin.id}`); + + expect(response.status).toBe(204); + }); + + it('lets a regular account be removed', async () => { + const { admin, regular } = await seed(); + + const response = await request(asAdmin(admin)).delete(`/api/users/${regular.id}`); + + expect(response.status).toBe(204); + }); +}); + +describe('reading and changing accounts', () => { + it('lists every account for an administrator', async () => { + const { admin } = await seed(); + + const response = await request(asAdmin(admin)).get('/api/users'); + + expect(response.status).toBe(200); + expect(response.body.users.map((u) => u.username).sort()).toEqual(['admin', 'regular']); + }); + + it('leaves the caller out of the list offered for sharing', async () => { + const { regular } = await seed(); + + const response = await request(asRegular(regular)).get('/api/users/shareable'); + + expect(response.status).toBe(200); + expect(response.body.users.map((u) => u.id)).not.toContain(regular.id); + }); + + it('says not found rather than failing, for an account that is not there', async () => { + const { admin } = await seed(); + + const patched = await request(asAdmin(admin)) + .patch('/api/users/nobody-at-all') + .send({ roles: ['user'] }); + const deleted = await request(asAdmin(admin)).delete('/api/users/nobody-at-all'); + + expect(patched.status).toBe(404); + expect(deleted.status).toBe(404); + }); + + it('creates an account and gives back what it made', async () => { + const { admin } = await seed(); + + const response = await request(asAdmin(admin)) + .post('/api/users') + .send({ email: 'new@example.com', password: 'correct horse battery staple' }); + + expect(response.status).toBe(201); + // The username falls back to the local part of the address. + expect(response.body.user).toMatchObject({ email: 'new@example.com', username: 'new' }); + }); + + it('never returns a password hash', async () => { + const { admin } = await seed(); + + const response = await request(asAdmin(admin)).get('/api/users'); + + const serialised = JSON.stringify(response.body); + expect(serialised).not.toMatch(/passwordHash|password_hash|\$2[aby]\$/); + }); +}); + +/** + * An account locked by failed sign-ins, seen from the administration screen. + * + * The lock frees itself after AUTH_LOCK_MINUTES and nothing else could free it: + * no list showed which accounts were locked, and releasing one meant deleting a + * row from auth_locks by hand. Asked for upstream in nxzai/NextExplorer#370. + * Locked here the way a person locks it — five wrong passwords — and checked + * released the way it matters: the right password signs in again. + */ +describe('an account locked by failed sign-ins', () => { + const PASSWORD = 'correct horse battery staple'; + + const lockOut = async (users, identifier) => { + for (let attempt = 0; attempt < 5; attempt += 1) { + await users.attemptLocalLogin({ identifier, password: 'wrong password' }); + } + }; + + const signInStatus = async (users, identifier, password = PASSWORD) => { + try { + return (await users.attemptLocalLogin({ identifier, password })) ? 'signed-in' : 'refused'; + } catch (error) { + return error.status === 423 ? 'locked' : `error ${error.message}`; + } + }; + + const listed = async (app, id) => { + const response = await request(app).get('/api/users'); + return response.body.users.find((user) => user.id === id); + }; + + it('shows in the list, with the moment it frees itself', async () => { + const { users, admin, regular } = await seed(); + await lockOut(users, 'regular'); + + const app = asAdmin(admin); + const locked = await listed(app, regular.id); + const other = await listed(app, admin.id); + + expect(Date.parse(locked.lockedUntil)).toBeGreaterThan(Date.now()); + expect(other.lockedUntil).toBeNull(); + }, 30_000); + + it('is no longer shown once the lock has run out', async () => { + const { users, admin, regular } = await seed(); + await lockOut(users, 'regular'); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + db.prepare('UPDATE auth_locks SET locked_until = ? WHERE key = ?').run( + new Date(Date.now() - 60_000).toISOString(), + regular.id + ); + + expect((await listed(asAdmin(admin), regular.id)).lockedUntil).toBeNull(); + }, 30_000); + + it('can be released by an administrator, after which the password signs in', async () => { + const { users, admin, regular } = await seed(); + await lockOut(users, 'regular'); + // The lock is real before it is released, or the release proves nothing. + expect(await signInStatus(users, 'regular')).toBe('locked'); + + const response = await request(asAdmin(admin)).delete(`/api/users/${regular.id}/lock`); + + expect(response.status).toBe(204); + expect(await signInStatus(users, 'regular')).toBe('signed-in'); + expect((await listed(asAdmin(admin), regular.id)).lockedUntil).toBeNull(); + }, 30_000); + + /** + * Releasing clears the count as well as the deadline. Clearing only the + * deadline would leave five failures on the books, and the very next typo + * would lock the account again. + */ + it('starts the count again, so one more typo does not lock it straight back', async () => { + const { users, admin, regular } = await seed(); + await lockOut(users, 'regular'); + + await request(asAdmin(admin)).delete(`/api/users/${regular.id}/lock`); + await users.attemptLocalLogin({ identifier: 'regular', password: 'one more typo' }); + + expect(await signInStatus(users, 'regular')).toBe('signed-in'); + }, 30_000); + + it('cannot be released by someone who is not an administrator', async () => { + const { users, admin, regular } = await seed(); + await lockOut(users, 'admin'); + + const response = await request(asRegular(regular)).delete(`/api/users/${admin.id}/lock`); + + expect(response.status).toBe(403); + expect(await signInStatus(users, 'admin')).toBe('locked'); + }, 30_000); + + it('answers not found for an account that does not exist', async () => { + const { admin } = await seed(); + + const response = await request(asAdmin(admin)).delete('/api/users/no-such-account/lock'); + + expect(response.status).toBe(404); + // The reason, not only the status: a route that did not exist answered 404 + // too, and this test passed before there was anything to test. + expect(response.body.error?.message).toMatch(/User not found/); + }); + + it('treats releasing an account that is not locked as done, not as an error', async () => { + const { admin, regular } = await seed(); + + const response = await request(asAdmin(admin)).delete(`/api/users/${regular.id}/lock`); + + expect(response.status).toBe(204); + }); +}); diff --git a/backend/tests/routes/version-files-admin.test.js b/backend/tests/routes/version-files-admin.test.js new file mode 100644 index 000000000..3939d62ef --- /dev/null +++ b/backend/tests/routes/version-files-admin.test.js @@ -0,0 +1,336 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import express from 'express'; +import request from 'supertest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Every file that has a history, for an administrator. + * + * The routes beside these answer about one file by its path, with that file's + * own rights. These answer "where has the space gone", which the panel could + * never answer: the histories worth finding include files that are no longer + * there, and a path nobody can name is a path nobody audits. + * + * What is checked here is the shape of that answer, and the two things that + * make it safe to expose at all — that only an administrator reaches it, and + * that deleting from it touches exactly the versions named and nothing else. + */ + +let envContext; +let users; +let app; +let db; + +const load = (relative) => require(modulePath(relative)); + +const volume = (...segments) => path.join(envContext.volumeDir, ...segments); + +const write = async (relative, content) => { + await fs.mkdir(path.dirname(volume(relative)), { recursive: true }); + await fs.writeFile(volume(relative), content); +}; + +const buildApp = () => { + const application = express(); + application.use(express.json()); + application.use((req, _res, next) => { + const who = req.get('x-test-user'); + if (who) req.user = users[who]; + // An automation credential, to check that carrying an administrator's + // account is not the same as being one. + if (req.get('x-test-token')) req.apiToken = { id: 'token-1', scope: 'write' }; + next(); + }); + application.use('/api', load('src/routes/editor')); + application.use('/api', load('src/routes/versionsAdmin')); + application.use('/api', load('src/routes/versions')); + application.use(load('src/middleware/errorHandler').errorHandler); + return application; +}; + +const as = (who) => ({ + get: (url) => request(app).get(url).set('x-test-user', who), + post: (url, body) => request(app).post(url).set('x-test-user', who).send(body), + put: (url, body) => request(app).put(url).set('x-test-user', who).send(body), +}); + +const list = (who = 'admin', query = '') => as(who).get(`/api/versions/admin/files${query}`); + +/** Save through the text editor, as `who`: the ordinary way a version appears. */ +const edit = async (who, filePath, content) => { + const response = await as(who).put('/api/editor', { path: filePath, content }); + expect(response.status).toBe(200); +}; + +const rowFor = (body, name) => body.files.find((file) => file.name === name); + +beforeEach(async () => { + envContext = await setupTestEnv({ tag: 'versions-admin-' }); + const usersService = load('src/services/users'); + const make = async (name, roles) => + usersService.createLocalUser({ + email: `${name}@example.com`, + username: name, + displayName: name[0].toUpperCase() + name.slice(1), + password: 'secret123', + roles, + }); + users = { + admin: await make('admin', ['admin']), + alice: await make('alice', ['user']), + }; + db = await load('src/services/db').getDb(); + app = buildApp(); + await write('Projects/notes.md', 'one\n'); + await write('Projects/report_2026.md', 'one\n'); + await write('Photos/holiday.txt', 'one\n'); +}); + +afterEach(async () => { + load('src/services/trash/maintenance').stop(); + vi.restoreAllMocks(); + await envContext.cleanup(); +}); + +describe('who may read the list', () => { + it('answers an administrator', async () => { + const response = await list('admin'); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ files: [], total: 0, totalBytes: 0 }); + }); + + it('refuses somebody who is not one', async () => { + const response = await list('alice'); + + expect(response.status).toBe(403); + }); + + it('refuses an API token, whoever it belongs to', async () => { + const response = await request(app) + .get('/api/versions/admin/files') + .set('x-test-user', 'admin') + .set('x-test-token', 'yes'); + + expect(response.status).toBe(403); + expect(response.body.error.message).toMatch(/API token/i); + }); + + it('refuses one on the delete route too, which is the one that destroys', async () => { + const response = await request(app) + .post('/api/versions/admin/files/anything/delete') + .set('x-test-user', 'admin') + .set('x-test-token', 'yes') + .send({ all: true }); + + expect(response.status).toBe(403); + }); +}); + +describe('the list itself', () => { + it('names every file that has versions, with what its history holds', async () => { + await edit('alice', 'Projects/notes.md', 'two\n'); + await edit('alice', 'Projects/notes.md', 'three\n'); + await edit('alice', 'Photos/holiday.txt', 'a much longer second content\n'); + + const response = await list(); + + expect(response.status).toBe(200); + expect(response.body.total).toBe(2); + expect(response.body.totalVersions).toBe(3); + // A zone is a top-level folder of the volume root, so `Projects` is the + // zone and the history's path inside it is the file name. The address a + // browser could open is the two put back together. + expect(rowFor(response.body, 'notes.md')).toMatchObject({ + name: 'notes.md', + relativePath: 'notes.md', + folder: '', + path: 'Projects/notes.md', + state: 'live', + versions: 2, + zone: { kind: 'volume', name: 'Projects' }, + }); + }); + + it('leaves out a file whose versions have all gone', async () => { + await edit('alice', 'Projects/notes.md', 'two\n'); + const before = await list(); + const file = rowFor(before.body, 'notes.md'); + + await as('admin').post(`/api/versions/admin/files/${file.id}/delete`, { all: true }); + + expect((await list()).body.files).toEqual([]); + }); + + it('orders by the space a history takes, which is the question being asked', async () => { + await edit('alice', 'Projects/notes.md', 'x\n'); + await edit('alice', 'Photos/holiday.txt', `${'y'.repeat(500)}\n`); + await edit('alice', 'Photos/holiday.txt', 'z\n'); + + const response = await list(); + + expect(response.body.files.map((file) => file.name)).toEqual(['holiday.txt', 'notes.md']); + expect(response.body.files[0].bytes).toBeGreaterThan(response.body.files[1].bytes); + }); + + it('can be ordered by path instead', async () => { + await edit('alice', 'Projects/notes.md', 'x\n'); + await edit('alice', 'Photos/holiday.txt', `${'y'.repeat(500)}\n`); + + const response = await list('admin', '?sort=path'); + + expect(response.body.files.map((file) => file.path)).toEqual([ + 'Photos/holiday.txt', + 'Projects/notes.md', + ]); + }); + + it('refuses an order it does not have, rather than quietly using another', async () => { + const response = await list('admin', '?sort=whatever'); + + expect(response.status).toBe(400); + }); + + it('searches the path as it was typed, underscore and all', async () => { + await edit('alice', 'Projects/notes.md', 'two\n'); + await edit('alice', 'Projects/report_2026.md', 'two\n'); + + // `report_2026` as a LIKE pattern would match `report-2026` and anything + // else with a character there. Nothing here should match but the one file. + const response = await list('admin', '?q=report_2026'); + + expect(response.body.files.map((file) => file.name)).toEqual(['report_2026.md']); + expect(response.body.total).toBe(1); + }); + + it('narrows to a state, and counts only what the filter matched', async () => { + await edit('alice', 'Projects/notes.md', 'two\n'); + await edit('alice', 'Photos/holiday.txt', 'two\n'); + const store = load('src/services/versions/store'); + const gone = rowFor((await list()).body, 'holiday.txt'); + store.setFileState(db, gone.id, 'orphaned', { orphanedAt: new Date().toISOString() }); + + const orphaned = await list('admin', '?state=orphaned'); + + expect(orphaned.body.files.map((file) => file.name)).toEqual(['holiday.txt']); + expect(orphaned.body.total).toBe(1); + expect((await list('admin', '?state=live')).body.total).toBe(1); + expect((await list()).body.total).toBe(2); + }); + + it('pages, and says how many there are altogether rather than how many it sent', async () => { + await edit('alice', 'Projects/notes.md', 'two\n'); + await edit('alice', 'Projects/report_2026.md', 'two\n'); + await edit('alice', 'Photos/holiday.txt', 'two\n'); + + const first = await list('admin', '?limit=2&sort=path'); + const second = await list('admin', '?limit=2&offset=2&sort=path'); + + expect(first.body.files).toHaveLength(2); + expect(first.body.total).toBe(3); + expect(second.body.files).toHaveLength(1); + expect(second.body.total).toBe(3); + expect(second.body.files[0].name).toBe('report_2026.md'); + }); +}); + +describe('one history', () => { + it('reads the versions of a history by its own id, newest first', async () => { + await edit('alice', 'Projects/notes.md', 'second content\n'); + await edit('alice', 'Projects/notes.md', 'third\n'); + const file = rowFor((await list()).body, 'notes.md'); + + const response = await as('admin').get(`/api/versions/admin/files/${file.id}`); + + expect(response.status).toBe(200); + expect(response.body.file).toMatchObject({ + name: 'notes.md', + path: 'Projects/notes.md', + state: 'live', + }); + expect(response.body.versions).toMatchObject([ + // What Alice replaced, so hers. + { size: 15, author: { id: users.alice.id, label: 'Alice' }, source: 'editor' }, + // What the file held before the application ever wrote it. + { size: 4, author: null, source: 'external' }, + ]); + expect(response.body.totalBytes).toBe(19); + }); + + it('answers an id nothing has with not found', async () => { + const response = await as('admin').get('/api/versions/admin/files/nothing'); + + expect(response.status).toBe(404); + }); +}); + +describe('deleting from it', () => { + it('deletes the versions named and leaves the others', async () => { + await edit('alice', 'Projects/notes.md', 'two\n'); + await edit('alice', 'Projects/notes.md', 'three\n'); + const file = rowFor((await list()).body, 'notes.md'); + const before = (await as('admin').get(`/api/versions/admin/files/${file.id}`)).body.versions; + + const response = await as('admin').post(`/api/versions/admin/files/${file.id}/delete`, { + ids: [before[0].id], + }); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ deleted: 1, remaining: 1 }); + const after = (await as('admin').get(`/api/versions/admin/files/${file.id}`)).body.versions; + expect(after.map((version) => version.id)).toEqual([before[1].id]); + // The file is a file, not a version of one. + expect(await fs.readFile(volume('Projects/notes.md'), 'utf8')).toBe('three\n'); + }); + + it('refuses a version that belongs to another file, and leaves it alone', async () => { + await edit('alice', 'Projects/notes.md', 'two\n'); + await edit('alice', 'Photos/holiday.txt', 'two\n'); + const mine = rowFor((await list()).body, 'notes.md'); + const other = rowFor((await list()).body, 'holiday.txt'); + const theirs = (await as('admin').get(`/api/versions/admin/files/${other.id}`)).body + .versions[0]; + + const response = await as('admin').post(`/api/versions/admin/files/${mine.id}/delete`, { + ids: [theirs.id], + }); + + expect(response.body.items).toEqual([{ id: theirs.id, status: 'not-found' }]); + expect(response.body.deleted).toBe(0); + const survivors = (await as('admin').get(`/api/versions/admin/files/${other.id}`)).body + .versions; + expect(survivors.map((version) => version.id)).toEqual([theirs.id]); + }); + + it('keeps the row of a file that is still there, because the next save reads it', async () => { + await edit('alice', 'Projects/notes.md', 'two\n'); + const file = rowFor((await list()).body, 'notes.md'); + + await as('admin').post(`/api/versions/admin/files/${file.id}/delete`, { all: true }); + + expect(load('src/services/versions/store').getFile(db, file.id)).not.toBeNull(); + }); + + it('takes the row with it when the file itself is gone', async () => { + await edit('alice', 'Projects/notes.md', 'two\n'); + const store = load('src/services/versions/store'); + const file = rowFor((await list()).body, 'notes.md'); + store.setFileState(db, file.id, 'orphaned', { orphanedAt: new Date().toISOString() }); + + await as('admin').post(`/api/versions/admin/files/${file.id}/delete`, { all: true }); + + expect(store.getFile(db, file.id)).toBeNull(); + }); + + it('refuses a request that names nothing at all', async () => { + await edit('alice', 'Projects/notes.md', 'two\n'); + const file = rowFor((await list()).body, 'notes.md'); + + const response = await as('admin').post(`/api/versions/admin/files/${file.id}/delete`, {}); + + expect(response.status).toBe(400); + }); +}); diff --git a/backend/tests/routes/versions-browse-access.test.js b/backend/tests/routes/versions-browse-access.test.js new file mode 100644 index 000000000..bfea0f593 --- /dev/null +++ b/backend/tests/routes/versions-browse-access.test.js @@ -0,0 +1,91 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import express from 'express'; +import request from 'supertest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Whether a listing says its files show their history, which is what decides + * whether the Versions entry appears in the menu: always for a place someone + * may read, and through a share only when its owner turned it on. + */ + +let envContext; +let users; +let app; + +const load = (relative) => require(modulePath(relative)); + +beforeEach(async () => { + envContext = await setupTestEnv({ tag: 'versions-browse-', env: { SHARES_ENABLED: 'true' } }); + users = { + alice: await load('src/services/users').createLocalUser({ + email: 'alice@example.com', + username: 'alice', + displayName: 'Alice', + password: 'secret123', + roles: ['user'], + }), + }; + await fs.mkdir(path.join(envContext.volumeDir, 'Projects'), { recursive: true }); + await fs.writeFile(path.join(envContext.volumeDir, 'Projects', 'notes.md'), '# notes\n'); + + app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + const who = req.get('x-test-user'); + if (who?.startsWith('guest:')) { + req.guestSession = { id: 'guest-session', shareId: who.slice('guest:'.length) }; + } else if (who) { + req.user = users[who]; + } + next(); + }); + app.use('/api', load('src/routes/browse')); + app.use('/api/shares', load('src/routes/shares')); + app.use('/api/share', load('src/routes/shares')); + app.use(load('src/middleware/errorHandler').errorHandler); +}); + +afterEach(async () => { + await envContext.cleanup(); +}); + +describe('what a listing says about file histories', () => { + it('shows them in a place someone may read', async () => { + const response = await request(app).get('/api/browse/Projects').set('x-test-user', 'alice'); + + expect(response.status).toBe(200); + expect(response.body.access.canSeeVersions).toBe(true); + }); + + it('shows them through a share only once its owner turns them on', async () => { + const folder = await request(app) + .post('/api/shares') + .set('x-test-user', 'alice') + .send({ sourcePath: 'Projects', sharingType: 'anyone' }); + const file = await request(app) + .post('/api/shares') + .set('x-test-user', 'alice') + .send({ sourcePath: 'Projects/notes.md', sharingType: 'anyone' }); + const browseAs = (share) => + request(app) + .get(`/api/share/${share.shareToken}/browse/`) + .set('x-test-user', `guest:${share.id}`); + + expect((await browseAs(folder.body)).body.access.canSeeVersions).toBe(false); + expect((await browseAs(file.body)).body.access.canSeeVersions).toBe(false); + + for (const share of [folder.body, file.body]) { + await request(app) + .put(`/api/shares/${share.id}`) + .set('x-test-user', 'alice') + .send({ versionsVisible: true }); + } + + expect((await browseAs(folder.body)).body.access.canSeeVersions).toBe(true); + expect((await browseAs(file.body)).body.access.canSeeVersions).toBe(true); + }); +}); diff --git a/backend/tests/routes/versions-editor-capture.test.js b/backend/tests/routes/versions-editor-capture.test.js new file mode 100644 index 000000000..eacdf9a81 --- /dev/null +++ b/backend/tests/routes/versions-editor-capture.test.js @@ -0,0 +1,174 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import cookieParser from 'cookie-parser'; +import express from 'express'; +import request from 'supertest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { createTestApp, modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What the text editor's saves leave in a file's history — from inside the + * application and through a share link. + * + * Both used to write over the file in place: a crash in the middle of a save + * left it truncated, and nothing of what it held before was kept. They now + * write beside it and rename, and every save keeps what it replaced. + */ + +let env; + +const load = (relative) => require(modulePath(relative)); + +beforeEach(async () => { + env = await setupTestEnv({ tag: 'versions-editor-', env: { USER_VOLUMES: 'true' } }); + await fs.mkdir(path.join(env.volumeDir, 'Projects'), { recursive: true }); +}); + +afterEach(async () => { + load('src/services/trash/maintenance').stop(); + await env.cleanup(); +}); + +const volume = (...segments) => path.join(env.volumeDir, ...segments); + +/** The versions kept for a file under `root`, newest first, with what each holds. */ +const versionsUnder = async (root, inside) => { + const db = await load('src/services/db').getDb(); + const zones = load('src/services/trash/zones'); + const trashStore = load('src/services/trash/store'); + const store = load('src/services/versions/store'); + const zone = trashStore.listZones(db).find((candidate) => candidate.root === root); + const file = zone ? store.findFileAt(db, zone.id, inside) : null; + if (!file) return []; + return Promise.all( + store.listVersionsOfFile(db, file.id).map(async (version) => ({ + ...version, + content: await fs.readFile(path.join(zones.versionsDirectory(zone.root), version.id), 'utf8'), + })) + ); +}; + +describe('the text editor', () => { + const app = () => + createTestApp({ + router: load('src/routes/editor'), + mountPath: '/api', + user: { id: 'user-1', username: 'alice', displayName: 'Alice', roles: ['admin'] }, + errorHandler: load('src/middleware/errorHandler').errorHandler, + }); + + it('keeps what every save replaces, credited to whoever wrote it', async () => { + await fs.writeFile(volume('Projects/notes.md'), '# Draft\n'); + const application = app(); + + for (const content of ['# Second\n', '# Third\n']) { + const response = await request(application) + .put('/api/editor') + .send({ path: 'Projects/notes.md', content }); + expect(response.status).toBe(200); + } + + expect(await fs.readFile(volume('Projects/notes.md'), 'utf8')).toBe('# Third\n'); + const [newest, oldest] = await versionsUnder(volume('Projects'), 'notes.md'); + expect(newest).toMatchObject({ + content: '# Second\n', + authorId: 'user-1', + authorLabel: 'Alice', + source: 'editor', + }); + expect(oldest).toMatchObject({ content: '# Draft\n', source: 'external' }); + }); + + it('replaces the file in one rename, keeping its permissions and leaving nothing beside it', async () => { + await fs.writeFile(volume('Projects/run.sh'), 'echo one\n'); + await fs.chmod(volume('Projects/run.sh'), 0o750); + const before = await fs.stat(volume('Projects/run.sh')); + + await request(app()) + .put('/api/editor') + .send({ path: 'Projects/run.sh', content: 'echo two\n' }); + + const after = await fs.stat(volume('Projects/run.sh')); + expect(after.ino).not.toBe(before.ino); + expect(after.mode & 0o777).toBe(0o750); + // Beside the file, only the zone the versions live in, which no listing shows. + expect(await fs.readdir(volume('Projects'))).toEqual(['.nextexplorer', 'run.sh']); + }); + + it('creates a new file without a history', async () => { + const response = await request(app()) + .put('/api/editor') + .send({ path: 'Projects/new.txt', content: 'hello' }); + + expect(response.status).toBe(200); + expect(await fs.readFile(volume('Projects/new.txt'), 'utf8')).toBe('hello'); + expect(await versionsUnder(volume('Projects'), 'new.txt')).toEqual([]); + }); +}); + +describe('the text editor through a share link', () => { + const buildApp = ({ user } = {}) => { + const application = express(); + application.use(express.json()); + application.use(cookieParser()); + application.use((req, _res, next) => { + req.session = user ? { localUserId: user.id } : {}; + next(); + }); + application.use(load('src/middleware/authMiddleware')); + application.use('/api/shares', load('src/routes/shares')); + application.use('/api/share', load('src/routes/shares')); + application.use(load('src/middleware/errorHandler').errorHandler); + return application; + }; + + it('keeps what a visitor replaces, for the owner to find', async () => { + const assignedRoot = path.join(env.tmpRoot, 'assigned'); + await fs.mkdir(assignedRoot, { recursive: true }); + await fs.writeFile(path.join(assignedRoot, 'minutes.txt'), 'as written by the owner'); + const owner = await load('src/services/users').createLocalUser({ + email: 'owner@example.com', + username: 'owner', + displayName: 'Owner', + password: 'secret123', + roles: ['user'], + }); + await load('src/services/userVolumesService').addVolumeToUser({ + userId: owner.id, + label: 'Assigned', + volumePath: assignedRoot, + accessMode: 'readwrite', + }); + const create = await request(buildApp({ user: owner })) + .post('/api/shares') + .send({ + sourcePath: 'Assigned/minutes.txt', + accessMode: 'readwrite', + sharingType: 'anyone', + }); + expect(create.status).toBe(201); + + const save = await request(buildApp()) + .put(`/api/share/${create.body.shareToken}/editor`) + .send({ content: 'amended by a visitor' }); + + expect(save.status).toBe(200); + expect(await fs.readFile(path.join(assignedRoot, 'minutes.txt'), 'utf8')).toBe( + 'amended by a visitor' + ); + const [version] = await versionsUnder(assignedRoot, 'minutes.txt'); + expect(version).toMatchObject({ content: 'as written by the owner', source: 'external' }); + + await request(buildApp()) + .put(`/api/share/${create.body.shareToken}/editor`) + .send({ content: 'amended again' }); + const [newest] = await versionsUnder(assignedRoot, 'minutes.txt'); + expect(newest).toMatchObject({ + content: 'amended by a visitor', + authorId: null, + authorLabel: 'share-link', + source: 'share-editor', + }); + }); +}); diff --git a/backend/tests/routes/versions-office-capture.test.js b/backend/tests/routes/versions-office-capture.test.js new file mode 100644 index 000000000..4202d953e --- /dev/null +++ b/backend/tests/routes/versions-office-capture.test.js @@ -0,0 +1,325 @@ +import fs from 'node:fs/promises'; +import http from 'node:http'; +import path from 'node:path'; +import jwt from 'jsonwebtoken'; +import request from 'supertest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { createTestApp, modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What an office editor's saves leave in a document's history. + * + * ONLYOFFICE and Collabora save on their own every few seconds while someone + * types. Kept one by one, those saves would fill a volume with near copies of + * the same document — which is what Nextcloud does with Collabora's. What is + * kept is the document as it was before the session, a save someone asked for, + * and the state a previous session left; and a save from an editor that was + * open before a restore never undoes it. + */ + +const ONLYOFFICE_SECRET = 'onlyoffice-versions-secret'; +const COLLABORA_SECRET = 'collabora-versions-secret'; + +let env; +let documentServer; +let port; +const served = new Map(); + +const load = (relative) => require(modulePath(relative)); + +beforeEach(async () => { + served.clear(); + documentServer = http.createServer((req, res) => { + if (!served.has(req.url)) { + res.statusCode = 404; + res.end(); + return; + } + res.setHeader('Content-Type', 'application/octet-stream'); + res.end(served.get(req.url)); + }); + await new Promise((resolve) => documentServer.listen(0, '127.0.0.1', resolve)); + port = documentServer.address().port; + + env = await setupTestEnv({ + tag: 'versions-office-', + env: { + PUBLIC_URL: 'https://files.example.com', + ONLYOFFICE_URL: `http://127.0.0.1:${port}`, + ONLYOFFICE_SECRET, + COLLABORA_URL: 'https://collabora.example.com', + COLLABORA_SECRET, + }, + }); + await fs.mkdir(path.join(env.volumeDir, 'Projects'), { recursive: true }); +}); + +afterEach(async () => { + load('src/services/trash/maintenance').stop(); + await new Promise((resolve) => documentServer.close(resolve)); + await env.cleanup(); +}); + +const volume = (...segments) => path.join(env.volumeDir, ...segments); + +/** A document's versions, newest first, with what each one holds. */ +const versionsOf = async (relative) => { + const db = await load('src/services/db').getDb(); + const zones = load('src/services/trash/zones'); + const trashStore = load('src/services/trash/store'); + const store = load('src/services/versions/store'); + const [volumeName, ...inside] = relative.split('/'); + const zone = trashStore.listZones(db).find((candidate) => candidate.root === volume(volumeName)); + const file = zone ? store.findFileAt(db, zone.id, inside.join('/')) : null; + if (!file) return []; + return Promise.all( + store.listVersionsOfFile(db, file.id).map(async (version) => ({ + ...version, + content: await fs.readFile(path.join(zones.versionsDirectory(zone.root), version.id), 'utf8'), + })) + ); +}; + +const markRestored = async (relative, restoredAt) => { + const db = await load('src/services/db').getDb(); + const trashStore = load('src/services/trash/store'); + const store = load('src/services/versions/store'); + const [volumeName, ...inside] = relative.split('/'); + const zone = trashStore.listZones(db).find((candidate) => candidate.root === volume(volumeName)); + const file = store.findFileAt(db, zone.id, inside.join('/')); + store.setRestoredAt(db, file.id, restoredAt); +}; + +describe('ONLYOFFICE saves', () => { + const app = () => + createTestApp({ + router: load('src/routes/onlyoffice'), + mountPath: '/api', + user: { id: 'user-1', username: 'alice', roles: ['admin'] }, + errorHandler: load('src/middleware/errorHandler').errorHandler, + }); + + let counter = 0; + const save = async (application, relative, content, body = {}, query = '') => { + counter += 1; + const url = `/document-${counter}.docx`; + served.set(url, content); + const response = await request(application) + .post(`/api/onlyoffice/callback?path=${encodeURIComponent(relative)}${query}`) + .set('Authorization', `Bearer ${jwt.sign({ callback: true }, ONLYOFFICE_SECRET)}`) + .send({ + status: 6, + forcesavetype: 0, + key: 'session-1', + url: `http://127.0.0.1:${port}${url}`, + ...body, + }); + expect(response.body).toEqual({ error: 0 }); + return response; + }; + + it('keeps the document as it was before the session, and not every automatic save', async () => { + await fs.writeFile(volume('Projects/offer.docx'), 'before'); + const application = app(); + + await save(application, 'Projects/offer.docx', 'autosave one'); + await save(application, 'Projects/offer.docx', 'autosave two'); + await save(application, 'Projects/offer.docx', 'autosave three'); + + expect(await fs.readFile(volume('Projects/offer.docx'), 'utf8')).toBe('autosave three'); + expect((await versionsOf('Projects/offer.docx')).map((version) => version.content)).toEqual([ + 'before', + ]); + }); + + it('keeps what the editor’s own Save button saved once the session saves over it', async () => { + await fs.writeFile(volume('Projects/offer.docx'), 'before'); + const application = app(); + + await save(application, 'Projects/offer.docx', 'typing'); + await save(application, 'Projects/offer.docx', 'saved on purpose', { forcesavetype: 1 }); + await save(application, 'Projects/offer.docx', 'typing again'); + + expect((await versionsOf('Projects/offer.docx')).map((version) => version.content)).toEqual([ + 'saved on purpose', + 'before', + ]); + }); + + it('keeps the state a finished session left, credited to whoever made the last change', async () => { + await fs.writeFile(volume('Projects/offer.docx'), 'before'); + const application = app(); + + await save(application, 'Projects/offer.docx', 'final by bob', { + status: 2, + key: 'monday', + history: { changes: [{ user: { id: 'user-2', name: 'Bob' } }] }, + }); + await save(application, 'Projects/offer.docx', 'tuesday', { key: 'tuesday' }); + + const [newest, oldest] = await versionsOf('Projects/offer.docx'); + expect(newest).toMatchObject({ + content: 'final by bob', + authorId: 'user-2', + authorLabel: 'Bob', + source: 'onlyoffice', + }); + expect(oldest.content).toBe('before'); + }); + + it('sets aside a save from an editor opened before the document was restored', async () => { + await fs.writeFile(volume('Projects/offer.docx'), 'before'); + const application = app(); + await save(application, 'Projects/offer.docx', 'restored', { + key: 'new-session', + forcesavetype: 1, + }); + await markRestored('Projects/offer.docx', new Date().toISOString()); + const backend = jwt.sign( + { + typ: 'nextexplorer-backend', + absolutePath: volume('Projects/offer.docx'), + logicalPath: 'Projects/offer.docx', + canWrite: true, + userId: 'user-3', + iat: Math.floor(Date.now() / 1000) - 120, + }, + ONLYOFFICE_SECRET + ); + + await save( + application, + 'Projects/offer.docx', + 'stale editor content', + { key: 'old-session' }, + `&backend=${encodeURIComponent(backend)}` + ); + + expect(await fs.readFile(volume('Projects/offer.docx'), 'utf8')).toBe('restored'); + const [aside] = await versionsOf('Projects/offer.docx'); + expect(aside).toMatchObject({ + content: 'stale editor content', + aside: true, + authorId: 'user-3', + }); + }); + + it('leaves the document whole when the download fails half way', async () => { + await fs.writeFile(volume('Projects/offer.docx'), 'before'); + const application = app(); + + const response = await request(application) + .post('/api/onlyoffice/callback?path=Projects%2Foffer.docx') + .set('Authorization', `Bearer ${jwt.sign({ callback: true }, ONLYOFFICE_SECRET)}`) + .send({ status: 6, key: 'k', url: `http://127.0.0.1:${port}/missing.docx` }); + + expect(response.body).toEqual({ error: 1 }); + expect(await fs.readFile(volume('Projects/offer.docx'), 'utf8')).toBe('before'); + expect((await fs.readdir(volume('Projects'))).filter((name) => name.endsWith('.tmp'))).toEqual( + [] + ); + }); +}); + +describe('Collabora saves', () => { + const app = () => + createTestApp({ + router: load('src/routes/collabora'), + mountPath: '/api', + errorHandler: load('src/middleware/errorHandler').errorHandler, + }); + + const token = (extra = {}) => + jwt.sign( + { + typ: 'nextexplorer-wopi', + fileId: 'file-1', + absolutePath: volume('Projects/plan.odt'), + canWrite: true, + userId: 'user-1', + userName: 'Alice', + ...extra, + }, + COLLABORA_SECRET + ); + + const put = ( + application, + content, + { lock = 'lock-1', autosave = true, accessToken = token() } = {} + ) => + request(application) + .post('/api/collabora/wopi/files/file-1/contents') + .query({ access_token: accessToken }) + .set('Content-Type', 'application/octet-stream') + .set('X-WOPI-Lock', lock) + .set('X-COOL-WOPI-IsAutosave', autosave ? 'true' : 'false') + .send(Buffer.from(content)); + + it('keeps the document as it was before the session, and not every automatic save', async () => { + await fs.writeFile(volume('Projects/plan.odt'), 'before'); + const application = app(); + + for (const content of ['autosave one', 'autosave two', 'autosave three']) { + expect((await put(application, content)).status).toBe(200); + } + + expect(await fs.readFile(volume('Projects/plan.odt'), 'utf8')).toBe('autosave three'); + expect((await versionsOf('Projects/plan.odt')).map((version) => version.content)).toEqual([ + 'before', + ]); + }); + + it('keeps a save someone asked for once the session saves over it', async () => { + await fs.writeFile(volume('Projects/plan.odt'), 'before'); + const application = app(); + + await put(application, 'typing'); + await put(application, 'saved on purpose', { autosave: false }); + await put(application, 'typing again'); + + expect((await versionsOf('Projects/plan.odt')).map((version) => version.content)).toEqual([ + 'saved on purpose', + 'before', + ]); + }); + + it('credits the state a session left to whoever was editing, for the next session', async () => { + await fs.writeFile(volume('Projects/plan.odt'), 'before'); + const application = app(); + + await put(application, 'alice was here', { lock: 'monday' }); + await put(application, 'bob now', { + lock: 'tuesday', + accessToken: token({ userId: 'user-2', userName: 'Bob' }), + }); + + const [newest] = await versionsOf('Projects/plan.odt'); + expect(newest).toMatchObject({ + content: 'alice was here', + authorId: 'user-1', + authorLabel: 'Alice', + source: 'collabora', + }); + }); + + it('sets aside a save from a session opened before the document was restored', async () => { + await fs.writeFile(volume('Projects/plan.odt'), 'before'); + const application = app(); + await put(application, 'restored', { autosave: false }); + await markRestored('Projects/plan.odt', new Date().toISOString()); + + const response = await put(application, 'stale', { + lock: 'old', + accessToken: token({ iat: Math.floor(Date.now() / 1000) - 300 }), + }); + + expect(response.status).toBe(200); + expect(await fs.readFile(volume('Projects/plan.odt'), 'utf8')).toBe('restored'); + expect((await versionsOf('Projects/plan.odt'))[0]).toMatchObject({ + content: 'stale', + aside: true, + }); + }); +}); diff --git a/backend/tests/routes/versions.test.js b/backend/tests/routes/versions.test.js new file mode 100644 index 000000000..62f3f99ef --- /dev/null +++ b/backend/tests/routes/versions.test.js @@ -0,0 +1,602 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import express from 'express'; +import request from 'supertest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A file's history through the API, the way the Versions panel uses it: who + * may see it, download from it, put a version back, take one out as a copy or + * over another file, name and pin one, and delete them — from inside the + * application and through a share, whose owner decides what it shows. + */ + +let envContext; +let users; +let app; +let db; + +const load = (relative) => require(modulePath(relative)); + +const volume = (...segments) => path.join(envContext.volumeDir, ...segments); + +const write = async (relative, content) => { + await fs.mkdir(path.dirname(volume(relative)), { recursive: true }); + await fs.writeFile(volume(relative), content); +}; + +const read = (relative) => fs.readFile(volume(relative), 'utf8'); + +const buildApp = () => { + const application = express(); + application.use(express.json()); + application.use((req, _res, next) => { + const who = req.get('x-test-user'); + if (who?.startsWith('guest:')) { + req.guestSession = { id: 'guest-session', shareId: who.slice('guest:'.length) }; + } else if (who) { + req.user = users[who]; + } + next(); + }); + application.use('/api', load('src/routes/editor')); + application.use('/api', load('src/routes/versions')); + application.use('/api/shares', load('src/routes/shares')); + application.use(load('src/middleware/errorHandler').errorHandler); + return application; +}; + +const as = (who) => ({ + get: (url) => request(app).get(url).set('x-test-user', who), + post: (url, body) => request(app).post(url).set('x-test-user', who).send(body), + put: (url, body) => request(app).put(url).set('x-test-user', who).send(body), + patch: (url, body) => request(app).patch(url).set('x-test-user', who).send(body), +}); + +const history = (who, filePath) => + as(who).get(`/api/versions?path=${encodeURIComponent(filePath)}`); + +/** Save through the text editor, as `who`. */ +const edit = async (who, filePath, content) => { + const response = await as(who).put('/api/editor', { path: filePath, content }); + expect(response.status).toBe(200); +}; + +const setRules = (rules) => + load('src/services/settingsService').setSystemSetting('system', 'access', { rules }); + +beforeEach(async () => { + envContext = await setupTestEnv({ tag: 'versions-routes-', env: { SHARES_ENABLED: 'true' } }); + const usersService = load('src/services/users'); + const make = async (name, roles) => + usersService.createLocalUser({ + email: `${name}@example.com`, + username: name, + displayName: name[0].toUpperCase() + name.slice(1), + password: 'secret123', + roles, + }); + users = { + admin: await make('admin', ['admin']), + alice: await make('alice', ['user']), + bob: await make('bob', ['user']), + }; + db = await load('src/services/db').getDb(); + app = buildApp(); + await write('Projects/notes.md', '# From outside\n'); + await fs.mkdir(volume('Photos'), { recursive: true }); +}); + +afterEach(async () => { + load('src/services/trash/maintenance').stop(); + vi.restoreAllMocks(); + await envContext.cleanup(); +}); + +describe('the history of a file', () => { + it('lists the versions newest first, with who wrote each, and what the file is now', async () => { + await edit('alice', 'Projects/notes.md', '# Alice one\n'); + await edit('bob', 'Projects/notes.md', '# Bob two\n'); + + const response = await history('alice', 'Projects/notes.md'); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ + enabled: true, + file: { + name: 'notes.md', + path: 'Projects/notes.md', + size: 10, + author: { id: users.bob.id, label: 'Bob' }, + source: 'editor', + }, + rights: { see: true, download: true, restore: true, remove: true }, + totalBytes: 12 + 15, + }); + expect(response.body.versions).toMatchObject([ + { + size: 12, + author: { id: users.alice.id, label: 'Alice' }, + source: 'editor', + available: true, + }, + { size: 15, author: null, source: 'external', pinned: false, label: null }, + ]); + }); + + it('names authors as their accounts are called now', async () => { + await edit('alice', 'Projects/notes.md', '# Alice\n'); + await edit('bob', 'Projects/notes.md', '# Bob\n'); + db.prepare('UPDATE users SET display_name = ? WHERE id = ?').run( + 'Alice Martin', + users.alice.id + ); + + const response = await history('bob', 'Projects/notes.md'); + + expect(response.body.versions[0].author).toEqual({ id: users.alice.id, label: 'Alice Martin' }); + }); + + it('answers a file that has no history yet with none', async () => { + const response = await history('alice', 'Projects/notes.md'); + + expect(response.status).toBe(200); + expect(response.body.versions).toEqual([]); + expect(response.body.file.author).toBeNull(); + }); + + it('refuses what is not a file it can show', async () => { + expect((await history('alice', 'Projects')).status).toBe(400); + expect((await history('alice', 'Projects/missing.md')).status).toBe(404); + expect((await as('alice').get('/api/versions')).status).toBe(400); + }); + + it('shows nothing to someone who cannot read the file', async () => { + await edit('alice', 'Projects/notes.md', '# Alice\n'); + await setRules([{ path: 'Projects', recursive: true, permissions: 'hidden' }]); + + expect((await history('bob', 'Projects/notes.md')).status).toBe(403); + expect((await as('guest:nothing').get('/api/versions?path=Projects/notes.md')).status).toBe( + 403 + ); + }); +}); + +describe('reading a version', () => { + const firstVersion = async () => { + await edit('alice', 'Projects/notes.md', '# Alice\n'); + return (await history('alice', 'Projects/notes.md')).body.versions[0]; + }; + + it('downloads it under the file’s name with the version’s date', async () => { + const version = await firstVersion(); + + const response = await as('bob') + .get(`/api/versions/${version.id}/content?path=Projects/notes.md`) + .buffer(true) + .parse((res, done) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => done(null, Buffer.concat(chunks))); + }); + + expect(response.status).toBe(200); + expect(response.body.toString('utf8')).toBe('# From outside\n'); + expect(response.headers['content-disposition']).toMatch( + /^attachment;.*notes \(version \d{4}-\d{2}-\d{2} \d{2}-\d{2}\)\.md/ + ); + expect(response.headers['cache-control']).toContain('no-store'); + }); + + it('reads its text without writing anything', async () => { + const version = await firstVersion(); + + const response = await as('bob').get(`/api/versions/${version.id}/text?path=Projects/notes.md`); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ name: 'notes.md', content: '# From outside\n' }); + expect(await read('Projects/notes.md')).toBe('# Alice\n'); + }); + + it('sends the text of a large version compressed', async () => { + const earlier = '# Notes\n\nUne ligne de plus, et encore une.\n'.repeat(2000); + await write('Projects/notes.md', earlier); + await edit('alice', 'Projects/notes.md', '# Rewritten\n'); + const [version] = (await history('alice', 'Projects/notes.md')).body.versions; + + const response = await as('bob') + .get(`/api/versions/${version.id}/text?path=Projects/notes.md`) + .set('Accept-Encoding', 'gzip, deflate'); + + expect(response.status).toBe(200); + expect(response.headers['content-encoding']).toBe('gzip'); + expect(response.headers['cache-control']).toContain('no-store'); + expect(response.body).toMatchObject({ name: 'notes.md', content: earlier }); + }); + + it('never reaches a version through a file it does not belong to', async () => { + const version = await firstVersion(); + // A file with a history of its own: the version must be refused for not + // being one of its versions, not for the file having none. + await write('Photos/other.md', 'other'); + await edit('alice', 'Photos/other.md', 'other, edited'); + + const response = await as('alice').get(`/api/versions/${version.id}/text?path=Photos/other.md`); + + expect(response.status).toBe(404); + }); +}); + +describe('putting a version back', () => { + const twoVersions = async () => { + await edit('alice', 'Projects/notes.md', '# Alice\n'); + await edit('alice', 'Projects/notes.md', '# Alice again\n'); + return (await history('alice', 'Projects/notes.md')).body.versions; + }; + + it('restores the file as the version had it, keeping what it replaces as a version', async () => { + const [, oldest] = await twoVersions(); + + const response = await as('alice').post(`/api/versions/${oldest.id}/restore`, { + path: 'Projects/notes.md', + }); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ status: 'replaced', path: 'Projects/notes.md' }); + expect(await read('Projects/notes.md')).toBe('# From outside\n'); + const after = (await history('alice', 'Projects/notes.md')).body; + expect(after.versions.map((version) => version.id)).toContain(oldest.id); + expect(after.versions[0]).toMatchObject({ source: 'editor', size: 14 }); + expect(after.file).toMatchObject({ source: 'restore', author: { label: 'Alice' } }); + const file = db.prepare("SELECT restored_at FROM version_files WHERE state = 'live'").get(); + expect(file.restored_at).toEqual(expect.any(String)); + }); + + it('refuses someone who may read the file but not change it', async () => { + const [, oldest] = await twoVersions(); + await setRules([{ path: 'Projects', recursive: true, permissions: 'ro' }]); + + const listing = await history('bob', 'Projects/notes.md'); + expect(listing.body.rights).toEqual({ + see: true, + download: true, + restore: false, + remove: false, + }); + const response = await as('bob').post(`/api/versions/${oldest.id}/restore`, { + path: 'Projects/notes.md', + }); + + expect(response.status).toBe(403); + expect(await read('Projects/notes.md')).toBe('# Alice again\n'); + }); + + it('takes a version out as a new file in a folder someone chose', async () => { + const [, oldest] = await twoVersions(); + + const named = await as('alice').post(`/api/versions/${oldest.id}/copy`, { + path: 'Projects/notes.md', + destination: 'Photos', + name: 'notes before.md', + }); + const unnamed = await as('alice').post(`/api/versions/${oldest.id}/copy`, { + path: 'Projects/notes.md', + destination: 'Photos', + }); + const again = await as('alice').post(`/api/versions/${oldest.id}/copy`, { + path: 'Projects/notes.md', + destination: 'Photos', + }); + + expect(named.body).toEqual({ path: 'Photos/notes before.md', name: 'notes before.md' }); + expect(await read('Photos/notes before.md')).toBe('# From outside\n'); + expect(unnamed.body.name).toMatch(/^notes \(version \d{4}-\d{2}-\d{2} \d{2}-\d{2}\)\.md$/); + expect(again.body.name).not.toBe(unnamed.body.name); + expect(await read('Projects/notes.md')).toBe('# Alice again\n'); + }); + + /** + * A copy is a new file. One that arrives under its name while the version's + * content is being written — a file saved over SMB, another copy — is someone + * else's: it stays as it is, it does not become an earlier version of the + * copy, and the copy takes the next name. + */ + it('never replaces, nor keeps as its version, a file that arrives under the name meanwhile', async () => { + const [, oldest] = await twoVersions(); + const target = volume('Photos', 'notes before.md'); + const theirs = Buffer.from('dropped over SMB while the copy was written\n'); + const copyFile = fs.copyFile.bind(fs); + let arrived = false; + vi.spyOn(fs, 'copyFile').mockImplementation(async (from, to, mode) => { + if (!arrived && path.dirname(to) === volume('Photos')) { + arrived = true; + await fs.writeFile(target, theirs); + } + return copyFile(from, to, mode); + }); + + const response = await as('alice').post(`/api/versions/${oldest.id}/copy`, { + path: 'Projects/notes.md', + destination: 'Photos', + name: 'notes before.md', + }); + + expect(arrived).toBe(true); + expect(response.status).toBe(200); + expect(response.body).toEqual({ + path: 'Photos/notes before (1).md', + name: 'notes before (1).md', + }); + expect(await fs.readFile(target)).toEqual(theirs); + expect(await read('Photos/notes before (1).md')).toBe('# From outside\n'); + expect((await fs.readdir(volume('Photos'))).sort()).toEqual([ + 'notes before (1).md', + 'notes before.md', + ]); + expect((await history('alice', 'Photos/notes before.md')).body.versions).toEqual([]); + expect((await history('alice', 'Photos/notes before (1).md')).body.versions).toEqual([]); + }); + + it('refuses a copy into a folder that is not one, or a name that is not one', async () => { + const [, oldest] = await twoVersions(); + + const noFolder = await as('alice').post(`/api/versions/${oldest.id}/copy`, { + path: 'Projects/notes.md', + destination: 'Projects/notes.md', + }); + const badName = await as('alice').post(`/api/versions/${oldest.id}/copy`, { + path: 'Projects/notes.md', + destination: 'Photos', + name: '../escape.md', + }); + + expect(noFolder.status).toBe(400); + expect(badName.status).toBe(400); + }); + + it('puts a version over another file, whose own content becomes its version', async () => { + const [, oldest] = await twoVersions(); + await write('Photos/other.md', 'what other held'); + + const response = await as('alice').post(`/api/versions/${oldest.id}/replace`, { + path: 'Projects/notes.md', + target: 'Photos/other.md', + }); + + expect(response.status).toBe(200); + expect(await read('Photos/other.md')).toBe('# From outside\n'); + const other = (await history('alice', 'Photos/other.md')).body.versions; + expect(other).toHaveLength(1); + const text = await as('alice').get(`/api/versions/${other[0].id}/text?path=Photos/other.md`); + expect(text.body.content).toBe('what other held'); + }); + + it('refuses a copy into a folder where files cannot be created', async () => { + const [, oldest] = await twoVersions(); + await setRules([{ path: 'Photos', recursive: true, permissions: 'ro' }]); + + const response = await as('bob').post(`/api/versions/${oldest.id}/copy`, { + path: 'Projects/notes.md', + destination: 'Photos', + }); + + expect(response.status).toBe(403); + expect(await fs.readdir(volume('Photos'))).toEqual([]); + }); + + it('never puts a version over a file someone may not change', async () => { + const [, oldest] = await twoVersions(); + await write('Photos/other.md', 'what other held'); + await setRules([{ path: 'Photos', recursive: true, permissions: 'ro' }]); + + const response = await as('bob').post(`/api/versions/${oldest.id}/replace`, { + path: 'Projects/notes.md', + target: 'Photos/other.md', + }); + + expect(response.status).toBe(403); + expect(await read('Photos/other.md')).toBe('what other held'); + }); + + it('never puts a version over a folder', async () => { + const [, oldest] = await twoVersions(); + + const response = await as('alice').post(`/api/versions/${oldest.id}/replace`, { + path: 'Projects/notes.md', + target: 'Photos', + }); + + expect(response.status).toBe(400); + }); +}); + +describe('naming, pinning and deleting versions', () => { + const threeVersions = async () => { + await edit('alice', 'Projects/notes.md', '# one\n'); + await edit('alice', 'Projects/notes.md', '# two\n'); + await edit('alice', 'Projects/notes.md', '# three\n'); + return (await history('alice', 'Projects/notes.md')).body.versions; + }; + + it('names and pins a version, and takes the name away again', async () => { + const [version] = await threeVersions(); + + const named = await as('alice').patch(`/api/versions/${version.id}`, { + path: 'Projects/notes.md', + label: ' Sent to the client ', + pinned: true, + }); + expect(named.body).toMatchObject({ label: 'Sent to the client', pinned: true }); + + const cleared = await as('alice').patch(`/api/versions/${version.id}`, { + path: 'Projects/notes.md', + label: '', + }); + expect(cleared.body).toMatchObject({ label: null, pinned: true }); + }); + + it('refuses a name that is too long, or a pin that is not true or false', async () => { + const [version] = await threeVersions(); + + const tooLong = await as('alice').patch(`/api/versions/${version.id}`, { + path: 'Projects/notes.md', + label: 'x'.repeat(201), + }); + const notBoolean = await as('alice').patch(`/api/versions/${version.id}`, { + path: 'Projects/notes.md', + pinned: 'yes', + }); + + expect(tooLong.status).toBe(400); + expect(notBoolean.status).toBe(400); + }); + + it('deletes the versions chosen, then all of them, leaving the file alone', async () => { + const versions = await threeVersions(); + + const some = await as('alice').post('/api/versions/delete', { + path: 'Projects/notes.md', + ids: [versions[0].id, versions[2].id, 'not-one-of-them'], + }); + expect(some.body.deleted).toBe(2); + expect(some.body.items).toContainEqual({ id: 'not-one-of-them', status: 'not-found' }); + expect((await history('alice', 'Projects/notes.md')).body.versions.map((v) => v.id)).toEqual([ + versions[1].id, + ]); + + const all = await as('alice').post('/api/versions/delete', { + path: 'Projects/notes.md', + all: true, + }); + expect(all.body.deleted).toBe(1); + expect((await history('alice', 'Projects/notes.md')).body.versions).toEqual([]); + expect(await read('Projects/notes.md')).toBe('# three\n'); + }); + + it('never deletes a version of another file through this one', async () => { + await threeVersions(); + await write('Photos/other.md', 'other one'); + await edit('alice', 'Photos/other.md', 'other two'); + const [otherVersion] = (await history('alice', 'Photos/other.md')).body.versions; + + const response = await as('alice').post('/api/versions/delete', { + path: 'Projects/notes.md', + ids: [otherVersion.id], + }); + + expect(response.body).toEqual({ + items: [{ id: otherVersion.id, status: 'not-found' }], + deleted: 0, + }); + expect((await history('alice', 'Photos/other.md')).body.versions).toHaveLength(1); + }); + + it('lets only whoever may delete the file delete its versions', async () => { + const [version] = await threeVersions(); + await setRules([{ path: 'Projects', recursive: true, permissions: 'ro' }]); + + const response = await as('bob').post('/api/versions/delete', { + path: 'Projects/notes.md', + ids: [version.id], + }); + + expect(response.status).toBe(403); + expect((await history('admin', 'Projects/notes.md')).body.versions).toHaveLength(3); + }); + + it('refuses a request that names nothing to delete', async () => { + await threeVersions(); + + const response = await as('alice').post('/api/versions/delete', { path: 'Projects/notes.md' }); + + expect(response.status).toBe(400); + }); +}); + +describe('the history through a share', () => { + const share = async (body) => { + const response = await as('alice').post('/api/shares', { + sourcePath: 'Projects/notes.md', + ...body, + }); + expect(response.status).toBe(201); + return response.body; + }; + + const withHistory = async () => { + await edit('alice', 'Projects/notes.md', '# Alice\n'); + return (await history('alice', 'Projects/notes.md')).body.versions[0]; + }; + + it('shows none through a link for anyone until its owner turns it on', async () => { + const version = await withHistory(); + const link = await share({ sharingType: 'anyone', accessMode: 'readwrite' }); + expect(link).toMatchObject({ versionsVisible: false, versionsDownload: false }); + const visitor = as(`guest:${link.id}`); + const through = `share/${link.shareToken}`; + + expect((await visitor.get(`/api/versions?path=${through}`)).status).toBe(403); + + await as('alice').put(`/api/shares/${link.id}`, { versionsVisible: true }); + const listed = await visitor.get(`/api/versions?path=${through}`); + expect(listed.status).toBe(200); + expect(listed.body.rights).toEqual({ see: true, download: false, restore: true, remove: true }); + expect((await visitor.get(`/api/versions/${version.id}/content?path=${through}`)).status).toBe( + 403 + ); + + await as('alice').put(`/api/shares/${link.id}`, { versionsDownload: true }); + expect((await visitor.get(`/api/versions/${version.id}/content?path=${through}`)).status).toBe( + 200 + ); + }); + + it('shows the history to the accounts a share names, by default', async () => { + await withHistory(); + const named = await share({ sharingType: 'users', userIds: [users.bob.id] }); + expect(named).toMatchObject({ versionsVisible: true, versionsDownload: true }); + + const response = await history('bob', `share/${named.shareToken}`); + + expect(response.status).toBe(200); + expect(response.body.versions).toHaveLength(1); + }); + + it('lets a visitor restore only through a share that lets them write', async () => { + const version = await withHistory(); + const readOnly = await share({ sharingType: 'anyone', versionsVisible: true }); + const readWrite = await share({ + sharingType: 'anyone', + accessMode: 'readwrite', + versionsVisible: true, + }); + + const refused = await as(`guest:${readOnly.id}`).post(`/api/versions/${version.id}/restore`, { + path: `share/${readOnly.shareToken}`, + }); + const restored = await as(`guest:${readWrite.id}`).post(`/api/versions/${version.id}/restore`, { + path: `share/${readWrite.shareToken}`, + }); + + expect(refused.status).toBe(403); + expect(restored.status).toBe(200); + expect(await read('Projects/notes.md')).toBe('# From outside\n'); + const [newest] = (await history('alice', 'Projects/notes.md')).body.versions; + expect(newest.author).toEqual({ id: users.alice.id, label: 'Alice' }); + expect((await history('alice', 'Projects/notes.md')).body.file.author).toEqual({ + id: null, + label: 'share-link', + }); + }); + + it('refuses share options that are not true or false', async () => { + const response = await as('alice').post('/api/shares', { + sourcePath: 'Projects/notes.md', + versionsVisible: 'yes', + }); + + expect(response.status).toBe(400); + }); +}); diff --git a/backend/tests/routes/volumes.test.js b/backend/tests/routes/volumes.test.js new file mode 100644 index 000000000..fdd093286 --- /dev/null +++ b/backend/tests/routes/volumes.test.js @@ -0,0 +1,112 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import express from 'express'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Which volumes a caller is told about. Four different answers depending on who + * is asking and whether `USER_VOLUMES` is on — and one of them, the empty list + * for a share visitor, is the difference between a link to one folder and a map + * of the whole server. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async (env = {}) => { + currentEnv = await setupTestEnv({ tag: 'volumes-route-', env }); + const dbService = currentEnv.requireFresh('src/services/db'); + const db = await dbService.getDb(); + const now = new Date().toISOString(); + for (const [id, username, roles] of [ + ['admin-1', 'admin', '["admin"]'], + ['user-1', 'regular', '["user"]'], + ]) { + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, ?, ?, ?)` + ).run(id, `${username}@example.com`, username, username, roles, now, now); + } + await fs.mkdir(path.join(currentEnv.volumeDir, 'Media'), { recursive: true }); + await fs.mkdir(path.join(currentEnv.volumeDir, 'Documents'), { recursive: true }); + return db; +}; + +const buildApp = (user) => { + const routes = currentEnv.requireFresh('src/routes/volumes'); + const { errorHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + const app = express(); + app.use((req, _res, next) => { + if (user) req.user = user; + next(); + }); + app.use('/api', routes); + app.use(errorHandler); + return app; +}; + +const ADMIN = { id: 'admin-1', roles: ['admin'] }; +const REGULAR = { id: 'user-1', roles: ['user'] }; + +describe('who is told which volumes exist', () => { + /** + * A share visitor carries a guest session and no account. Listing volumes for + * them turns a link to one folder into a map of the server. + */ + it('tells a visitor with no account nothing', async () => { + await seed(); + + const response = await request(buildApp(null)).get('/api/volumes'); + + expect(response.status).toBe(200); + expect(response.body).toEqual([]); + }); + + it('shows every volume when the feature is off', async () => { + await seed(); + + const response = await request(buildApp(REGULAR)).get('/api/volumes'); + + expect(response.status).toBe(200); + expect(response.body.map((v) => v.name).sort()).toEqual(['Documents', 'Media']); + }); + + it('shows every volume to an administrator even when the feature is on', async () => { + await seed({ USER_VOLUMES: 'true' }); + + const response = await request(buildApp(ADMIN)).get('/api/volumes'); + + expect(response.body.map((v) => v.name).sort()).toEqual(['Documents', 'Media']); + }); + + it('shows a regular account only what it was assigned', async () => { + const db = await seed({ USER_VOLUMES: 'true' }); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO user_volumes (id, user_id, label, path, access_mode, created_at, updated_at) + VALUES ('uv1','user-1','Media', ?, 'readwrite', ?, ?)` + ).run(path.join(currentEnv.volumeDir, 'Media'), now, now); + + const response = await request(buildApp(REGULAR)).get('/api/volumes'); + + expect(response.status).toBe(200); + expect(response.body.map((v) => v.name)).toEqual(['Media']); + expect(response.body[0]).toMatchObject({ kind: 'volume', accessMode: 'readwrite' }); + }); + + it('gives a regular account with no assignment an empty list', async () => { + await seed({ USER_VOLUMES: 'true' }); + + const response = await request(buildApp(REGULAR)).get('/api/volumes'); + + expect(response.body).toEqual([]); + }); +}); diff --git a/backend/tests/routes/zip-compress-safety.test.js b/backend/tests/routes/zip-compress-safety.test.js index 69d0d1d5d..65ed50db1 100644 --- a/backend/tests/routes/zip-compress-safety.test.js +++ b/backend/tests/routes/zip-compress-safety.test.js @@ -154,10 +154,10 @@ const arriveBeforeTaking = (target, arrive) => { const waitFor = async (condition, what) => { const deadline = Date.now() + 5000; - // eslint-disable-next-line no-await-in-loop + while (!(await condition())) { if (Date.now() > deadline) throw new Error(`Timed out waiting for ${what}`); - // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, 20)); } }; diff --git a/backend/tests/scripts/delete-guard-lint.test.js b/backend/tests/scripts/delete-guard-lint.test.js index 1b75c803e..67523d034 100644 --- a/backend/tests/scripts/delete-guard-lint.test.js +++ b/backend/tests/scripts/delete-guard-lint.test.js @@ -28,7 +28,7 @@ const lint = async (relativeFile, code) => { // ESLint puts the restricted name in front of the message for one of the two // rules, so the reason is what is matched, not the whole line. const GUARD = expect.stringContaining( - 'Deleting from disk goes through services/trash (see .eslintrc.cjs).' + 'Deleting from disk goes through services/trash (see eslint.config.mjs).' ); describe('removing from disk outside the trash', () => { diff --git a/backend/tests/security/onlyoffice-session-end-exposure.test.js b/backend/tests/security/onlyoffice-session-end-exposure.test.js new file mode 100644 index 000000000..1d548f5f2 --- /dev/null +++ b/backend/tests/security/onlyoffice-session-end-exposure.test.js @@ -0,0 +1,233 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import express from 'express'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Who may end an editing session. + * + * The route exists to be called from a page that is being closed, which means + * it is called by a beacon — a request with no reply anybody reads, sent while + * the tab it came from is disappearing. That is a shape worth looking at + * twice: a route meant to be easy to reach at an awkward moment must not be + * easy to reach from somewhere else entirely. + * + * Ending somebody's session is not a catastrophe — they lose an editing + * session, and the document is saved on the way — but it is an interruption + * anybody could cause, repeatedly, from outside. So the whole stack is wired + * here, session store and authentication included, rather than the router + * alone: what is being held is the gate, not the handler. + * + * Passwords are hashed with bcrypt at cost 12, hence the timeouts. + */ + +const PASSWORD = 'secret123'; +const FILE = 'report.docx'; + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const build = async () => { + currentEnv = await setupTestEnv({ + tag: 'onlyoffice-session-end-exposure-', + env: { + AUTH_ENABLED: 'true', + AUTH_MODE: 'local', + PUBLIC_URL: 'https://files.example.test', + ONLYOFFICE_URL: 'http://127.0.0.1:1', + ONLYOFFICE_SECRET: 'onlyoffice-test-secret', + }, + }); + + await fs.writeFile(path.join(currentEnv.volumeDir, FILE), Buffer.from('original')); + + const { configureSession } = currentEnv.requireFresh('src/middleware/session'); + const authMiddleware = currentEnv.requireFresh('src/middleware/authMiddleware'); + const authRoutes = currentEnv.requireFresh('src/routes/auth'); + const userRoutes = currentEnv.requireFresh('src/routes/users'); + const onlyofficeRoutes = currentEnv.requireFresh('src/routes/onlyoffice'); + const { errorHandler, notFoundHandler } = currentEnv.requireFresh('src/middleware/errorHandler'); + + const app = express(); + app.use(express.json()); + app.use(cookieParser()); + configureSession(app); + app.use(authMiddleware); + app.use('/api/auth', authRoutes); + app.use('/api', userRoutes); + app.use('/api', onlyofficeRoutes); + app.use(notFoundHandler); + app.use(errorHandler); + + return app; +}; + +const setUpOwner = async (app) => { + const browser = request.agent(app); + const response = await browser + .post('/api/auth/setup') + .send({ email: 'owner@example.com', username: 'owner', password: PASSWORD }); + expect(response.status).toBe(201); + return browser; +}; + +/** A second account, made by the administrator and signed in for itself. */ +const setUpOther = async (app, owner) => { + const created = await owner.post('/api/users').send({ + email: 'other@example.com', + username: 'other', + password: PASSWORD, + roles: ['user'], + }); + expect(created.status).toBe(201); + + const other = request.agent(app); + const signedIn = await other + .post('/api/auth/login') + .send({ email: 'other@example.com', password: PASSWORD }); + expect(signedIn.status).toBe(200); + return other; +}; + +const openEditor = async (browser) => { + const config = await browser.post('/api/onlyoffice/config').send({ path: FILE }); + expect(config.status).toBe(200); + return config.body.forceSaveSessionId; +}; + +describe('ending an editing session, from outside', () => { + it('is refused to somebody with no session of their own', { timeout: 30000 }, async () => { + const app = await build(); + const owner = await setUpOwner(app); + const sessionId = await openEditor(owner); + + // A cross-site beacon arrives exactly like this: the session cookie is + // `SameSite=Lax` and is not attached to a POST another site made, so what + // reaches the server is a request from nobody. It is refused before the + // route is reached, which is why the whole stack is wired here. + const stranger = request(app); + const refused = await stranger + .post('/api/onlyoffice/session-end') + .send({ path: FILE, sessionId }); + expect(refused.status).toBe(401); + + // And the session is untouched: the owner's editor carries on. + const heartbeat = await owner + .post('/api/onlyoffice/session-heartbeat') + .send({ path: FILE, sessionId }); + expect(heartbeat.status).toBe(200); + }); + + it('is refused to another account holding the identifier', { timeout: 40000 }, async () => { + const app = await build(); + const owner = await setUpOwner(app); + const other = await setUpOther(app, owner); + const sessionId = await openEditor(owner); + + // The identifier travels in a request body, so it is not a secret: the + // question is whether holding it is enough. It is not — a session belongs + // to whoever opened it. + const refused = await other.post('/api/onlyoffice/session-end').send({ path: FILE, sessionId }); + expect(refused.status).toBe(403); + + const heartbeat = await owner + .post('/api/onlyoffice/session-heartbeat') + .send({ path: FILE, sessionId }); + expect(heartbeat.status).toBe(200); + }); + + it('is refused for a file the caller cannot reach', { timeout: 40000 }, async () => { + const app = await build(); + const owner = await setUpOwner(app); + const other = await setUpOther(app, owner); + await openEditor(owner); + + // The other account's own session, aimed at the owner's personal folder — + // a path it may not read. The refusal comes from the access check, before + // anything is ended. + const refused = await other + .post('/api/onlyoffice/session-end') + .send({ path: 'personal/owner/report.docx', sessionId: 'anything' }); + expect(refused.status).toBeGreaterThanOrEqual(400); + expect(refused.status).toBeLessThan(500); + }); + + it('ends it for the account that opened it', { timeout: 30000 }, async () => { + const app = await build(); + const owner = await setUpOwner(app); + const sessionId = await openEditor(owner); + + const ended = await owner.post('/api/onlyoffice/session-end').send({ path: FILE, sessionId }); + expect(ended.status).toBe(200); + expect(ended.body).toMatchObject({ ended: true }); + + const afterwards = await owner + .post('/api/onlyoffice/session-heartbeat') + .send({ path: FILE, sessionId }); + expect(afterwards.status).toBe(403); + }); + + it('cannot be reached by an API token either', { timeout: 30000 }, async () => { + const app = await build(); + const owner = await setUpOwner(app); + const sessionId = await openEditor(owner); + + const minted = await owner + .post('/api/auth/tokens') + .send({ password: PASSWORD, name: 'Script', scope: 'write' }); + expect(minted.status).toBe(201); + + // The scope lets it write, and the account is the same one — the session + // check alone would have let this through, because the session does belong + // to that account. What stops it is the door: a token has no browser, so + // it has no editing session, so it has no business ending one. Written + // after this test found it open. + const refused = await request(app) + .post('/api/onlyoffice/session-end') + .set('Authorization', `Bearer ${minted.body.secret}`) + .send({ path: FILE, sessionId }); + expect(refused.status).toBe(403); + + const heartbeat = await owner + .post('/api/onlyoffice/session-heartbeat') + .send({ path: FILE, sessionId }); + expect(heartbeat.status).toBe(200); + }); + + it('cannot open an editing session with a token either', { timeout: 30000 }, async () => { + const app = await build(); + const owner = await setUpOwner(app); + + const minted = await owner + .post('/api/auth/tokens') + .send({ password: PASSWORD, name: 'Script', scope: 'write' }); + expect(minted.status).toBe(201); + + // The other half of the same door. A token that could ask for a + // configuration would be handed an editing session, and the document would + // be marked as being edited by a script that is not editing it — with + // nothing to close it but the timeout. + for (const route of [ + '/api/onlyoffice/config', + '/api/onlyoffice/session-heartbeat', + '/api/onlyoffice/force-save', + '/api/collabora/config', + ]) { + const refused = await request(app) + .post(route) + .set('Authorization', `Bearer ${minted.body.secret}`) + .send({ path: FILE, sessionId: 'anything' }); + expect(refused.status, `${route} was open to a token`).toBe(403); + expect(refused.body.error.code).toBe('AUTH_TOKEN_NOT_ALLOWED'); + } + }); +}); diff --git a/backend/tests/services/access-control.test.js b/backend/tests/services/access-control.test.js index 1c5689d86..b751ef096 100644 --- a/backend/tests/services/access-control.test.js +++ b/backend/tests/services/access-control.test.js @@ -1,11 +1,7 @@ import { describe, it, expect } from 'vitest'; import { setupTestEnv } from '../helpers/env-test-utils.js'; -const ACCESS_MODULES = [ - 'src/services/storage/jsonStorage', - 'src/services/settingsService', - 'src/services/accessControlService', -]; +const ACCESS_MODULES = ['src/services/settingsService', 'src/services/accessControlService']; const createAccessContext = async () => { const envContext = await setupTestEnv({ diff --git a/backend/tests/services/access-manager.test.js b/backend/tests/services/access-manager.test.js index 3ff61b169..c8fc10f77 100644 --- a/backend/tests/services/access-manager.test.js +++ b/backend/tests/services/access-manager.test.js @@ -569,6 +569,7 @@ describe('accessManager — what a share grants', () => { canDelete: false, canUpload: false, canCreateFolder: false, + canCreateFile: false, effectivePermission: 'ro', }); }); @@ -585,10 +586,28 @@ describe('accessManager — what a share grants', () => { canDelete: true, canUpload: true, canCreateFolder: true, + canCreateFile: true, effectivePermission: 'rw', }); }); + it.each([ + ['allowDelete', 'canDelete'], + ['allowUpload', 'canUpload'], + ['allowCreateFolder', 'canCreateFolder'], + ['allowCreateFile', 'canCreateFile'], + ])('withholds %s on its own, leaving the rest of the write grant', async (flag, granted) => { + const { share, guestSession } = await openShare(`grant-without-${flag}`, { + accessMode: 'readwrite', + [flag]: false, + }); + + const access = await accessTo(share, { guestSession }); + + expect(access[granted]).toBe(false); + expect(access.canWrite).toBe(true); + }); + it('never lets a share be shared again', async () => { const { share, guestSession } = await openShare('grant-no-resharing', { accessMode: 'readwrite', diff --git a/backend/tests/services/archive-browse.test.js b/backend/tests/services/archive-browse.test.js new file mode 100644 index 000000000..65545c389 --- /dev/null +++ b/backend/tests/services/archive-browse.test.js @@ -0,0 +1,268 @@ +import { describe, it, expect } from 'vitest'; + +const { + parseRecords, + describeEntries, + entryPathOf, + levelOf, +} = require('../../src/services/archiveBrowseService'); + +/** + * Reading an archive's table of contents, without unpacking it. + * + * Everything here works on the text `7z l -slt` prints, because that is where + * the archive's own words arrive: a name in an archive is somebody else's + * input, and this is the layer that decides what it is allowed to mean. What + * it decides has to hold for names nobody would type — a path that climbs out + * of the archive, a Windows drive letter, an equals sign in a filename, a + * newline in one — so those are what most of this is about. + */ + +/** The shape 7-Zip prints: its own header, ten dashes, then a block per entry. */ +const listing = (blocks) => + [ + '7-Zip (z) 26.03 (x64) : Copyright (c) 1999-2026 Igor Pavlov', + '', + 'Listing archive: /volumes/Work/backup.zip', + '', + '--', + 'Path = /volumes/Work/backup.zip', + 'Type = zip', + 'Physical Size = 4096', + '', + '----------', + ...blocks, + ].join('\n'); + +const entry = (fields) => + Object.entries(fields) + .map(([key, value]) => `${key} = ${value}`) + .concat('') + .join('\n'); + +describe('the records 7-Zip prints', () => { + it('reads one record per entry, and none of its own header', () => { + const records = parseRecords( + listing([ + entry({ Path: 'notes.txt', Size: 12, Attributes: 'A_ -rw-r--r--' }), + entry({ Path: 'docs', Size: 0, Attributes: 'D_ drwxr-xr-x' }), + ]) + ); + + expect(records).toHaveLength(2); + expect(records[0]).toMatchObject({ Path: 'notes.txt', Size: '12' }); + expect(records.some((record) => String(record.Type) === 'zip')).toBe(false); + }); + + /** A filename may hold " = " as easily as any other characters. */ + it('splits a line at the first separator, not at every one', () => { + const [record] = parseRecords(listing([entry({ Path: 'a = b = c.txt', Size: 1 })])); + + expect(record.Path).toBe('a = b = c.txt'); + }); + + /** + * A name with a newline in it is printed raw, so its second half arrives as + * a line with no key at all. Dropped, it would leave a truncated name that + * reads like a different file — and that file is the one a read would go + * looking for. + */ + it('keeps a name that runs onto the next line', () => { + const [record] = parseRecords( + listing([['Path = first\nsecond.txt', 'Size = 3', 'Attributes = A_', ''].join('\n')]) + ); + + expect(record.Path).toBe('first\nsecond.txt'); + expect(record.Size).toBe('3'); + }); + + it('has nothing to say about output with no entries in it', () => { + expect(parseRecords('7-Zip (z) 26.03\n\nno archive here\n')).toEqual([]); + expect(parseRecords('')).toEqual([]); + }); +}); + +describe('where an entry is allowed to be', () => { + it.each([ + ['docs/report.txt', 'docs/report.txt'], + ['./docs/report.txt', 'docs/report.txt'], + ['docs//report.txt', 'docs/report.txt'], + ['docs\\report.txt', 'docs/report.txt'], + ['/docs/report.txt', 'docs/report.txt'], + ])('reads %j as %j', (written, expected) => { + expect(entryPathOf(written)).toBe(expected); + }); + + /** + * The names a crafted archive carries. None of them is a place inside the + * archive, and none of them is ever handed back as one. + */ + it.each([ + ['../../etc/passwd'], + ['docs/../../etc/passwd'], + ['..'], + ['C:/Windows/System32'], + ['c:\\Windows'], + [''], + ['/'], + ['.'], + ])('refuses %j', (written) => { + expect(entryPathOf(written)).toBeNull(); + }); + + it('refuses what is not a string at all', () => { + expect(entryPathOf(undefined)).toBeNull(); + expect(entryPathOf(42)).toBeNull(); + }); +}); + +describe('what an archive says about itself', () => { + it('reports each entry with its size, its date and whether it is a folder', () => { + const { entries } = describeEntries( + listing([ + entry({ + Path: 'docs/report.txt', + Size: 4096, + Modified: '2026-09-16 11:22:33', + Attributes: 'A_ -rw-r--r--', + }), + entry({ + Path: 'docs', + Size: 0, + Modified: '2026-09-16 11:22:30', + Attributes: 'D_ drwxr-xr-x', + }), + ]) + ); + + expect(entries).toEqual([ + { + path: 'docs/report.txt', + isDirectory: false, + size: 4096, + modified: '2026-09-16 11:22:33', + encrypted: false, + }, + { + path: 'docs', + isDirectory: true, + size: 0, + modified: '2026-09-16 11:22:30', + encrypted: false, + }, + ]); + }); + + it('reads a folder from either of the two ways 7-Zip says so', () => { + const { entries } = describeEntries( + listing([ + entry({ Path: 'by-attribute', Attributes: 'D_ drwxr-xr-x' }), + entry({ Path: 'by-flag', Folder: '+', Attributes: '' }), + ]) + ); + + expect(entries.map((item) => item.isDirectory)).toEqual([true, true]); + }); + + it('marks an entry whose contents are encrypted', () => { + const { entries } = describeEntries( + listing([entry({ Path: 'secret.txt', Size: 10, Encrypted: '+', Attributes: 'A_' })]) + ); + + expect(entries[0].encrypted).toBe(true); + }); + + it('says nothing rather than something wrong about a size or a date it cannot read', () => { + const { entries } = describeEntries( + listing([entry({ Path: 'odd.txt', Size: 'huge', Modified: 'yesterday', Attributes: 'A_' })]) + ); + + expect(entries[0]).toMatchObject({ size: null, modified: null }); + }); + + /** Counted rather than shown: what cannot be somewhere is not shown as somewhere. */ + it('leaves out the entries that point outside the archive, and counts them', () => { + const { entries, outside } = describeEntries( + listing([ + entry({ Path: '../../etc/passwd', Size: 1, Attributes: 'A_' }), + entry({ Path: 'C:/Windows/notepad.exe', Size: 2, Attributes: 'A_' }), + entry({ Path: 'safe.txt', Size: 3, Attributes: 'A_' }), + ]) + ); + + expect(entries.map((item) => item.path)).toEqual(['safe.txt']); + expect(outside).toBe(2); + }); +}); + +describe('one level of an archive', () => { + const entries = [ + { path: 'notes.txt', isDirectory: false, size: 10, modified: null, encrypted: false }, + { path: 'docs/report.txt', isDirectory: false, size: 20, modified: null, encrypted: false }, + { path: 'docs/deep/inner.txt', isDirectory: false, size: 30, modified: null, encrypted: false }, + { + path: 'photos', + isDirectory: true, + size: 0, + modified: '2026-01-01 00:00:00', + encrypted: false, + }, + ]; + + it('shows the folders and files of the top, and nothing from below it', () => { + const level = levelOf(entries, ''); + + expect(level.entries.map((item) => `${item.name}${item.isDirectory ? '/' : ''}`)).toEqual([ + 'docs/', + 'photos/', + 'notes.txt', + ]); + }); + + /** + * A zip of `docs/report.txt` may hold that one entry and nothing else: the + * folder exists because something is in it, not because it was written down. + */ + it('shows a folder nothing in the archive names', () => { + const level = levelOf([entries[1]], ''); + + expect(level.entries).toEqual([ + { + name: 'docs', + path: 'docs', + isDirectory: true, + size: null, + modified: null, + encrypted: false, + }, + ]); + }); + + it('prefers what the archive says about a folder to what it works out', () => { + const level = levelOf(entries, ''); + const photos = level.entries.find((item) => item.name === 'photos'); + + expect(photos.modified).toBe('2026-01-01 00:00:00'); + }); + + it('goes down a level without showing the level below that', () => { + const level = levelOf(entries, 'docs'); + + expect(level.entries.map((item) => item.name)).toEqual(['deep', 'report.txt']); + expect(level.entries.find((item) => item.name === 'deep').path).toBe('docs/deep'); + }); + + it('knows a folder that is only an entry of its own', () => { + expect(levelOf(entries, 'photos').exists).toBe(true); + expect(levelOf(entries, 'photos').entries).toEqual([]); + }); + + it('does not invent a level that is not there', () => { + expect(levelOf(entries, 'nowhere').exists).toBe(false); + }); + + /** `doc` is not `docs`, however much of it is a prefix. */ + it('does not take a folder for one whose name it begins', () => { + expect(levelOf(entries, 'doc').exists).toBe(false); + }); +}); diff --git a/backend/tests/services/archive-cache.test.js b/backend/tests/services/archive-cache.test.js new file mode 100644 index 000000000..44eb30fb3 --- /dev/null +++ b/backend/tests/services/archive-cache.test.js @@ -0,0 +1,200 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What the cache of decompressed archives may keep. + * + * Every copy in it can be made again from the archive it came from, so nothing + * here is ever missed — which is what makes a budget the right answer rather + * than a worry. Without one, two backups opened once fill a cache directory + * somebody sized for thumbnails, and the thumbnails go instead. + */ + +let currentEnv; + +afterEach(async () => { + try { + await currentEnv?.requireFresh('src/services/archiveCacheService').stopArchiveCacheWork(); + } catch (_) { + // Never loaded, which is as stopped as it gets. + } + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async (env = {}) => { + currentEnv = await setupTestEnv({ tag: 'archive-cache-', env }); + const service = currentEnv.requireFresh('src/services/archiveCacheService'); + const directory = service.cacheDirectory(); + await fs.mkdir(directory, { recursive: true }); + return { service, directory }; +}; + +/** + * A cached copy of the given size, last read `agedMs` ago. + * + * The names are what this service writes — a version, then a hash — because + * that shape is the whole of its permission to remove anything. + */ +const writeCached = async (directory, name, { size = 1024, agedMs = 0 } = {}) => { + const file = path.join(directory, name); + await fs.writeFile(file, Buffer.alloc(size)); + if (agedMs) { + const when = new Date(Date.now() - agedMs); + await fs.utimes(file, when, when); + } + return file; +}; + +const remaining = async (directory) => (await fs.readdir(directory)).sort(); + +describe('sweeping the cache of decompressed archives', () => { + it('keeps what is recent and within the budget', async () => { + const { service, directory } = await seed(); + await writeCached(directory, 'v1-aaaa.inner'); + await writeCached(directory, 'v1-bbbb.inner'); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual(['v1-aaaa.inner', 'v1-bbbb.inner']); + }); + + it('takes one nobody has opened in a long time', async () => { + const { service, directory } = await seed(); + await writeCached(directory, 'v1-0d0d.inner', { agedMs: 400 * 24 * 60 * 60 * 1000 }); + await writeCached(directory, 'v1-e0e0.inner'); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual(['v1-e0e0.inner']); + }); + + /** Least recently read first: what costs least to make again. */ + it('comes down to the budget, oldest first', async () => { + const { service, directory } = await seed({ ARCHIVE_CACHE_MAX_SIZE: '3K' }); + await writeCached(directory, 'v1-f1f1.inner', { size: 2048, agedMs: 3 * 60 * 60 * 1000 }); + await writeCached(directory, 'v1-f2f2.inner', { size: 2048, agedMs: 2 * 60 * 60 * 1000 }); + await writeCached(directory, 'v1-f3f3.inner', { size: 2048, agedMs: 60 * 60 * 1000 }); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual(['v1-f3f3.inner']); + }); + + /** A copy being made is not rubbish; one abandoned by a stopped run is. */ + it('takes a temporary file a stopped run left behind', async () => { + const { service, directory } = await seed(); + await writeCached(directory, 'v1-aaaa.inner.tmp-1-2', { agedMs: 3 * 60 * 60 * 1000 }); + await writeCached(directory, 'v1-bbbb.inner.tmp-1-2'); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual(['v1-bbbb.inner.tmp-1-2']); + }); + + /** + * `/cache` is a directory on somebody's disk, and what else they keep there + * is theirs: the names this service writes are what it may take away. + */ + it('never touches a file it did not write', async () => { + const { service, directory } = await seed({ ARCHIVE_CACHE_MAX_SIZE: '1K' }); + await writeCached(directory, 'notes.txt', { size: 4096, agedMs: 400 * 24 * 60 * 60 * 1000 }); + await writeCached(directory, 'inner', { size: 4096 }); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual(['inner', 'notes.txt']); + }); + + it('passes quietly when nothing has ever been cached', async () => { + currentEnv = await setupTestEnv({ tag: 'archive-cache-' }); + const service = currentEnv.requireFresh('src/services/archiveCacheService'); + + await expect(service.sweepArchiveCache()).resolves.toBeUndefined(); + }); +}); + +/** + * A tree is a directory rather than a file, and the sweep has to read it as + * one: its size is what is under it, and taking it means taking all of it. + */ +const writeTree = async (directory, name, { files = 2, size = 1024, agedMs = 0 } = {}) => { + const tree = path.join(directory, name); + await fs.mkdir(path.join(tree, 'nested'), { recursive: true }); + for (let index = 0; index < files; index += 1) { + await fs.writeFile(path.join(tree, 'nested', `file-${index}`), Buffer.alloc(size)); + } + if (agedMs) { + const when = new Date(Date.now() - agedMs); + await fs.utimes(tree, when, when); + } + return tree; +}; + +describe('sweeping the extracted trees of solid archives', () => { + it('counts what is under a tree, not the directory entry', async () => { + const { service, directory } = await seed({ ARCHIVE_CACHE_MAX_SIZE: '3K' }); + // Six kilobytes in two files, against a budget of three. + await writeTree(directory, 'v1-aaaa.tree', { files: 2, size: 3072 }); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual([]); + }); + + it('keeps one that fits, whole', async () => { + const { service, directory } = await seed(); + await writeTree(directory, 'v1-bbbb.tree', { files: 2, size: 16 }); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual(['v1-bbbb.tree']); + expect(await fs.readdir(path.join(directory, 'v1-bbbb.tree', 'nested'))).toHaveLength(2); + }); + + it('takes one nobody has opened in a long time', async () => { + const { service, directory } = await seed(); + await writeTree(directory, 'v1-cccc.tree', { agedMs: 400 * 24 * 60 * 60 * 1000 }); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual([]); + }); + + /** Files and trees are the same cache, and share its budget. */ + it('weighs trees and copies against one budget, oldest first', async () => { + const { service, directory } = await seed({ ARCHIVE_CACHE_MAX_SIZE: '5K' }); + await writeTree(directory, 'v1-d1d1.tree', { + files: 1, + size: 4096, + agedMs: 3 * 60 * 60 * 1000, + }); + await writeCached(directory, 'v1-d2d2.inner', { size: 4096, agedMs: 60 * 60 * 1000 }); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual(['v1-d2d2.inner']); + }); + + it('takes a half-made tree a stopped run left behind', async () => { + const { service, directory } = await seed(); + await writeTree(directory, 'v1-eeee.tree.tmp-1-2', { agedMs: 3 * 60 * 60 * 1000 }); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual([]); + }); + + it('never touches a directory it did not write', async () => { + const { service, directory } = await seed({ ARCHIVE_CACHE_MAX_SIZE: '1K' }); + await writeTree(directory, 'somebody-elses-work', { files: 2, size: 4096 }); + + await service.sweepArchiveCache(); + + expect(await remaining(directory)).toEqual(['somebody-elses-work']); + }); +}); diff --git a/backend/tests/services/archive-primitives.test.js b/backend/tests/services/archive-primitives.test.js index 2ccead661..14eaadc61 100644 --- a/backend/tests/services/archive-primitives.test.js +++ b/backend/tests/services/archive-primitives.test.js @@ -141,7 +141,9 @@ describe('recognising a wrong password in what the extractor said', () => { const service = await setup(); expect(service.isArchivePasswordError(new Error('No space left on device'))).toBe(false); - expect(service.isArchivePasswordError(new Error('Cannot open the file as archive'))).toBe(false); + expect(service.isArchivePasswordError(new Error('Cannot open the file as archive'))).toBe( + false + ); }); it('says no rather than throwing on something that is not an error', async () => { diff --git a/backend/tests/services/archiveService.test.js b/backend/tests/services/archiveService.test.js index 42967d310..160fb0578 100644 --- a/backend/tests/services/archiveService.test.js +++ b/backend/tests/services/archiveService.test.js @@ -1,6 +1,5 @@ import { describe, expect, it } from 'vitest'; -// eslint-disable-next-line global-require const { normalizeArchivePassword, isArchivePasswordError, @@ -16,7 +15,9 @@ describe('archive service password handling', () => { }); it('rejects values that cannot safely be sent to the extractor prompt', () => { - expect(() => normalizeArchivePassword('line one\nline two')).toThrow('Invalid archive password.'); + expect(() => normalizeArchivePassword('line one\nline two')).toThrow( + 'Invalid archive password.' + ); expect(() => normalizeArchivePassword('tab\tpassword')).toThrow('Invalid archive password.'); expect(() => normalizeArchivePassword('x'.repeat(4097))).toThrow('Invalid archive password.'); expect(() => normalizeArchivePassword({ secret: 'nope' })).toThrow('Invalid archive password.'); diff --git a/backend/tests/services/collabora-discovery.test.js b/backend/tests/services/collabora-discovery.test.js index af300b067..5c10869a0 100644 --- a/backend/tests/services/collabora-discovery.test.js +++ b/backend/tests/services/collabora-discovery.test.js @@ -21,9 +21,7 @@ describe('Collabora Discovery Service', () => { const docx = map.get('docx'); expect(docx).toBeDefined(); - expect(docx.edit).toBe( - 'https://office.example.com/loleaflet/123/loleaflet.html?WOPISrc=' - ); + expect(docx.edit).toBe('https://office.example.com/loleaflet/123/loleaflet.html?WOPISrc='); expect(docx.view).toBe( 'https://office.example.com/loleaflet/123/loleaflet.html?permission=readonly&WOPISrc=' ); diff --git a/backend/tests/services/copy-restricted-acl.test.js b/backend/tests/services/copy-restricted-acl.test.js new file mode 100644 index 000000000..6036bed67 --- /dev/null +++ b/backend/tests/services/copy-restricted-acl.test.js @@ -0,0 +1,144 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { createRequire } from 'node:module'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +const require = createRequire(import.meta.url); + +/** + * A ZFS dataset with `aclmode=restricted` refuses the chmod that `rsync -a` + * performs after writing a file, because new files there must inherit the + * directory's ACL untouched (nxzai/NextExplorer#367). rsync copies the contents + * correctly and only then fails with exit 23. + * + * That cannot be reproduced on a test machine, so rsync is replaced by a script + * that behaves the way rsync does there: it copies, records the arguments it + * was given, and refuses to set permissions unless told not to try. + */ +const RSYNC_STUB = `#!/bin/sh +echo "$@" >> "$RSYNC_ARGS_LOG" +case "$@" in + *--no-perms*) + exit 0 + ;; + *) + echo 'rsync: [receiver] failed to set permissions on "/mnt/dest/file": Operation not permitted (1)' >&2 + exit 23 + ;; +esac +`; + +/** An exit 23 that has nothing to do with permissions must not be retried. */ +const RSYNC_STUB_OTHER_FAILURE = `#!/bin/sh +echo "$@" >> "$RSYNC_ARGS_LOG" +echo 'rsync: [sender] link_stat "/mnt/src/gone" failed: No such file or directory (2)' >&2 +exit 23 +`; + +let tmpDir; +let originalPath; +let originalPreserve; + +const installFakeRsync = async (script) => { + const binDir = path.join(tmpDir, 'bin'); + await fs.mkdir(binDir, { recursive: true }); + const rsyncPath = path.join(binDir, 'rsync'); + await fs.writeFile(rsyncPath, script, { mode: 0o755 }); + process.env.PATH = `${binDir}:${originalPath}`; + process.env.RSYNC_ARGS_LOG = path.join(tmpDir, 'args.log'); +}; + +const argsLog = async () => { + const raw = await fs.readFile(process.env.RSYNC_ARGS_LOG, 'utf8').catch(() => ''); + return raw.trim().split('\n').filter(Boolean); +}; + +const loadService = () => { + delete require.cache[require.resolve('../../src/config/env')]; + delete require.cache[require.resolve('../../src/services/fileTransferService')]; + return require('../../src/services/fileTransferService'); +}; + +describe('copying where the destination refuses a chmod', () => { + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'copy-acl-test-')); + originalPath = process.env.PATH; + originalPreserve = process.env.COPY_PRESERVE_PERMISSIONS; + }); + + afterEach(async () => { + process.env.PATH = originalPath; + if (originalPreserve === undefined) delete process.env.COPY_PRESERVE_PERMISSIONS; + else process.env.COPY_PRESERVE_PERMISSIONS = originalPreserve; + delete process.env.RSYNC_ARGS_LOG; + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + describe('deciding whether to try again', () => { + it('recognises a refused chmod', () => { + const { isPermissionPreservationFailure } = loadService(); + + expect( + isPermissionPreservationFailure( + 23, + 'rsync: [receiver] failed to set permissions on "/mnt/x": Operation not permitted (1)' + ) + ).toBe(true); + }); + + // Exit 23 covers far more than permissions — a source file that vanished + // mid-copy gets the same code, and retrying without -p would not help. + it('leaves any other partial failure alone', () => { + const { isPermissionPreservationFailure } = loadService(); + + expect( + isPermissionPreservationFailure(23, 'link_stat "/mnt/gone" failed: No such file (2)') + ).toBe(false); + expect(isPermissionPreservationFailure(1, 'failed to set permissions')).toBe(false); + expect(isPermissionPreservationFailure(0, '')).toBe(false); + expect(isPermissionPreservationFailure(23, undefined)).toBe(false); + }); + }); + + describe('the default: preserve, and fall back only where it is refused', () => { + it('copies again without preserving permissions', async () => { + await installFakeRsync(RSYNC_STUB); + const { copyWithNativeRsync } = loadService(); + + await expect(copyWithNativeRsync('/src/file', '/dest/file')).resolves.toBeUndefined(); + + const calls = await argsLog(); + expect(calls).toHaveLength(2); + // Preserving is still what is attempted first. + expect(calls[0]).not.toContain('--no-perms'); + expect(calls[1]).toContain('--no-perms'); + }); + + it('gives up on a failure that copying differently cannot fix', async () => { + await installFakeRsync(RSYNC_STUB_OTHER_FAILURE); + const { copyWithNativeRsync } = loadService(); + + await expect(copyWithNativeRsync('/src/file', '/dest/file')).rejects.toThrow(/link_stat/); + + // One attempt, not two: nothing here suggests a second would do better. + expect(await argsLog()).toHaveLength(1); + }); + }); + + describe('the permanent mode', () => { + // Where every copy would fail the same way, paying for a doomed attempt and + // a retry each time is waste. + it('never attempts to preserve permissions', async () => { + process.env.COPY_PRESERVE_PERMISSIONS = 'false'; + await installFakeRsync(RSYNC_STUB); + const { copyWithNativeRsync } = loadService(); + + await expect(copyWithNativeRsync('/src/file', '/dest/file')).resolves.toBeUndefined(); + + const calls = await argsLog(); + expect(calls).toHaveLength(1); + expect(calls[0]).toContain('--no-perms'); + }); + }); +}); diff --git a/backend/tests/services/db-single-open.test.js b/backend/tests/services/db-single-open.test.js new file mode 100644 index 000000000..4cf226b4f --- /dev/null +++ b/backend/tests/services/db-single-open.test.js @@ -0,0 +1,98 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * How many times `app.db` is opened, and how many times a statement is compiled. + * + * Everything that starts with the server asks for the database at once: the session + * store, the settings, the trash sweep, the search index, the favourites. `getDb` + * checked whether a connection was already open and opened one when it was not — and + * every caller that arrived before the first one had finished saw "not open" and opened + * another. Four connections at every start, four runs of the migrations over the same + * file in parallel, and whichever finished last became the one everybody used. + * + * The number that matters is therefore 1, and it is the constructor that is counted + * rather than anything the code says about itself. + */ + +let env; + +afterEach(async () => { + vi.restoreAllMocks(); + if (env) { + env.requireFresh('src/services/db').closeDb?.(); + await env.cleanup(); + } + env = null; +}); + +describe('opening the application database', () => { + it('happens once, however many callers ask at the same moment', async () => { + env = await setupTestEnv({ tag: 'db-single-open-' }); + const db = env.requireFresh('src/services/db'); + + // Five callers in the same turn, as the start does. + const handles = await Promise.all([db.getDb(), db.getDb(), db.getDb(), db.getDb(), db.getDb()]); + + // The same connection, not five that happen to point at the same file. + expect(new Set(handles).size).toBe(1); + }); + + it('gives every later caller the connection it already has', async () => { + env = await setupTestEnv({ tag: 'db-single-open-' }); + const db = env.requireFresh('src/services/db'); + + const first = await db.getDb(); + const second = await db.getDb(); + + expect(second).toBe(first); + }); + + it('opens again after it has been closed', async () => { + env = await setupTestEnv({ tag: 'db-single-open-' }); + const db = env.requireFresh('src/services/db'); + + const first = await db.getDb(); + db.closeDb(); + const second = await db.getDb(); + + expect(second).not.toBe(first); + // And the new one works, which a closed handle would not. + expect(second.prepare('SELECT 1 AS one').get().one).toBe(1); + }); +}); + +describe('a statement asked for twice', () => { + it('is compiled once', async () => { + env = await setupTestEnv({ tag: 'db-prepared-' }); + const db = await env.requireFresh('src/services/db').getDb(); + const { prepared } = env.requireFresh('src/services/db'); + const compile = vi.spyOn(db, 'prepare'); + + const sql = 'SELECT COUNT(*) AS total FROM users WHERE id = ?'; + const a = prepared(db, sql); + const b = prepared(db, sql); + + expect(b).toBe(a); + expect(compile).toHaveBeenCalledTimes(1); + // And it is a working statement, not a cached object that only looks like one. + expect(a.get('nobody').total).toBe(0); + }); + + it('is compiled again for a different connection', async () => { + env = await setupTestEnv({ tag: 'db-prepared-' }); + const service = env.requireFresh('src/services/db'); + const sql = 'SELECT COUNT(*) AS total FROM users'; + + const first = await service.getDb(); + const one = service.prepared(first, sql); + service.closeDb(); + const second = await service.getDb(); + const two = service.prepared(second, sql); + + // A statement belongs to the connection that compiled it; handing the old one to a + // new connection is how a cache keyed on the SQL alone breaks. + expect(two).not.toBe(one); + }); +}); diff --git a/backend/tests/services/ffmpeg-failure-reporting.test.js b/backend/tests/services/ffmpeg-failure-reporting.test.js new file mode 100644 index 000000000..4bf014227 --- /dev/null +++ b/backend/tests/services/ffmpeg-failure-reporting.test.js @@ -0,0 +1,192 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { hasFfmpeg } from '../helpers/media-tools.js'; + +/** + * A whole-file budget, against a vitest default of five seconds. + * + * Every test here waits on a spawned ffmpeg. The suite runs one worker per + * core, so on a contended machine that process is not slow — it is queued, and + * a budget sized for an idle machine turns ordinary scheduling into a failure + * that names the wrong thing. Deliberately oversubscribing the pool failed + * these in half the runs before this. + */ +vi.setConfig({ testTimeout: 30_000 }); + +/** + * Whose failure it is when a thumbnail cannot be made. + * + * Production logs carried this, four times in one minute, against ordinary + * H.264 films: + * + * "Input buffer contains unsupported image format" + * at Sharp.toFile (/app/node_modules/sharp/dist/output.cjs:90:19) + * + * Sharp's name, an image-format complaint, about a video file — and no way to + * act on it. What had actually happened is that ffmpeg failed, wrote nothing to + * standard output, and sharp was handed an empty buffer. ffmpeg's own + * explanation went to a stderr pipe nobody read. + * + * That unread pipe was the second defect and the more serious one: a pipe fills + * at 64 KB and the writer then blocks on it forever. A run that only ever + * succeeds quietly never fills it, so nothing showed until a file ffmpeg had a + * lot to say about arrived — and that file would hang rather than fail. + */ + +let ctx; + +/** + * What the service logged. + * + * A thumbnail failure is swallowed on purpose — one unreadable file must not + * stop a listing — so the log line is the only place it appears, and therefore + * the only place worth asserting. Testing a thrown error instead would be + * testing a layer nobody reads. + * + * The spy has to be attached after the environment is built, not before: the + * setup drops every application module from the require cache, so a logger + * taken at the top of the file is not the object the service will end up + * holding. + */ +let logged; + +const watchLogger = () => { + logged = []; + const logger = require('../../src/utils/logger'); + vi.spyOn(logger, 'error').mockImplementation((fields, message) => { + logged.push({ fields, message }); + }); + vi.spyOn(logger, 'warn').mockImplementation(() => {}); +}; + +const setup = async () => { + const service = await buildEnv(); + watchLogger(); + return service; +}; + +const buildEnv = async () => { + ctx = await setupTestEnv({ + tag: 'ffmpeg-failure-', + env: { THUMBNAILS: 'true' }, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/services/ffmpegRunner', + 'src/services/thumbnailService', + ], + }); + return ctx.requireFresh('src/services/thumbnailService'); +}; + +afterEach(async () => { + vi.restoreAllMocks(); + if (!ctx) return; + const service = ctx.loaded?.('src/services/thumbnailService'); + try { + await service?.stopThumbnailWork?.(); + } catch (_) { + // Nothing in flight. + } + await ctx.cleanup(); + ctx = null; +}); + +/** A file with a video's name and nothing a decoder can use inside it. */ +const writeUndecodableVideo = async (dir, name = 'broken.mp4') => { + const target = path.join(dir, name); + await fs.writeFile(target, Buffer.alloc(64 * 1024, 0x7f)); + return target; +}; + +/** + * Ask for a thumbnail that cannot be made, and return the error the service + * logged. Generation is queued, so the request returns long before the work + * fails. + */ +const failureFrom = async (service, source) => { + await service.queueThumbnailGeneration(source, { priority: 10 }); + + // Twenty-five seconds of patience, against a vitest default of five. What is + // being waited on is a spawned ffmpeg, and the suite runs one worker per + // core: on a contended machine the process is not slow, it is queued. A + // budget that matched the happy path turned that into "the service never + // reported a failure", which is the opposite of what happened. + for (let attempt = 0; attempt < 500; attempt += 1) { + const entry = logged.find(({ message }) => message === 'Thumbnail generation failed'); + if (entry) return entry.fields.err; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error('the service never reported a failure'); +}; + +describe.skipIf(!(await hasFfmpeg()))('a video ffmpeg cannot decode', () => { + it('fails rather than writing a thumbnail', async () => { + const service = await setup(); + const source = await writeUndecodableVideo(ctx.volumeDir); + + expect(await failureFrom(service, source)).toBeInstanceOf(Error); + }); + + /** + * The point of the change. Before it, this message was sharp's, and the log + * sent whoever read it looking at the image pipeline for a fault that was + * never there. + */ + it('is reported as ffmpeg failing, not as sharp failing', async () => { + const service = await setup(); + const source = await writeUndecodableVideo(ctx.volumeDir); + + const error = await failureFrom(service, source); + + expect(error.message).toMatch(/ffmpeg exited with/i); + }); + + it('says what ffmpeg exited with', async () => { + const service = await setup(); + const source = await writeUndecodableVideo(ctx.volumeDir); + + const error = await failureFrom(service, source); + + expect(error.message).toMatch(/exited with -?\d+/); + }); + + /** + * ffmpeg's own words, which are the only part of this that says what to do + * about the file. Without them the exit code alone is a number. + */ + it("carries ffmpeg's own explanation", async () => { + const service = await setup(); + const source = await writeUndecodableVideo(ctx.volumeDir); + + const error = await failureFrom(service, source); + + expect(error.message.length).toBeGreaterThan('FFmpeg exited with 1'.length); + }); + + /** The original error is kept, so nothing is lost by renaming the failure. */ + it('keeps the underlying error as its cause', async () => { + const service = await setup(); + const source = await writeUndecodableVideo(ctx.volumeDir); + + const error = await failureFrom(service, source); + + expect(error.cause).toBeInstanceOf(Error); + }); +}); + +describe.skipIf(!(await hasFfmpeg()))('a HEIC ffmpeg cannot decode', () => { + /** The second builder had the same unread pipe and the same wrong name. */ + it('is reported as ffmpeg failing too', async () => { + const service = await setup(); + const source = path.join(ctx.volumeDir, 'broken.heic'); + await fs.writeFile(source, Buffer.alloc(32 * 1024, 0x11)); + + const error = await failureFrom(service, source); + + expect(error?.message).toMatch(/ffmpeg exited with/i); + }); +}); diff --git a/backend/tests/services/ffmpeg-thumbnails.test.js b/backend/tests/services/ffmpeg-thumbnails.test.js new file mode 100644 index 000000000..16f831466 --- /dev/null +++ b/backend/tests/services/ffmpeg-thumbnails.test.js @@ -0,0 +1,208 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import sharp from 'sharp'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { ffmpegReadsHeif, hasFfmpeg, HEIC_FIXTURE } from '../helpers/media-tools.js'; + +/** + * A whole-file budget, against a vitest default of five seconds. + * + * Every test here waits on a spawned ffmpeg. The suite runs one worker per + * core, so on a contended machine that process is not slow — it is queued, and + * a budget sized for an idle machine turns ordinary scheduling into a failure + * that names the wrong thing. Deliberately oversubscribing the pool failed + * these in half the runs before this. + */ +vi.setConfig({ testTimeout: 30_000 }); + +const execFileAsync = promisify(execFile); + +/** + * Video and HEIC thumbnails, made from real files by a real ffmpeg. + * + * `fluent-ffmpeg` was removed and its seven calls replaced with plain process + * spawning. The whole suite stayed green through that change, which proves + * nothing at all: not one test decoded a frame. A builder API swapped for an + * argument list can produce a command that runs, exits zero and writes nothing, + * and the only visible symptom is a thumbnail that never appears. + * + * So this encodes a clip, asks the service for a thumbnail, and looks at the + * pixels that come out. Skipped where ffmpeg is absent, which is stated rather + * than silent. + */ + +let ctx; + +// Asked once, at load: skipIf needs the answer before the tests are declared. +const ffmpeg = await hasFfmpeg(); +const heif = ffmpeg && (await ffmpegReadsHeif()); + +const setup = async (env = {}) => { + ctx = await setupTestEnv({ + tag: 'ffmpeg-thumbs-', + env: { THUMBNAILS: 'true', ...env }, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/services/ffmpegRunner', + 'src/services/thumbnailService', + ], + }); + return ctx; +}; + +afterEach(async () => { + if (ctx) { + const service = ctx.loaded?.('src/services/thumbnailService'); + try { + await service?.stopThumbnailWork?.(); + } catch (_) { + // Nothing in flight. + } + await ctx.cleanup(); + ctx = null; + } +}); + +/** + * An eight-second clip whose left half is red and right half is blue. + * + * Longer than the default seek point of five seconds on purpose: a clip shorter + * than the seek yields no frame at all, which looks exactly like a broken + * decoder and is how the first version of this file failed. + */ +const makeClip = async (file, args = []) => { + await execFileAsync('ffmpeg', [ + '-hide_banner', + '-loglevel', + 'error', + '-y', + '-f', + 'lavfi', + '-i', + 'color=c=red:size=64x64:duration=8:rate=10', + '-f', + 'lavfi', + '-i', + 'color=c=blue:size=64x64:duration=8:rate=10', + '-filter_complex', + '[0:v][1:v]hstack=inputs=2', + ...args, + file, + ]); +}; + +/** + * Ask for a thumbnail and wait for the file. Generation is queued, so the call + * returns before the work is done. + */ +const thumbnailFor = async (env, service, source) => { + const result = await service.queueThumbnailGeneration(source, { priority: 10 }); + const thumbDir = path.join(env.cacheDir, 'thumbnails'); + // Long enough to survive a contended machine: this waits on a spawned + // ffmpeg, which under load is queued rather than slow. + for (let attempt = 0; attempt < 500; attempt += 1) { + const entries = await fs.readdir(thumbDir).catch(() => []); + const done = entries.filter((name) => name.endsWith('.webp')); + if (done.length) return path.join(thumbDir, done[0]); + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error(`no thumbnail appeared (queue said ${JSON.stringify(result)})`); +}; + +const colourAt = async (file, x, y) => { + const { data, info } = await sharp(file).raw().toBuffer({ resolveWithObject: true }); + const i = (y * info.width + x) * info.channels; + return { r: data[i], g: data[i + 1], b: data[i + 2] }; +}; + +describe('a video thumbnail', () => { + it.skipIf(!ffmpeg)('is produced from a real clip', async () => { + const env = await setup(); + const service = env.requireFresh('src/services/thumbnailService'); + const source = path.join(env.volumeDir, 'clip.mp4'); + await makeClip(source, ['-c:v', 'libx264', '-pix_fmt', 'yuv420p']); + + const thumb = await thumbnailFor(env, service, source); + + expect((await fs.stat(thumb)).size).toBeGreaterThan(0); + }); + + /** + * The assertion an argument-list mistake cannot survive. A command that runs + * and writes nothing useful still exits zero; a picture with red on the left + * and blue on the right came from decoding the actual frame. + */ + it.skipIf(!ffmpeg)('holds the picture that was in the clip', async () => { + const env = await setup(); + const service = env.requireFresh('src/services/thumbnailService'); + const source = path.join(env.volumeDir, 'clip.mp4'); + await makeClip(source, ['-c:v', 'libx264', '-pix_fmt', 'yuv420p']); + + const thumb = await thumbnailFor(env, service, source); + const meta = await sharp(thumb).metadata(); + const left = await colourAt(thumb, 4, Math.floor(meta.height / 2)); + const right = await colourAt(thumb, meta.width - 4, Math.floor(meta.height / 2)); + + expect(meta.format).toBe('webp'); + expect(left.r).toBeGreaterThan(140); + expect(left.b).toBeLessThan(90); + expect(right.b).toBeGreaterThan(140); + expect(right.r).toBeLessThan(90); + }); + + it.skipIf(!ffmpeg)('works for a container ffmpeg has to seek into', async () => { + const env = await setup(); + const service = env.requireFresh('src/services/thumbnailService'); + const source = path.join(env.volumeDir, 'clip.mkv'); + await makeClip(source, ['-c:v', 'libx264', '-pix_fmt', 'yuv420p']); + + await expect(thumbnailFor(env, service, source)).resolves.toBeTruthy(); + }); + + /** + * Seeking by a percentage is the branch that needs ffprobe: the duration has + * to be read before the seek point can be worked out. It is a separate code + * path from the fixed seek, and the one that silently falls back. + */ + it.skipIf(!ffmpeg)('seeks by percentage, which means ffprobe answered', async () => { + const env = await setup({ THUMBNAIL_VIDEO_SEEK_PERCENT: '0.5' }); + const service = env.requireFresh('src/services/thumbnailService'); + const source = path.join(env.volumeDir, 'clip.mp4'); + await makeClip(source, ['-c:v', 'libx264', '-pix_fmt', 'yuv420p']); + + await expect(thumbnailFor(env, service, source)).resolves.toBeTruthy(); + }); + + it.skipIf(!ffmpeg)('gives up quietly on a file that is not a video at all', async () => { + const env = await setup(); + const service = env.requireFresh('src/services/thumbnailService'); + const source = path.join(env.volumeDir, 'broken.mp4'); + await fs.writeFile(source, Buffer.from('not a video')); + + await expect(service.queueThumbnailGeneration(source, { priority: 10 })).resolves.toBeDefined(); + }); +}); + +describe('a HEIC thumbnail', () => { + // The same ffmpeg has to be new enough for HEIF (7.1); older ones cannot open it. + it.skipIf(!heif)('is produced, and holds the picture', async () => { + const env = await setup(); + const service = env.requireFresh('src/services/thumbnailService'); + const source = path.join(env.volumeDir, 'photo.heic'); + await fs.copyFile(HEIC_FIXTURE, source); + + const thumb = await thumbnailFor(env, service, source); + + const meta = await sharp(thumb).metadata(); + const left = await colourAt(thumb, 3, Math.floor(meta.height / 2)); + const right = await colourAt(thumb, meta.width - 3, Math.floor(meta.height / 2)); + + expect(left.r).toBeGreaterThan(140); + expect(right.b).toBeGreaterThan(140); + }); +}); diff --git a/backend/tests/services/file-transfer-cancel.test.js b/backend/tests/services/file-transfer-cancel.test.js new file mode 100644 index 000000000..3bec65097 --- /dev/null +++ b/backend/tests/services/file-transfer-cancel.test.js @@ -0,0 +1,255 @@ +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import fsSync from 'node:fs'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +let envContext; + +beforeAll(async () => { + envContext = await setupTestEnv({ + tag: 'file-transfer-cancel-test-', + env: { FOLDER_SIZE_MODE: 'off' }, + modules: [ + 'src/services/fileTransferService', + 'src/services/folderSizeHooks', + 'src/config/env', + 'src/config/index', + 'src/services/accessManager', + 'src/services/users', + ], + }); +}); + +afterAll(async () => { + await envContext.cleanup(); +}); + +describe('Transfer cancellation', () => { + it('removes the partial target and keeps the source when a copy is cancelled', async () => { + const { executeTransfer } = envContext.requireFresh('src/services/fileTransferService'); + const sourceDir = path.join(envContext.tmpRoot, 'source'); + const destinationDir = path.join(envContext.tmpRoot, 'destination'); + const sourcePath = path.join(sourceDir, 'large.bin'); + const destinationPath = path.join(destinationDir, 'large.bin'); + + await fs.mkdir(sourceDir, { recursive: true }); + await fs.writeFile(sourcePath, Buffer.alloc(4 * 1024 * 1024, 7)); + await fs.mkdir(destinationDir, { recursive: true }); + + const controller = new AbortController(); + const prep = { + destinationRelative: 'destination', + destinationAbsolute: destinationDir, + totalBytes: 4 * 1024 * 1024, + plans: [ + { + sourceAbsolute: sourcePath, + sourceRelative: 'source/large.bin', + isDirectory: false, + size: 4 * 1024 * 1024, + desiredName: 'large.bin', + }, + ], + }; + + await expect( + executeTransfer( + prep, + 'copy', + ({ copiedBytes }) => { + if (copiedBytes > 0) controller.abort(); + }, + { signal: controller.signal } + ) + ).rejects.toMatchObject({ code: 'OPERATION_CANCELLED' }); + + await expect(fs.stat(sourcePath)).resolves.toMatchObject({ size: 4 * 1024 * 1024 }); + await expect(fs.stat(destinationPath)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + /** + * A copy writes under a hidden name inside the folder it goes to, so nothing + * is under its own name to delete while it runs. Deleting the folder it is + * writing into stops it first, and waits for it to remove its hidden entry. + */ + it('cancels a copy before deleting the folder it is writing into', async () => { + const { executeTransfer, deleteItems } = envContext.requireFresh( + 'src/services/fileTransferService' + ); + const usersService = envContext.requireFresh('src/services/users'); + const user = await usersService.createLocalUser({ + email: 'transfer-delete@example.com', + username: 'transfer-delete', + displayName: 'Transfer Delete', + password: 'secret123', + roles: ['admin'], + }); + const sourceDir = path.join(envContext.volumeDir, 'Nvm', 'source'); + const destinationDir = path.join(envContext.volumeDir, 'Nvm', 'destination'); + const sourcePath = path.join(sourceDir, 'active'); + const destinationPath = path.join(destinationDir, 'active'); + + await fs.mkdir(sourcePath, { recursive: true }); + await fs.mkdir(destinationDir, { recursive: true }); + await Promise.all( + Array.from({ length: 4 }, (_, index) => + fs.writeFile( + path.join(sourcePath, `part-${index}.bin`), + Buffer.alloc(8 * 1024 * 1024, index) + ) + ) + ); + + const prep = { + destinationRelative: 'Nvm/destination', + destinationAbsolute: destinationDir, + totalBytes: 32 * 1024 * 1024, + plans: [ + { + sourceAbsolute: sourcePath, + sourceRelative: 'Nvm/source/active', + isDirectory: true, + size: 32 * 1024 * 1024, + desiredName: 'active', + }, + ], + }; + let deletion; + let seenWhileWriting = null; + const transfer = executeTransfer(prep, 'copy', ({ copiedBytes }) => { + if (copiedBytes > 0 && !deletion) { + seenWhileWriting = fsSync.readdirSync(destinationDir); + // For good: what is under test is the wait for the writer, not the trash. + deletion = deleteItems([{ path: 'Nvm', name: 'destination', kind: 'directory' }], { + user, + permanent: true, + }); + } + }); + + await expect(transfer).rejects.toMatchObject({ code: 'OPERATION_CANCELLED' }); + await expect(deletion).resolves.toMatchObject([{ path: 'Nvm/destination', status: 'deleted' }]); + expect(seenWhileWriting).toHaveLength(1); + expect(seenWhileWriting[0]).toMatch(/^\.nextexplorer-copying-/); + await expect(fs.stat(sourcePath)).resolves.toMatchObject({ isDirectory: expect.any(Function) }); + await expect(fs.stat(destinationPath)).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(fs.stat(destinationDir)).rejects.toMatchObject({ code: 'ENOENT' }); + }); +}); + +/** + * Removing a plain file used to fork `rm -rf` for it, on Linux only — which is + * to say in the container. That is ~1.2 ms of process setup against ~0.06 ms + * of actual work, so a two-thousand-file selection spent over two seconds + * doing nothing but starting processes. A directory still earns its fork: the + * recursion runs natively and killing the process cancels it. + */ +describe('Native removal', () => { + it('never forks for a single file, even where the native path exists', async () => { + const { shouldRemoveNatively } = await import('../../src/services/fileTransferService.js'); + + // The flag is passed explicitly: it is false on anything but Linux, and a + // test that only ever sees false would pass without checking anything. + expect(shouldRemoveNatively(false, true)).toBe(false); + }); + + it('keeps the native path for directories', async () => { + const { shouldRemoveNatively } = await import('../../src/services/fileTransferService.js'); + + expect(shouldRemoveNatively(true, true)).toBe(true); + expect(shouldRemoveNatively(true, false)).toBe(false); + }); +}); + +/** + * Removals run several at a time, because on network storage each one is + * mostly waiting. Overlapping them must not change what the caller sees: the + * results stay in the order they were asked for, the progress counter only + * ever moves forward, and cancelling still stops the operation. + */ +describe('Bulk deletion', () => { + let bulkEnv; + + afterEach(async () => { + if (bulkEnv) { + await bulkEnv.cleanup(); + bulkEnv = null; + } + }); + + const seed = async (count) => { + bulkEnv = await setupTestEnv({ + tag: 'bulk-delete-', + modules: [ + 'src/config/env', + 'src/config/index', + 'src/utils/pathUtils', + 'src/services/db', + 'src/services/users', + 'src/services/sharesService', + 'src/services/accessManager', + 'src/services/authorizationService', + 'src/services/fileTransferService', + ], + }); + + const dir = path.join(bulkEnv.volumeDir, 'bulk'); + await fs.mkdir(dir, { recursive: true }); + const items = []; + for (let i = 0; i < count; i += 1) { + await fs.writeFile(path.join(dir, `f${i}.bin`), 'x'); + items.push({ path: 'bulk', name: `f${i}.bin` }); + } + return { items, service: bulkEnv.requireFresh('src/services/fileTransferService') }; + }; + + it('answers in the order it was asked, with a counter that only grows', async () => { + const { items, service } = await seed(60); + const user = { id: 'u1', roles: ['admin'] }; + + const counts = []; + const results = await service.deleteItems(items, { + user, + guestSession: null, + onProgress: (p) => counts.push(p.completedItems), + }); + + expect(results).toHaveLength(60); + // Out-of-order completion must not leak into the answer. + results.forEach((result, index) => { + expect(result.path).toContain(`f${index}.bin`); + expect(result.status).toBe('trashed'); + }); + expect(counts).toEqual([...counts].sort((a, b) => a - b)); + expect(counts.at(-1)).toBe(60); + + // Only the trash's own reserved folder is left: `bulk` is a volume, and + // what was deleted from it is kept there. + const remaining = await fs.readdir(path.join(bulkEnv.volumeDir, 'bulk')); + expect(remaining).toEqual(['.nextexplorer']); + }); + + it('stops when cancelled', async () => { + const { items, service } = await seed(60); + const controller = new AbortController(); + + const deletion = service.deleteItems(items, { + user: { id: 'u1', roles: ['admin'] }, + guestSession: null, + signal: controller.signal, + onProgress: (p) => { + if (p.completedItems >= 8) controller.abort(); + }, + }); + + await expect(deletion).rejects.toThrow(); + // Cancelling means "stop", not "undo": some files are already gone. The + // trash's reserved folder is not one of the files. + const remaining = (await fs.readdir(path.join(bulkEnv.volumeDir, 'bulk'))).filter( + (name) => name !== '.nextexplorer' + ); + expect(remaining.length).toBeGreaterThan(0); + expect(remaining.length).toBeLessThan(60); + }); +}); diff --git a/backend/tests/services/file-transfer-conflicts.test.js b/backend/tests/services/file-transfer-conflicts.test.js new file mode 100644 index 000000000..5b9d5285a --- /dev/null +++ b/backend/tests/services/file-transfer-conflicts.test.js @@ -0,0 +1,457 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Where a copy or a move can lose or duplicate somebody's files. + * + * The transfer service has two jobs that a defect turns into data loss: it must + * never write over something already at the destination, and a move must never + * remove the source until the copy that replaces it is complete. Around those + * sit the refusals that keep a folder from swallowing itself, the skip that + * makes moving a thing onto its own shelf a no-op, and the authorization that + * decides a caller may write where they are pointing at all. These exercise the + * states a bug would reach, through the real service and real temporary + * directories rather than assumptions about what it does. + * + * The engine is pinned to `stream` so the in-application copy — the one that + * handles symbolic links, file modes and byte-by-byte progress — runs on every + * machine, rather than only where the native `rsync` path is unavailable. + */ + +let currentEnv; + +const insertAdmin = async (env) => { + const db = await env.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('admin', 'admin@example.com', 1, 'admin', 'Admin', '["admin"]', ?, ?)` + ).run(now, now); + return { id: 'admin', roles: ['admin'] }; +}; + +const setup = async () => { + currentEnv = await setupTestEnv({ + tag: 'file-transfer-conflicts-', + env: { FILE_TRANSFER_ENGINE: 'stream', FOLDER_SIZE_MODE: 'off' }, + }); + const service = currentEnv.requireFresh('src/services/fileTransferService'); + const user = await insertAdmin(currentEnv); + return { service, volume: currentEnv.volumeDir, user }; +}; + +const exists = (target) => + fs.lstat(target).then( + () => true, + () => false + ); + +/** Run a transfer end to end, the way the route does: prepare, then execute. */ +const runTransfer = async (service, items, destination, operation, options = {}) => { + const { user, onProgress, signal } = options; + const prep = await service.prepareTransfer(items, destination, operation, { user }); + return service.executeTransfer(prep, operation, onProgress, { signal }); +}; + +afterEach(async () => { + vi.restoreAllMocks(); + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +describe('choosing a name at the destination', () => { + it('suffixes a copy whose name is already taken and leaves the existing file untouched', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Source'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Dest'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Source', 'note.txt'), 'the newcomer'); + await fs.writeFile(path.join(volume, 'Dest', 'note.txt'), 'the incumbent'); + + const result = await runTransfer( + service, + [{ path: 'Source', name: 'note.txt' }], + 'Dest', + 'copy', + { user } + ); + + // The file that was already there keeps its name and its contents. + expect(await fs.readFile(path.join(volume, 'Dest', 'note.txt'), 'utf8')).toBe('the incumbent'); + // The copy lands beside it under a suffixed name, so nothing is overwritten. + expect(result.items[0].to).toBe('Dest/note (1).txt'); + expect(await fs.readFile(path.join(volume, 'Dest', 'note (1).txt'), 'utf8')).toBe( + 'the newcomer' + ); + }); + + it('duplicates a copy into the source folder itself rather than skipping it', async () => { + // A move onto its own shelf is a no-op; a copy is a genuine duplicate. The + // two must not be conflated — the skip belongs to move alone. + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Here'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Here', 'file.txt'), 'body'); + + const result = await runTransfer( + service, + [{ path: 'Here', name: 'file.txt' }], + 'Here', + 'copy', + { user } + ); + + expect(result.items[0].skipped).toBeUndefined(); + expect(result.items[0].to).toBe('Here/file (1).txt'); + expect((await fs.readdir(path.join(volume, 'Here'))).sort()).toEqual([ + 'file (1).txt', + 'file.txt', + ]); + }); +}); + +describe('a folder that would contain itself', () => { + it('refuses to copy a folder into one of its own subfolders, and says why', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Nvm', 'Tree', 'inner'), { recursive: true }); + + await expect( + service.prepareTransfer([{ path: 'Nvm', name: 'Tree' }], 'Nvm/Tree/inner', 'copy', { user }) + ).rejects.toThrow(/into itself/i); + }); + + it('refuses to move a folder into one of its own subfolders', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Nvm', 'Tree', 'inner'), { recursive: true }); + + await expect( + service.prepareTransfer([{ path: 'Nvm', name: 'Tree' }], 'Nvm/Tree/inner', 'move', { user }) + ).rejects.toThrow(/into itself/i); + // Nothing was moved: the folder and its subfolder are both still there. + expect(await exists(path.join(volume, 'Nvm', 'Tree', 'inner'))).toBe(true); + }); + + it('refuses when the destination is the folder itself', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Nvm', 'Solo'), { recursive: true }); + + await expect( + service.prepareTransfer([{ path: 'Nvm', name: 'Solo' }], 'Nvm/Solo', 'copy', { user }) + ).rejects.toThrow(/into itself/i); + }); +}); + +describe('a move onto the place it already is', () => { + it('skips a move whose destination is the source folder, leaving the file exactly where it was', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Shelf'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Shelf', 'keep.txt'), 'unchanged'); + + const result = await runTransfer( + service, + [{ path: 'Shelf', name: 'keep.txt' }], + 'Shelf', + 'move', + { user } + ); + + expect(result.items[0]).toMatchObject({ skipped: true }); + // The file is neither moved nor duplicated: one file, same contents. + expect(await fs.readdir(path.join(volume, 'Shelf'))).toEqual(['keep.txt']); + expect(await fs.readFile(path.join(volume, 'Shelf', 'keep.txt'), 'utf8')).toBe('unchanged'); + }); +}); + +describe('a move across devices', () => { + // A same-filesystem move is an atomic rename. When the destination is on + // another device the rename fails with EXDEV and the move becomes a copy + // followed by a deletion of the source. That fallback is the one that can + // lose data, so it is forced here by making the move's own rename, or link + // for a file, report EXDEV. Only those: the copy it falls back to is written + // under a hidden name and put in place the same way, on one device. + const forceCrossDevice = (movedFrom) => { + const fsp = require('fs/promises'); + const crossDevice = () => { + const error = new Error('cross-device link not permitted'); + error.code = 'EXDEV'; + return error; + }; + for (const method of ['rename', 'link']) { + const real = fsp[method].bind(fsp); + vi.spyOn(fsp, method).mockImplementation(async (from, to) => { + if (from !== movedFrom) return real(from, to); + throw crossDevice(); + }); + } + }; + + it('copies the entry across and only then removes the source', async () => { + const { service, volume, user } = await setup(); + forceCrossDevice(path.join(volume, 'From', 'payload.bin')); + await fs.mkdir(path.join(volume, 'From'), { recursive: true }); + await fs.mkdir(path.join(volume, 'To'), { recursive: true }); + await fs.writeFile(path.join(volume, 'From', 'payload.bin'), 'the only copy'); + + const result = await runTransfer( + service, + [{ path: 'From', name: 'payload.bin' }], + 'To', + 'move', + { user } + ); + + expect(result.items[0]).toMatchObject({ from: 'From/payload.bin', to: 'To/payload.bin' }); + expect(await fs.readFile(path.join(volume, 'To', 'payload.bin'), 'utf8')).toBe('the only copy'); + // The source is gone only because the copy is complete. + expect(await exists(path.join(volume, 'From', 'payload.bin'))).toBe(false); + }); + + it('leaves the source whole and removes the half-written destination when cancelled mid-copy', async () => { + const { service, volume, user } = await setup(); + forceCrossDevice(path.join(volume, 'From', 'big')); + await fs.mkdir(path.join(volume, 'From', 'big'), { recursive: true }); + await fs.mkdir(path.join(volume, 'To'), { recursive: true }); + await Promise.all( + Array.from({ length: 4 }, (_, index) => + fs.writeFile( + path.join(volume, 'From', 'big', `part-${index}.bin`), + Buffer.alloc(2 * 1024 * 1024, index) + ) + ) + ); + + const controller = new AbortController(); + await expect( + runTransfer(service, [{ path: 'From', name: 'big' }], 'To', 'move', { + user, + signal: controller.signal, + onProgress: ({ copiedBytes }) => { + if (copiedBytes > 0) controller.abort(); + }, + }) + ).rejects.toMatchObject({ code: 'OPERATION_CANCELLED' }); + + // The move never got as far as deleting the source, and the partial copy + // was cleaned up: the only intact copy of the folder is the original. + expect(await exists(path.join(volume, 'From', 'big', 'part-0.bin'))).toBe(true); + expect(await exists(path.join(volume, 'To', 'big'))).toBe(false); + }); +}); + +describe('copying the contents of a folder with the in-application engine', () => { + it('copies a symbolic link inside a folder as a link, not as the file it points at', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Nvm', 'Linked'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Nvm', 'Dest'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Nvm', 'Linked', 'target.txt'), 'real contents'); + await fs.symlink('target.txt', path.join(volume, 'Nvm', 'Linked', 'alias.txt')); + + await runTransfer(service, [{ path: 'Nvm', name: 'Linked' }], 'Nvm/Dest', 'copy', { user }); + + const copiedAlias = await fs.lstat(path.join(volume, 'Nvm', 'Dest', 'Linked', 'alias.txt')); + expect(copiedAlias.isSymbolicLink()).toBe(true); + expect(await fs.readlink(path.join(volume, 'Nvm', 'Dest', 'Linked', 'alias.txt'))).toBe( + 'target.txt' + ); + }); + + it('preserves the mode of a copied file', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Bin'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Dest'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Bin', 'run.sh'), '#!/bin/sh\necho hi\n'); + await fs.chmod(path.join(volume, 'Bin', 'run.sh'), 0o750); + + await runTransfer(service, [{ path: 'Bin', name: 'run.sh' }], 'Dest', 'copy', { user }); + + const copied = await fs.lstat(path.join(volume, 'Dest', 'run.sh')); + expect(copied.mode & 0o777).toBe(0o750); + }); +}); + +describe('reporting how much has been copied', () => { + it('reports a byte count that climbs to the size of the whole tree', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Nvm', 'Payload', 'nested'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Nvm', 'Dest'), { recursive: true }); + const first = Buffer.alloc(3 * 1024 * 1024, 1); + const second = Buffer.alloc(2 * 1024 * 1024, 2); + await fs.writeFile(path.join(volume, 'Nvm', 'Payload', 'a.bin'), first); + await fs.writeFile(path.join(volume, 'Nvm', 'Payload', 'nested', 'b.bin'), second); + const treeBytes = first.length + second.length; + + const observed = []; + await runTransfer(service, [{ path: 'Nvm', name: 'Payload' }], 'Nvm/Dest', 'copy', { + user, + onProgress: ({ copiedBytes }) => observed.push(copiedBytes), + }); + + expect(observed.length).toBeGreaterThan(0); + // Never runs backwards, and finishes reporting exactly what was copied. + expect(observed).toEqual([...observed].sort((a, b) => a - b)); + expect(Math.max(...observed)).toBe(treeBytes); + }); +}); + +describe('cancelling part-way through a selection', () => { + it('keeps the entries already finished and removes only the one in flight', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Src'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Dest'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Src', 'done.txt'), 'finished first'); + await fs.writeFile(path.join(volume, 'Src', 'big.bin'), Buffer.alloc(8 * 1024 * 1024, 9)); + + const controller = new AbortController(); + await expect( + runTransfer( + service, + [ + { path: 'Src', name: 'done.txt' }, + { path: 'Src', name: 'big.bin' }, + ], + 'Dest', + 'copy', + { + user, + signal: controller.signal, + onProgress: ({ currentName }) => { + // The first entry is processed to completion before the second is + // named; abort the moment the second one begins. + if (currentName === 'big.bin') controller.abort(); + }, + } + ) + ).rejects.toMatchObject({ code: 'OPERATION_CANCELLED' }); + + // The completed entry survives; the interrupted one is removed, not left half-written. + expect(await fs.readFile(path.join(volume, 'Dest', 'done.txt'), 'utf8')).toBe('finished first'); + expect(await exists(path.join(volume, 'Dest', 'big.bin'))).toBe(false); + // Both sources are untouched by a copy. + expect(await exists(path.join(volume, 'Src', 'done.txt'))).toBe(true); + expect(await exists(path.join(volume, 'Src', 'big.bin'))).toBe(true); + }); +}); + +describe('authorization at the source and the destination', () => { + const setupWithAcl = async (rules) => { + currentEnv = await setupTestEnv({ + tag: 'file-transfer-acl-', + env: { FILE_TRANSFER_ENGINE: 'stream', FOLDER_SIZE_MODE: 'off' }, + }); + const accessControl = currentEnv.requireFresh('src/services/accessControlService'); + await accessControl.setRules(rules); + const service = currentEnv.requireFresh('src/services/fileTransferService'); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('user', 'user@example.com', 1, 'user', 'User', '["user"]', ?, ?)` + ).run(now, now); + return { service, volume: currentEnv.volumeDir, user: { id: 'user', roles: ['user'] } }; + }; + + it('refuses to move a file out of a read-only folder, saying why rather than only that it failed', async () => { + const { service, volume, user } = await setupWithAcl([ + { path: '/Locked', permissions: 'ro', recursive: true }, + ]); + await fs.mkdir(path.join(volume, 'Locked'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Open'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Locked', 'note.txt'), 'read only'); + + await expect( + service.prepareTransfer([{ path: 'Locked', name: 'note.txt' }], 'Open', 'move', { user }) + ).rejects.toThrow(/move items from this path/i); + // The file stays put: a refused move must not have removed the source. + expect(await exists(path.join(volume, 'Locked', 'note.txt'))).toBe(true); + }); + + it('refuses to copy into a read-only destination, saying why', async () => { + const { service, volume, user } = await setupWithAcl([ + { path: '/Locked', permissions: 'ro', recursive: true }, + ]); + await fs.mkdir(path.join(volume, 'Locked'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Open'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Open', 'note.txt'), 'contents'); + + await expect( + service.prepareTransfer([{ path: 'Open', name: 'note.txt' }], 'Locked', 'copy', { user }) + ).rejects.toThrow(/create items in the destination/i); + expect(await exists(path.join(volume, 'Locked', 'note.txt'))).toBe(false); + }); + + it('refuses to copy a folder into a place where folders may be made but files may not', async () => { + // A copied folder carries files, so the folder-creation right alone must not + // become a way past the file-creation restriction. The share hands out one + // without the other. + currentEnv = await setupTestEnv({ + tag: 'file-transfer-share-', + env: { FILE_TRANSFER_ENGINE: 'stream', FOLDER_SIZE_MODE: 'off' }, + }); + const service = currentEnv.requireFresh('src/services/fileTransferService'); + const sharesService = currentEnv.requireFresh('src/services/sharesService'); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('owner', 'owner@example.com', 1, 'owner', 'Owner', '["admin"]', ?, ?)` + ).run(now, now); + + const volume = currentEnv.volumeDir; + await fs.mkdir(path.join(volume, 'Space', 'folder', 'inside'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Space', 'folder', 'inside', 'child.txt'), 'child'); + + const share = await sharesService.createShare({ + ownerId: 'owner', + sourceSpace: 'volume', + sourcePath: 'Space', + isDirectory: true, + accessMode: 'readwrite', + allowCreateFolder: true, + allowCreateFile: false, + sharingType: 'anyone', + }); + const guestSession = { shareId: share.id }; + const token = share.shareToken; + + await expect( + service.prepareTransfer( + [{ path: `share/${token}`, name: 'folder' }], + `share/${token}`, + 'copy', + { guestSession } + ) + ).rejects.toThrow(/create files in the destination/i); + }); +}); + +describe('the reserved trash zone', () => { + it('refuses the .nextexplorer zone as a destination', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Vol', '.nextexplorer'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Vol', 'file.txt'), 'contents'); + + await expect( + service.prepareTransfer([{ path: 'Vol', name: 'file.txt' }], 'Vol/.nextexplorer', 'copy', { + user, + }) + ).rejects.toThrow(/reserved by the application/i); + }); + + it('refuses a path inside the .nextexplorer zone as a source', async () => { + const { service, volume, user } = await setup(); + await fs.mkdir(path.join(volume, 'Vol', '.nextexplorer', 'trash'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Dest'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Vol', '.nextexplorer', 'trash', 'buried.txt'), 'hidden'); + + await expect( + service.prepareTransfer( + [{ path: 'Vol/.nextexplorer/trash', name: 'buried.txt' }], + 'Dest', + 'copy', + { user } + ) + ).rejects.toThrow(/reserved by the application/i); + }); +}); diff --git a/backend/tests/services/file-transfer-engines.test.js b/backend/tests/services/file-transfer-engines.test.js index dbcd5e85e..49bb736a9 100644 --- a/backend/tests/services/file-transfer-engines.test.js +++ b/backend/tests/services/file-transfer-engines.test.js @@ -62,7 +62,7 @@ const copyWith = async (engine, name) => { process.env.FILE_TRANSFER_ENGINE = engine; const source = path.join(env.tmpRoot, 'source'); const destination = path.join(env.tmpRoot, name); - await transfer.copyEntry(source, destination, true); + await transfer.copyEntryWithProgress(source, destination, true); return describeTree(destination); }; @@ -109,7 +109,7 @@ describe('copying a folder', () => { process.env.FILE_TRANSFER_ENGINE = 'native'; const destination = path.join(env.tmpRoot, 'timed'); - await transfer.copyEntry(source, destination, true); + await transfer.copyEntryWithProgress(source, destination, true); const copiedTime = (await fs.stat(path.join(destination, 'top.txt'))).mtimeMs; expect(Math.abs(copiedTime - sourceTime)).toBeLessThan(1000); diff --git a/backend/tests/services/file-transfer-indexed-size.test.js b/backend/tests/services/file-transfer-indexed-size.test.js new file mode 100644 index 000000000..4a2a76189 --- /dev/null +++ b/backend/tests/services/file-transfer-indexed-size.test.js @@ -0,0 +1,74 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What a copy of a folder announces it will copy, when folder sizes are on. + * + * The total a transfer reports progress against comes from the folder-size + * index rather than a walk of the folder, which is the point of having one: a + * folder of a million files is not read twice, once to count it. The index is + * a database of its own, so this is also where a transfer asking the wrong + * database would show — not as a wrong total, but as a copy that fails before + * it starts, on a table that is not there. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const ADMIN = { id: 'admin', roles: ['admin'] }; + +const setup = async () => { + currentEnv = await setupTestEnv({ + tag: 'file-transfer-indexed-size-', + env: { FILE_TRANSFER_ENGINE: 'stream', FOLDER_SIZE_MODE: 'full' }, + }); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('admin', 'admin@example.com', 1, 'admin', 'Admin', '["admin"]', ?, ?)` + ).run(now, now); + + const volume = currentEnv.volumeDir; + await fs.mkdir(path.join(volume, 'Vol', 'Big'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Vol', 'Dest'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Vol', 'Big', 'payload.bin'), Buffer.alloc(10)); + + // The index and its helpers before the service, so that the service is + // loaded against these instances and this connection. + const index = await currentEnv.requireFresh('src/services/indexDb').getIndexDb(); + const folderSizeIndex = currentEnv.requireFresh('src/services/folderSizeIndex'); + const scope = currentEnv.requireFresh('src/services/folderSizeIndexer').getVolumeScope(); + const service = currentEnv.requireFresh('src/services/fileTransferService'); + return { service, index, folderSizeIndex, scope, volume, now }; +}; + +describe('copying a folder whose size is indexed', () => { + it('announces the size the index holds, and copies', async () => { + const { service, index, folderSizeIndex, scope, volume, now } = await setup(); + // Deliberately not the size on disk: a total of 7777 can only have been + // read from the index. + folderSizeIndex.upsertScanEntry(index, scope, { + absolutePath: path.join(volume, 'Vol', 'Big'), + sizeBytes: 7777, + entryCount: 1, + lastFullScanAt: now, + }); + + const prep = await service.prepareTransfer([{ path: 'Vol', name: 'Big' }], 'Vol/Dest', 'copy', { + user: ADMIN, + }); + expect(prep.totalBytes).toBe(7777); + + await service.executeTransfer(prep, 'copy', undefined, {}); + expect(await fs.readdir(path.join(volume, 'Vol', 'Dest', 'Big'))).toEqual(['payload.bin']); + }); +}); diff --git a/backend/tests/services/file-transfer-no-overwrite.test.js b/backend/tests/services/file-transfer-no-overwrite.test.js index e0e5ae81b..509a1f865 100644 --- a/backend/tests/services/file-transfer-no-overwrite.test.js +++ b/backend/tests/services/file-transfer-no-overwrite.test.js @@ -242,13 +242,17 @@ describe('a move to another disk', () => { it('keeps the source when the copy fails, and leaves nothing half-written', async () => { await setup(); seedAlbum(); - const cp = fsp.cp.bind(fsp); + // The copy reads and writes file by file, reporting as it goes, so the + // disk fills on the first file it opens rather than inside one `fs.cp`. + const open = fsp.open.bind(fsp); vi.spyOn(fsp, 'rename').mockRejectedValue( Object.assign(new Error('cross-device link not permitted'), { code: 'EXDEV' }) ); - vi.spyOn(fsp, 'cp').mockImplementation(async (from, to, options) => { - await cp(from, to, options); - throw Object.assign(new Error('No space left on device'), { code: 'ENOSPC' }); + vi.spyOn(fsp, 'open').mockImplementation(async (target, flags, mode) => { + if (String(flags) === 'wx') { + throw Object.assign(new Error('No space left on device'), { code: 'ENOSPC' }); + } + return open(target, flags, mode); }); await expect(transfer([{ path: 'From', name: 'Album' }], 'To', 'move')).rejects.toThrow( @@ -270,21 +274,31 @@ describe('a copy on its way', () => { it('is a hidden entry, recorded until it is in place', async () => { await setup(); seedAlbum(); - const cp = fsp.cp.bind(fsp); + // Caught at the rename that puts the copy in place: everything has been + // written under the hidden name, and nothing has taken the real one yet. + const rename = fsp.rename.bind(fsp); let seen = null; - vi.spyOn(fsp, 'cp').mockImplementation(async (from, to, options) => { - await cp(from, to, options); - seen = { - listing: names(at('To')), - recorded: records().map((name) => JSON.parse(read(path.join(journal(), name)))), - }; + vi.spyOn(fsp, 'rename').mockImplementation(async (from, to) => { + // The one that gives the copy its real name, not a rename inside it. + if (String(to) === at('To', 'Album')) { + seen = { + listing: names(at('To')), + recorded: records().map((name) => JSON.parse(read(path.join(journal(), name)))), + }; + } + return rename(from, to); }); await transfer([{ path: 'From', name: 'Album' }], 'To', 'copy'); - expect(seen.listing).toEqual([expect.stringMatching(/^\.nextexplorer-copy-/)]); + // Two entries, and both are the point: the copy under a hidden name, and + // the name it will take, held from the moment it was chosen. Nothing else + // can arrive under that name while the copy is being written, which is how + // two copies started at once end up as `Album` and `Album (1)` rather than + // one landing on top of the other. + expect(seen.listing.sort()).toEqual([expect.stringMatching(/^\.nextexplorer-copy/), 'Album']); expect(seen.recorded).toEqual([ - expect.objectContaining({ path: at('To', seen.listing[0]), kind: 'partial-copy' }), + expect.objectContaining({ path: at('To', seen.listing[0]), kind: 'staging-copy' }), ]); expect(names(at('To'))).toEqual(['Album']); expect(records()).toEqual([]); @@ -293,10 +307,14 @@ describe('a copy on its way', () => { it('that fails leaves neither the hidden copy nor anything under the name', async () => { await setup(); fs.writeFileSync(at('From', 'note.txt'), 'mine'); - const copyFile = fsp.copyFile.bind(fsp); - vi.spyOn(fsp, 'copyFile').mockImplementation(async (from, to, mode) => { - await copyFile(from, to, mode); - throw Object.assign(new Error('Input/output error'), { code: 'EIO' }); + // A file is written through a handle now, so that the copy can report what + // it has moved and be stopped; the disk fails as it is opened. + const open = fsp.open.bind(fsp); + vi.spyOn(fsp, 'open').mockImplementation(async (target, flags, mode) => { + if (String(flags) === 'wx') { + throw Object.assign(new Error('Input/output error'), { code: 'EIO' }); + } + return open(target, flags, mode); }); await expect(transfer([{ path: 'From', name: 'note.txt' }], 'To', 'copy')).rejects.toThrow( diff --git a/backend/tests/services/folder-preferences-as-rows.test.js b/backend/tests/services/folder-preferences-as-rows.test.js new file mode 100644 index 000000000..b447d1ae1 --- /dev/null +++ b/backend/tests/services/folder-preferences-as-rows.test.js @@ -0,0 +1,189 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What a user chose for one folder, held as a row of its own. + * + * It used to be two JSON values per account under `user_settings` — one map of + * sorts, one of views — read and rewritten whole on every change. Three things + * followed from that, and all three are asserted here: + * + * - Two tabs open on different folders overwrote each other. Each sent the + * whole map, so whichever saved last won and the other folder's choice was + * gone. + * - The map had to be capped, because it shipped entire on every load and was + * rewritten entire on every change. The hundred-and-first folder silently + * forgot the oldest. + * - Nothing could clean it up: a deleted folder's preferences stayed behind on + * every account that had ever opened it. + * + * The carry-over is asserted too. An installation that has the old values keeps + * them: they are moved into rows, and the values they came from are removed so + * a later version cannot read them back. + */ + +const MODULES = ['src/services/db', 'src/services/settingsService']; + +let envContext; +let settingsService; +let dbService; + +beforeEach(async () => { + envContext = await setupTestEnv({ tag: 'folder-preferences-', modules: MODULES }); + dbService = envContext.requireFresh('src/services/db'); + settingsService = envContext.requireFresh('src/services/settingsService'); + + const db = await dbService.getDb(); + const now = new Date().toISOString(); + for (const id of ['u-1', 'u-2']) { + db.prepare('INSERT INTO users (id, email, created_at, updated_at) VALUES (?, ?, ?, ?)').run( + id, + `${id}@example.com`, + now, + now + ); + } +}); + +afterEach(async () => { + await envContext.cleanup(); +}); + +const stored = (userId = 'u-1') => settingsService.getUserSettings(userId); + +describe('a folder’s remembered sort and view', () => { + it('is saved one folder at a time, so another folder’s choice survives it', async () => { + await settingsService.setUserFolderSort('u-1', 'Projects', { by: 'name', order: 'desc' }); + await settingsService.setUserFolderSort('u-1', 'Music', { by: 'size', order: 'asc' }); + + expect((await stored()).folderSorts).toMatchObject({ + Projects: { by: 'name', order: 'desc' }, + Music: { by: 'size', order: 'asc' }, + }); + }); + + it('keeps the sort when the view of the same folder is set, and the other way round', async () => { + await settingsService.setUserFolderSort('u-1', 'Projects', { by: 'name', order: 'desc' }); + await settingsService.setUserFolderView('u-1', 'Projects', { mode: 'photos' }); + + const settings = await stored(); + expect(settings.folderSorts.Projects).toMatchObject({ by: 'name', order: 'desc' }); + expect(settings.folderViews.Projects).toMatchObject({ mode: 'photos' }); + }); + + it('is one account’s alone', async () => { + await settingsService.setUserFolderView('u-1', 'Projects', { mode: 'list' }); + await settingsService.setUserFolderView('u-2', 'Projects', { mode: 'grid' }); + + expect((await stored('u-1')).folderViews.Projects).toMatchObject({ mode: 'list' }); + expect((await stored('u-2')).folderViews.Projects).toMatchObject({ mode: 'grid' }); + }); + + /** + * The hundred-and-first folder. As one value per account this was a ceiling, + * and the oldest entry was dropped to stay under it; as rows there is nothing + * to stay under. + */ + it('is remembered past the hundred the single value could hold', async () => { + for (let n = 0; n < 120; n += 1) { + await settingsService.setUserFolderSort('u-1', `Folder-${n}`, { by: 'name', order: 'asc' }); + } + + const { folderSorts } = await stored(); + expect(Object.keys(folderSorts)).toHaveLength(120); + expect(folderSorts['Folder-0']).toMatchObject({ by: 'name', order: 'asc' }); + }); + + it('refuses a view mode there is no such thing as, rather than storing it', async () => { + await settingsService.setUserFolderView('u-1', 'Projects', { mode: 'grid' }); + await settingsService.setUserFolderView('u-1', 'Projects', { mode: 'sideways' }); + + expect((await stored()).folderViews.Projects).toMatchObject({ mode: 'grid' }); + }); +}); + +/** + * The installation that already had them. + * + * Built by putting the database back the way schema 19 left it — the two values + * under `user_settings`, no table of rows, the version stamped back — and then + * opening it again, which is the migration this batch adds. + */ +describe('preferences carried over from the single value per account', () => { + const T = 1756300000000; + + const asSchema19 = async () => { + const db = await dbService.getDb(); + const setting = (id, userId, key, value) => + db + .prepare( + 'INSERT INTO user_settings (id, user_id, key, value, updated_at) VALUES (?, ?, ?, ?, ?)' + ) + .run(id, userId, key, value, new Date(T).toISOString()); + + setting( + 'us-1', + 'u-1', + 'folderSorts', + JSON.stringify({ + Projects: { by: 'name', order: 'desc', updatedAt: T }, + Docs: { by: 'size', order: 'asc', updatedAt: T + 100 }, + }) + ); + setting( + 'us-2', + 'u-1', + 'folderViews', + JSON.stringify({ Projects: { mode: 'grid', updatedAt: T + 200 } }) + ); + setting('us-3', 'u-2', 'folderSorts', '{ not json'); + setting('us-4', 'u-1', 'theme', '"dark"'); + + db.exec('DROP TABLE folder_preferences'); + db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run('schema_version', '19'); + await dbService.closeDb(); + + envContext.requireFresh('src/services/db'); + dbService = envContext.requireFresh('src/services/db'); + settingsService = envContext.requireFresh('src/services/settingsService'); + return dbService.getDb(); + }; + + it('makes one row per folder, merging the sort and the view under the later time', async () => { + const db = await asSchema19(); + + expect( + db + .prepare( + `SELECT user_id, path, sort_by, sort_order, view_mode + FROM folder_preferences ORDER BY user_id, path` + ) + .all() + ).toEqual([ + { user_id: 'u-1', path: 'Docs', sort_by: 'size', sort_order: 'asc', view_mode: null }, + { user_id: 'u-1', path: 'Projects', sort_by: 'name', sort_order: 'desc', view_mode: 'grid' }, + ]); + }); + + it('removes the values it read, including one it could not, and keeps the rest', async () => { + const db = await asSchema19(); + + expect(db.prepare('SELECT user_id, key FROM user_settings ORDER BY key').all()).toEqual([ + { user_id: 'u-1', key: 'theme' }, + ]); + }); + + it('serves them through what the application reads', async () => { + await asSchema19(); + + expect(await settingsService.getUserSettings('u-1')).toMatchObject({ + folderSorts: { + Docs: { by: 'size', order: 'asc' }, + Projects: { by: 'name', order: 'desc' }, + }, + folderViews: { Projects: { mode: 'grid' } }, + theme: 'dark', + }); + }); +}); diff --git a/backend/tests/services/folder-upload-batches.test.js b/backend/tests/services/folder-upload-batches.test.js new file mode 100644 index 000000000..940b614f5 --- /dev/null +++ b/backend/tests/services/folder-upload-batches.test.js @@ -0,0 +1,184 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import fsSync from 'node:fs'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A picked folder reaches the server as one request per file, dozens of them + * in flight at once. The folder they belong to is decided by whichever of them + * arrives first; every other one has to be told the same answer, and a second + * upload of the same folder has to be told a different one. + * + * Getting it wrong loses nothing on disk, which is why it would go unnoticed: + * a folder of photos comes back scattered over `photos (1)`, `photos (2)` and + * `photos (3)`, or two uploads of it are poured into one folder where the + * copies of each file sit side by side under numbered names. Both only show up + * under concurrency, which the sequential suite beside this one never has. + */ + +let envContext; +let service; + +const build = async () => { + envContext = await setupTestEnv({ tag: 'folder-upload-batches-' }); + service = envContext.requireFresh('src/services/uploadFolderTargetService'); + const destinationRoot = path.join(envContext.volumeDir, 'Inbox'); + await fs.mkdir(destinationRoot, { recursive: true }); + return destinationRoot; +}; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +const OWNER = { user: { id: 'owner' } }; + +describe('the files of one folder upload, arriving together', () => { + it('all land in the one folder the first of them reserved', async () => { + const destinationRoot = await build(); + await fs.mkdir(path.join(destinationRoot, 'photos')); + + const files = ['a.jpg', 'b.jpg', 'c.jpg', '2026/d.jpg', '2026/e.jpg', 'f.jpg']; + const landed = await Promise.all( + files.map((file) => + service.resolveFolderUploadRelativePath({ + relativePath: `photos/${file}`, + destinationRoot, + logicalBase: 'Inbox', + context: OWNER, + uploadBatchId: 'batch-together-01', + }) + ) + ); + + expect(landed).toEqual(files.map((file) => `photos (1)/${file}`)); + // One folder reserved, not one per file that raced for it. + expect((await fs.readdir(destinationRoot)).sort()).toEqual(['photos', 'photos (1)']); + }); +}); + +describe('two uploads of the same folder, started at the same time', () => { + it('are kept in separate folders rather than merged into one', async () => { + const destinationRoot = await build(); + + const [first, second] = await Promise.all([ + service.resolveFolderUploadRelativePath({ + relativePath: 'photos/a.jpg', + destinationRoot, + logicalBase: 'Inbox', + context: OWNER, + uploadBatchId: 'batch-first-0001', + }), + service.resolveFolderUploadRelativePath({ + relativePath: 'photos/a.jpg', + destinationRoot, + logicalBase: 'Inbox', + context: OWNER, + uploadBatchId: 'batch-second-001', + }), + ]); + + expect([first, second].sort()).toEqual(['photos (1)/a.jpg', 'photos/a.jpg']); + }); +}); + +describe('a folder arriving under the name an upload chose', () => { + /** + * The name was looked for first and created afterwards, with a recursive + * mkdir that succeeds on a folder already there: whatever arrived under it in + * between — another upload, a copy, a folder made over SMB — received this + * upload's files. These make it arrive just before that mkdir. + */ + const arriveJustBefore = (target, arrive) => { + const mkdir = fs.mkdir.bind(fs); + let arrived = false; + vi.spyOn(fs, 'mkdir').mockImplementation(async (candidate, options) => { + if (!arrived && candidate === target) { + arrived = true; + arrive(); + } + return mkdir(candidate, options); + }); + }; + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('is never poured into, and the upload takes the next name', async () => { + const destinationRoot = await build(); + const theirs = path.join(destinationRoot, 'photos'); + arriveJustBefore(theirs, () => { + fsSync.mkdirSync(theirs); + fsSync.writeFileSync(path.join(theirs, 'theirs.jpg'), 'theirs'); + }); + + const landed = await service.resolveFolderUploadRelativePath({ + relativePath: 'photos/a.jpg', + destinationRoot, + logicalBase: 'Inbox', + context: OWNER, + uploadBatchId: 'batch-arriving-01', + }); + + expect(landed).toBe('photos (1)/a.jpg'); + expect(await fs.readdir(theirs)).toEqual(['theirs.jpg']); + expect((await fs.readdir(destinationRoot)).sort()).toEqual(['photos', 'photos (1)']); + }); + + it('takes the next name even when what arrived is empty', async () => { + const destinationRoot = await build(); + const theirs = path.join(destinationRoot, 'photos'); + arriveJustBefore(theirs, () => fsSync.mkdirSync(theirs)); + + const landed = await service.resolveFolderUploadRelativePath({ + relativePath: 'photos/a.jpg', + destinationRoot, + logicalBase: 'Inbox', + context: OWNER, + uploadBatchId: 'batch-arriving-02', + }); + + expect(landed).toBe('photos (1)/a.jpg'); + }); + + it('still creates the destination itself when it is not there yet', async () => { + const destinationRoot = path.join(await build(), 'not yet made'); + + const landed = await service.resolveFolderUploadRelativePath({ + relativePath: 'photos/a.jpg', + destinationRoot, + logicalBase: 'Inbox/not yet made', + context: OWNER, + uploadBatchId: 'batch-new-root-01', + }); + + expect(landed).toBe('photos/a.jpg'); + expect(await fs.readdir(destinationRoot)).toEqual(['photos']); + }); +}); + +describe('reserving a folder where none can be made', () => { + /** + * Only a name already taken is worth trying the next number for. Anything + * else — a destination that has gone, a disk mounted read-only — fails the + * same way for every number, and treating it as a collision spent a hundred + * thousand attempts before answering with a reason that named the wrong + * problem. + */ + it('gives up at once with the real reason', async () => { + const destinationRoot = await build(); + const missing = path.join(destinationRoot, 'unmounted'); + + await expect( + service.reserveFolderUploadTarget({ + destinationRoot: missing, + logicalBase: 'Inbox/unmounted', + sourceRoot: 'photos', + context: OWNER, + }) + ).rejects.toMatchObject({ code: 'ENOENT' }); + }); +}); diff --git a/backend/tests/services/heic-thumbnail.test.js b/backend/tests/services/heic-thumbnail.test.js new file mode 100644 index 000000000..4c852cf7c --- /dev/null +++ b/backend/tests/services/heic-thumbnail.test.js @@ -0,0 +1,104 @@ +import { describe, it, expect } from 'vitest'; +import { spawn } from 'node:child_process'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import sharp from 'sharp'; + +import { ffmpegReadsHeif, HEIC_FIXTURE as FIXTURE } from '../helpers/media-tools.js'; + +/** + * HEIC thumbnails, after ImageMagick was dropped from the image. + * + * `convert` was the only thing ImageMagick was installed for — 9.8 MB of + * packages for one format — and ffmpeg was already here for video. The risk in + * swapping them is not "does ffmpeg open the file": it is that a HEIC from a + * phone is a *grid of HEVC tiles*, and a decoder that reads only the first item + * returns one square of the picture and reports success. So this decodes a real + * file and looks at where the colours ended up. + * + * The fixture is 530 bytes: left half red, right half blue. Asymmetric on + * purpose — a mirrored or rotated result is a different picture, and a test + * that only checked the dimensions would pass on all three. + */ + +/** + * Whether the ffmpeg on this machine can read HEIF at all. + * + * The still-image HEIF demuxer arrived in ffmpeg 7.1. A CI runner on Ubuntu + * 24.04 carries 6.1 and cannot open the fixture, so there these are skipped — + * reported as skipped. A separate CI job runs them against the ffmpeg the + * image ships, and requires HEIF there, so they do run somewhere on every push. + */ +const heif = await ffmpegReadsHeif(); + +/** The decode half of makeHeicThumb, run as the service runs it. */ +const decodeToWebp = (size) => + new Promise((resolve, reject) => { + const child = spawn('ffmpeg', [ + '-v', + 'error', + '-i', + FIXTURE, + '-map', + '0:v:0', + '-frames:v', + '1', + '-vf', + `scale=${size}:-1:flags=lanczos`, + '-vcodec', + 'png', + '-f', + 'image2pipe', + 'pipe:1', + ]); + const pipeline = sharp().webp({ quality: 80, effort: 3 }); + child.stdout.pipe(pipeline); + child.on('error', reject); + pipeline.toBuffer().then(resolve).catch(reject); + }); + +describe.skipIf(!heif)('HEIC thumbnails are decoded by ffmpeg', () => { + it('produces a WebP of the requested width', async () => { + const webp = await decodeToWebp(64); + const meta = await sharp(webp).metadata(); + + expect(meta.format).toBe('webp'); + expect(meta.width).toBe(64); + }); + + /** + * The assertion that a tiled decode cannot fake. Reading only the first tile, + * or losing the orientation, moves these two colours. + */ + it('keeps red on the left and blue on the right', async () => { + const webp = await decodeToWebp(64); + const { data, info } = await sharp(webp).raw().toBuffer({ resolveWithObject: true }); + const at = (x, y) => { + const i = (y * info.width + x) * info.channels; + return { r: data[i], g: data[i + 1], b: data[i + 2] }; + }; + + const left = at(6, Math.floor(info.height / 2)); + const right = at(info.width - 6, Math.floor(info.height / 2)); + + expect(left.r).toBeGreaterThan(200); + expect(left.b).toBeLessThan(60); + expect(right.b).toBeGreaterThan(200); + expect(right.r).toBeLessThan(60); + }); + + it('writes a file the thumbnail cache can serve', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'heic-thumb-')); + try { + const destination = path.join(dir, 'thumb.webp'); + await fs.writeFile(destination, await decodeToWebp(96)); + + const written = await fs.stat(destination); + expect(written.size).toBeGreaterThan(0); + expect((await sharp(destination).metadata()).format).toBe('webp'); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/backend/tests/services/legacy-cache-check.test.js b/backend/tests/services/legacy-cache-check.test.js new file mode 100644 index 000000000..5dcfa6ecc --- /dev/null +++ b/backend/tests/services/legacy-cache-check.test.js @@ -0,0 +1,86 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What early releases left in the cache directory. + * + * An installation that started on 1.1.7 or earlier kept its database in /cache; + * the move to /config that 1.1.8 made was removed in 2.0.3, so one that skipped + * the releases in between comes up on an empty app.db with its accounts unread + * in /cache. Nothing said so. Now the start does — and moves nothing, since + * which file holds what matters cannot be told from here. + */ + +let envContext; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +const setup = async () => { + envContext = await setupTestEnv({ tag: 'legacy-cache-' }); + const check = envContext.requireFresh('src/services/legacyCacheCheck'); + const log = { warn: vi.fn(), info: vi.fn() }; + const report = () => + check.reportLegacyCache({ + cacheDir: envContext.cacheDir, + configDir: envContext.configDir, + log, + }); + return { check, log, report, cache: envContext.cacheDir, config: envContext.configDir }; +}; + +describe('an app.db left in the cache directory', () => { + it('is reported as a warning naming both files, and left where it is', async () => { + const { log, report, cache, config } = await setup(); + fs.writeFileSync(path.join(cache, 'app.db'), 'SQLite format 3\0 with the old accounts'); + + const findings = report(); + + expect(findings).toEqual([expect.objectContaining({ name: 'app.db', kind: 'database' })]); + expect(log.warn).toHaveBeenCalledTimes(1); + expect(log.warn.mock.calls[0][0]).toMatchObject({ + legacyDatabase: path.join(cache, 'app.db'), + databaseInUse: path.join(config, 'app.db'), + }); + expect(fs.existsSync(path.join(cache, 'app.db'))).toBe(true); + }); +}); + +describe('links left in the cache directory', () => { + it('are mentioned as unused, not warned about, and not followed', async () => { + const { log, report, cache, config } = await setup(); + fs.writeFileSync(path.join(config, 'app-config.json'), '{}'); + fs.symlinkSync(path.join(config, 'app.db'), path.join(cache, 'app.db')); + fs.symlinkSync(path.join(config, 'app-config.json'), path.join(cache, 'app-config.json')); + fs.symlinkSync(path.join(config, 'extensions'), path.join(cache, 'extensions')); + + const findings = report(); + + expect(findings.map((finding) => [finding.name, finding.kind])).toEqual([ + ['app.db', 'link'], + ['app-config.json', 'link'], + ['extensions', 'link'], + ]); + expect(log.warn).not.toHaveBeenCalled(); + expect(log.info).toHaveBeenCalledTimes(1); + expect(log.info.mock.calls[0][0].links).toHaveLength(3); + }); +}); + +describe('a cache directory with nothing from early releases', () => { + it('says nothing', async () => { + const { log, report, cache } = await setup(); + fs.mkdirSync(path.join(cache, 'thumbnails'), { recursive: true }); + fs.writeFileSync(path.join(cache, 'index.db'), 'SQLite format 3\0'); + + expect(report()).toEqual([]); + expect(log.warn).not.toHaveBeenCalled(); + expect(log.info).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/tests/services/media-tracks.test.js b/backend/tests/services/media-tracks.test.js index 8aa427f74..211c354ba 100644 --- a/backend/tests/services/media-tracks.test.js +++ b/backend/tests/services/media-tracks.test.js @@ -49,22 +49,58 @@ const buildFixture = async (dir) => { const output = path.join(dir, 'film.mkv'); await execFileAsync('ffmpeg', [ - '-v', 'error', '-y', - '-f', 'lavfi', '-i', 'testsrc=size=160x120:rate=25:duration=3', - '-f', 'lavfi', '-i', 'sine=frequency=440:duration=3', - '-f', 'lavfi', '-i', 'sine=frequency=880:duration=3', - '-i', frSrt, - '-i', enSrt, - '-map', '0:v', '-map', '1:a', '-map', '2:a', '-map', '3:s', '-map', '4:s', - '-c:v', 'libx264', '-preset', 'ultrafast', - '-c:a:0', 'aac', '-c:a:1', 'ac3', '-c:s', 'srt', + '-v', + 'error', + '-y', + '-f', + 'lavfi', + '-i', + 'testsrc=size=160x120:rate=25:duration=3', + '-f', + 'lavfi', + '-i', + 'sine=frequency=440:duration=3', + '-f', + 'lavfi', + '-i', + 'sine=frequency=880:duration=3', + '-i', + frSrt, + '-i', + enSrt, + '-map', + '0:v', + '-map', + '1:a', + '-map', + '2:a', + '-map', + '3:s', + '-map', + '4:s', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-c:a:0', + 'aac', + '-c:a:1', + 'ac3', + '-c:s', + 'srt', // `fre` rather than `fra`: both are ISO 639-2 for French, and a container // may carry either. A caption menu that shows them as two languages is the // bug this tag exists to catch. - '-metadata:s:a:0', 'language=fre', '-metadata:s:a:0', 'title=VF', - '-metadata:s:a:1', 'language=eng', - '-metadata:s:s:0', 'language=fre', - '-metadata:s:s:1', 'language=eng', + '-metadata:s:a:0', + 'language=fre', + '-metadata:s:a:0', + 'title=VF', + '-metadata:s:a:1', + 'language=eng', + '-metadata:s:s:0', + 'language=fre', + '-metadata:s:s:1', + 'language=eng', output, ]); return output; @@ -158,10 +194,27 @@ describe.skipIf(!(await ffmpegAvailable()))('what the player is told about sound it('separates having audio from being able to play it', async () => { const acThree = path.join(fixtureDir, 'ac3-only.mkv'); await execFileAsync('ffmpeg', [ - '-v', 'error', '-y', - '-f', 'lavfi', '-i', 'testsrc=size=160x120:rate=25:duration=2', - '-f', 'lavfi', '-i', 'sine=frequency=440:duration=2', - '-map', '0:v', '-map', '1:a', '-c:v', 'libx264', '-preset', 'ultrafast', '-c:a', 'ac3', + '-v', + 'error', + '-y', + '-f', + 'lavfi', + '-i', + 'testsrc=size=160x120:rate=25:duration=2', + '-f', + 'lavfi', + '-i', + 'sine=frequency=440:duration=2', + '-map', + '0:v', + '-map', + '1:a', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-c:a', + 'ac3', acThree, ]); @@ -174,9 +227,18 @@ describe.skipIf(!(await ffmpegAvailable()))('what the player is told about sound it('reports no audio at all for a video without any', async () => { const silent = path.join(fixtureDir, 'silent.mkv'); await execFileAsync('ffmpeg', [ - '-v', 'error', '-y', - '-f', 'lavfi', '-i', 'testsrc=size=160x120:rate=25:duration=2', - '-an', '-c:v', 'libx264', '-preset', 'ultrafast', + '-v', + 'error', + '-y', + '-f', + 'lavfi', + '-i', + 'testsrc=size=160x120:rate=25:duration=2', + '-an', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', silent, ]); @@ -203,7 +265,9 @@ describe.skipIf(!(await ffmpegAvailable()))('language tags', () => { const tracks = await read(); const sidecar = tracks.subtitles.find((track) => track.source === 'sidecar'); - const embedded = tracks.subtitles.find((track) => track.language === 'fr' && track.index !== null); + const embedded = tracks.subtitles.find( + (track) => track.language === 'fr' && track.index !== null + ); expect(sidecar.language).toBe(embedded.language); }); @@ -211,10 +275,27 @@ describe.skipIf(!(await ffmpegAvailable()))('language tags', () => { it('leaves an untagged track without a language rather than inventing one', async () => { const untagged = path.join(fixtureDir, 'untagged.mkv'); await execFileAsync('ffmpeg', [ - '-v', 'error', '-y', - '-f', 'lavfi', '-i', 'testsrc=size=160x120:rate=25:duration=2', - '-f', 'lavfi', '-i', 'sine=frequency=440:duration=2', - '-map', '0:v', '-map', '1:a', '-c:v', 'libx264', '-preset', 'ultrafast', '-c:a', 'aac', + '-v', + 'error', + '-y', + '-f', + 'lavfi', + '-i', + 'testsrc=size=160x120:rate=25:duration=2', + '-f', + 'lavfi', + '-i', + 'sine=frequency=440:duration=2', + '-map', + '0:v', + '-map', + '1:a', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-c:a', + 'aac', untagged, ]); diff --git a/backend/tests/services/native-rsync-transfer.test.js b/backend/tests/services/native-rsync-transfer.test.js new file mode 100644 index 000000000..39707c501 --- /dev/null +++ b/backend/tests/services/native-rsync-transfer.test.js @@ -0,0 +1,123 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { NATIVE_RSYNC } from '../helpers/native-rsync.js'; + +const childProcess = require('node:child_process'); + +/** + * Copies through the native engine, with rsync really running. + * + * FILE_TRANSFER_ENGINE is native on Linux, so rsync copies in nearly every + * deployment. The other transfer suites pin the stream engine so they run + * everywhere, and a Mac cannot run rsync the way the service asks for it, so + * nothing proved that rsync copied anything, nor that a copy through it keeps + * the rule that nothing already at the destination is replaced. CI requires + * these (REQUIRE_NATIVE_RSYNC); elsewhere they skip, and say so. + */ + +let currentEnv; + +afterEach(async () => { + vi.restoreAllMocks(); + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const insertAdmin = async (env) => { + const db = await env.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('admin', 'admin@example.com', 1, 'admin', 'Admin', '["admin"]', ?, ?)` + ).run(now, now); + return { id: 'admin', roles: ['admin'] }; +}; + +const setup = async () => { + currentEnv = await setupTestEnv({ + tag: 'native-rsync-', + env: { FILE_TRANSFER_ENGINE: 'native', FOLDER_SIZE_MODE: 'off' }, + }); + // What the service spawns, recorded, so a copy that silently fell back to the + // in-application engine cannot pass for one rsync made. + const commands = []; + const spawn = childProcess.spawn; + vi.spyOn(childProcess, 'spawn').mockImplementation(function recordSpawn(command, ...rest) { + commands.push(command); + return spawn.call(this, command, ...rest); + }); + const service = currentEnv.requireFresh('src/services/fileTransferService'); + const user = await insertAdmin(currentEnv); + return { service, volume: currentEnv.volumeDir, user, commands }; +}; + +const runTransfer = async (service, items, destination, operation, { user } = {}) => { + const prep = await service.prepareTransfer(items, destination, operation, { user }); + return service.executeTransfer(prep, operation, undefined, {}); +}; + +const readTree = async (root) => { + const tree = {}; + const walk = async (directory, prefix) => { + for (const entry of await fs.readdir(directory, { withFileTypes: true })) { + const relative = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) await walk(path.join(directory, entry.name), relative); + else tree[relative] = await fs.readFile(path.join(directory, entry.name), 'utf8'); + } + }; + await walk(root, ''); + return tree; +}; + +describe.skipIf(!NATIVE_RSYNC)('a copy through rsync', () => { + it('copies a folder whole, and leaves nothing else at the destination', async () => { + const { service, volume, user, commands } = await setup(); + await fs.mkdir(path.join(volume, 'Source', 'Album', 'nested'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Source', 'Album', 'a.txt'), 'first'); + await fs.writeFile(path.join(volume, 'Source', 'Album', 'nested', 'b.txt'), 'second'); + await fs.mkdir(path.join(volume, 'Dest'), { recursive: true }); + + await runTransfer(service, [{ path: 'Source', name: 'Album' }], 'Dest', 'copy', { user }); + + expect(commands).toContain('rsync'); + expect(await fs.readdir(path.join(volume, 'Dest'))).toEqual(['Album']); + expect(await readTree(path.join(volume, 'Dest', 'Album'))).toEqual({ + 'a.txt': 'first', + 'nested/b.txt': 'second', + }); + }); + + it('copies a file beside one of the same name, leaving that one untouched', async () => { + const { service, volume, user, commands } = await setup(); + await fs.mkdir(path.join(volume, 'Source'), { recursive: true }); + await fs.mkdir(path.join(volume, 'Dest'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Source', 'note.txt'), 'the newcomer'); + await fs.writeFile(path.join(volume, 'Dest', 'note.txt'), 'the incumbent'); + + const result = await runTransfer( + service, + [{ path: 'Source', name: 'note.txt' }], + 'Dest', + 'copy', + { + user, + } + ); + + expect(commands).toContain('rsync'); + expect(result.items[0].to).toBe('Dest/note (1).txt'); + expect(await fs.readFile(path.join(volume, 'Dest', 'note.txt'), 'utf8')).toBe('the incumbent'); + expect(await fs.readFile(path.join(volume, 'Dest', 'note (1).txt'), 'utf8')).toBe( + 'the newcomer' + ); + expect((await fs.readdir(path.join(volume, 'Dest'))).sort()).toEqual([ + 'note (1).txt', + 'note.txt', + ]); + }); +}); diff --git a/backend/tests/services/onlyoffice-activity.test.js b/backend/tests/services/onlyoffice-activity.test.js new file mode 100644 index 000000000..3d0706cec --- /dev/null +++ b/backend/tests/services/onlyoffice-activity.test.js @@ -0,0 +1,83 @@ +import { createRequire } from 'module'; +import { describe, it, expect, beforeEach } from 'vitest'; + +const require = createRequire(import.meta.url); +const modulePath = require.resolve('../../src/services/onlyofficeActivityService'); + +/** + * Presence says "somebody is editing this right now", and it is shown to + * everyone browsing the folder. Two ways to get it wrong: + * + * - claim it too early. It used to be recorded when the editor asked for its + * configuration, which happens before anyone knows the document will open — + * a file the editor refused stayed marked as being edited until it expired. + * It is now recorded on the first heartbeat, which the client only starts + * once ONLYOFFICE reports the document ready. + * - announce it too often. Presence changes wake every browser waiting on a + * long poll, and the heartbeat fires every sixty seconds per open document. + */ + +let activity; +beforeEach(() => { + delete require.cache[modulePath]; + activity = require(modulePath); +}); + +const FILE = '/volume/report.docx'; +const USER = { id: 'u1', name: 'Alice' }; + +describe('ONLYOFFICE presence', () => { + it('reports nobody until a session declares itself', () => { + expect(activity.get(FILE)).toBeNull(); + }); + + it('records the document as open on the first heartbeat', () => { + activity.touch({ absolutePath: FILE, sessionId: 's1', user: USER }); + + expect(activity.get(FILE)).toMatchObject({ active: true, count: 1, users: ['Alice'] }); + }); + + it('announces a change once, not on every heartbeat', () => { + const before = activity.getVersion(); + activity.touch({ absolutePath: FILE, sessionId: 's1', user: USER }); + const afterFirst = activity.getVersion(); + + for (let i = 0; i < 10; i += 1) { + activity.touch({ absolutePath: FILE, sessionId: 's1', user: USER }); + } + + expect(afterFirst).toBeGreaterThan(before); + // Ten more beats, still one announcement: waking every open browser once a + // minute per document is the cost this avoids. + expect(activity.getVersion()).toBe(afterFirst); + }); + + it('keeps the name from the first beat when later ones omit it', () => { + activity.touch({ absolutePath: FILE, sessionId: 's1', user: USER }); + activity.touch({ absolutePath: FILE, sessionId: 's1' }); + + expect(activity.get(FILE).users).toEqual(['Alice']); + }); + + it('counts two people editing the same document', () => { + activity.touch({ absolutePath: FILE, sessionId: 's1', user: USER }); + activity.touch({ absolutePath: FILE, sessionId: 's2', user: { id: 'u2', name: 'Bob' } }); + + const presence = activity.get(FILE); + expect(presence.count).toBe(2); + expect(presence.users.sort()).toEqual(['Alice', 'Bob']); + }); + + it('forgets a session when its editor closes', () => { + activity.touch({ absolutePath: FILE, sessionId: 's1', user: USER }); + activity.close({ absolutePath: FILE, sessionId: 's1' }); + + expect(activity.get(FILE)).toBeNull(); + }); + + it('ignores a beat with nothing to identify it', () => { + expect(activity.touch({ absolutePath: FILE })).toBe(false); + expect(activity.touch({ sessionId: 's1' })).toBe(false); + expect(activity.get(FILE)).toBeNull(); + }); +}); diff --git a/backend/tests/services/onlyoffice-document-key-purge.test.js b/backend/tests/services/onlyoffice-document-key-purge.test.js new file mode 100644 index 000000000..a2c9e32b7 --- /dev/null +++ b/backend/tests/services/onlyoffice-document-key-purge.test.js @@ -0,0 +1,65 @@ +import { afterEach, describe, expect, it } from 'vitest'; + +import { modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * ONLYOFFICE document keys past their expiry. + * + * A key past its expiry is never handed out again, yet its row stayed: only a + * terminal callback from the Document Server released one, and an editor that + * never sent it — a closed browser, a restarted server — left a row for every + * document ever opened. The hourly expiry sweep now removes them. + */ + +let env; + +afterEach(async () => { + if (env) await env.cleanup(); + env = null; +}); + +const STAT = { mtimeMs: 1_700_000_000_000, ctimeMs: 1_700_000_000_000, size: 4096 }; +const DAY = 24 * 60 * 60 * 1000; + +const setup = async () => { + env = await setupTestEnv({ tag: 'onlyoffice-key-purge-' }); + const keys = require(modulePath('src/services/onlyofficeDocumentKeyService')); + const db = await require(modulePath('src/services/db')).getDb(); + const open = (relativePath) => + keys.resolveDocumentKey({ relativePath, stat: STAT, documentType: 'word', inUse: false }); + const expire = (relativePath, msAgo) => + db + .prepare('UPDATE onlyoffice_document_keys SET expires_at = ? WHERE relative_path = ?') + .run(new Date(Date.now() - msAgo).toISOString(), relativePath); + const paths = () => + db + .prepare('SELECT relative_path FROM onlyoffice_document_keys ORDER BY relative_path') + .all() + .map((row) => row.relative_path); + return { keys, open, expire, paths }; +}; + +describe('purging ONLYOFFICE document keys', () => { + it('removes the keys past their expiry, and only those', async () => { + const { keys, open, expire, paths } = await setup(); + await open('closed-a minute ago.docx'); + await open('closed-a month ago.docx'); + const stillOpen = await open('open.docx'); + expire('closed-a minute ago.docx', 60 * 1000); + expire('closed-a month ago.docx', 30 * DAY); + + await expect(keys.purgeExpiredDocumentKeys()).resolves.toBe(2); + + expect(paths()).toEqual(['open.docx']); + // The one still in use keeps its key: its editors share it. + await expect(open('open.docx')).resolves.toBe(stillOpen); + }); + + it('has nothing to do when every key is current', async () => { + const { keys, open, paths } = await setup(); + await open('report.docx'); + + await expect(keys.purgeExpiredDocumentKeys()).resolves.toBe(0); + expect(paths()).toEqual(['report.docx']); + }); +}); diff --git a/backend/tests/services/orphaned-bindings.test.js b/backend/tests/services/orphaned-bindings.test.js new file mode 100644 index 000000000..eafa01c4d --- /dev/null +++ b/backend/tests/services/orphaned-bindings.test.js @@ -0,0 +1,179 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +const MODULES = [ + 'src/config/env', + 'src/config/index', + 'src/services/db', + 'src/services/pathBindingsService', + 'src/services/orphanedBindingsService', + 'src/utils/pathUtils', +]; + +let envContext; + +const build = async ({ env = {} } = {}) => { + envContext = await setupTestEnv({ tag: 'orphaned-bindings-test-', modules: MODULES, env }); + const service = envContext.requireFresh('src/services/orphanedBindingsService'); + const dbService = envContext.requireFresh('src/services/db'); + const db = await dbService.getDb(); + + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ).run('user-1', 'user-1@example.com', 1, 'user-1', 'User 1', '["user"]', now, now); + + return { service, db }; +}; + +const addFavourite = (db, id, storedPath) => + db + .prepare( + `INSERT INTO favorites (id, user_id, path, label, position, created_at, updated_at) + VALUES (?, 'user-1', ?, ?, 0, datetime('now'), datetime('now'))` + ) + .run(id, storedPath, path.basename(storedPath) || storedPath); + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('reporting paths that point at a volume which is not there', () => { + it('says nothing when every volume is present', async () => { + const { service, db } = await build(); + await fs.mkdir(path.join(envContext.volumeDir, 'Documents'), { recursive: true }); + addFavourite(db, 'fav-1', 'Documents/Reports'); + + expect(await service.findOrphanedBindings()).toEqual([]); + }); + + it('reports a volume that is gone, and counts what points at it', async () => { + const { service, db } = await build(); + await fs.mkdir(path.join(envContext.volumeDir, 'Documents'), { recursive: true }); + addFavourite(db, 'fav-1', 'Documents/Reports'); + addFavourite(db, 'fav-2', 'OldNAS/Photos'); + addFavourite(db, 'fav-3', 'OldNAS/Videos'); + + const orphaned = await service.findOrphanedBindings(); + + expect(orphaned).toHaveLength(1); + expect(orphaned[0].volume).toBe('OldNAS'); + expect(orphaned[0].total).toBe(2); + expect(orphaned[0].tables.favorites).toBe(2); + }); + + // The check must never remove anything: an unmounted volume looks exactly + // like a deleted one, and deleting on that basis would be irreversible. + it('removes nothing it reports', async () => { + const { service, db } = await build(); + addFavourite(db, 'fav-1', 'OldNAS/Photos'); + + await service.findOrphanedBindings(); + await service.reportOrphanedBindings(); + + expect(db.prepare('SELECT COUNT(*) AS count FROM favorites').get().count).toBe(1); + }); + + // A per-user volume is addressed by its label, not by a directory under the + // volume root. Without that, every one of them would look missing. + it('accepts a per-user volume by its label', async () => { + const { service, db } = await build(); + db.prepare( + `INSERT INTO user_volumes (id, user_id, label, path, access_mode, created_at, updated_at) + VALUES ('vol-1', 'user-1', 'MyNAS', '/elsewhere/nas', 'readwrite', datetime('now'), datetime('now'))` + ).run(); + addFavourite(db, 'fav-1', 'MyNAS/Photos'); + + expect(await service.findOrphanedBindings()).toEqual([]); + }); + + // A share token names a space of its own, not a volume. + it('leaves a share path alone', async () => { + const { service, db } = await build(); + addFavourite(db, 'fav-1', 'share/abc123/Inner'); + + expect(await service.findOrphanedBindings()).toEqual([]); + }); + + it('leaves a personal-folder path alone where personal folders are enabled', async () => { + const { service, db } = await build({ env: { USER_DIR_ENABLED: 'true' } }); + addFavourite(db, 'fav-1', 'personal/Notes'); + + expect(await service.findOrphanedBindings()).toEqual([]); + }); + + // With personal folders switched off, `personal` is an ordinary volume name + // and a path under it really does point at a volume that is not there. + it('treats personal as a volume name where personal folders are off', async () => { + const { service, db } = await build({ env: { USER_DIR_ENABLED: 'false' } }); + addFavourite(db, 'fav-1', 'personal/Notes'); + + const orphaned = await service.findOrphanedBindings(); + expect(orphaned).toHaveLength(1); + expect(orphaned[0].volume).toBe('personal'); + }); + + it('reports nothing when there is nothing stored', async () => { + const { service } = await build(); + + expect(await service.findOrphanedBindings()).toEqual([]); + }); + + // An unreadable volume root would make every stored path look orphaned. + // Saying nothing beats crying wolf about all of them at once. + it('stays silent when the volume root cannot be read', async () => { + const { service, db } = await build(); + addFavourite(db, 'fav-1', 'Documents/Reports'); + await fs.rm(envContext.volumeDir, { recursive: true, force: true }); + + expect(await service.findOrphanedBindings()).toBeNull(); + }); +}); + +describe('marking favourites whose volume is not there', () => { + const listFavourites = async () => { + const favoritesService = envContext.requireFresh('src/services/favoritesService'); + return favoritesService.getFavorites('user-1'); + }; + + it('marks the one whose volume is gone, and leaves the others alone', async () => { + const { db } = await build(); + await fs.mkdir(path.join(envContext.volumeDir, 'Documents'), { recursive: true }); + addFavourite(db, 'fav-1', 'Documents/Reports'); + addFavourite(db, 'fav-2', 'OldNAS/Photos'); + + const favourites = await listFavourites(); + const byId = Object.fromEntries(favourites.map((f) => [f.id, f])); + + expect(byId['fav-2'].available).toBe(false); + // Present ones carry no flag at all rather than available: true, so nothing + // downstream has to know about the field to keep working. + expect(byId['fav-1'].available).toBeUndefined(); + }); + + // Marked, not hidden: the volume may well come back, and the favourite with it. + it('still returns the favourite it marks', async () => { + const { db } = await build(); + addFavourite(db, 'fav-1', 'OldNAS/Photos'); + + const favourites = await listFavourites(); + + expect(favourites).toHaveLength(1); + expect(favourites[0].path).toBe('OldNAS/Photos'); + }); + + // Marking everything unavailable would be worse than marking nothing. + it('marks nothing when the volume list cannot be established', async () => { + const { db } = await build(); + addFavourite(db, 'fav-1', 'Documents/Reports'); + await fs.rm(envContext.volumeDir, { recursive: true, force: true }); + + const favourites = await listFavourites(); + + expect(favourites[0].available).toBeUndefined(); + }); +}); diff --git a/backend/tests/services/passkeys.test.js b/backend/tests/services/passkeys.test.js new file mode 100644 index 000000000..ed36c2c2b --- /dev/null +++ b/backend/tests/services/passkeys.test.js @@ -0,0 +1,227 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +const { createCredential, signAssertion } = require('../helpers/soft-authenticator'); + +/** + * What the passkey store holds, and what it refuses to let go of. + * + * The routes cover a browser signing in; this covers the cases a browser + * cannot reach — an account with no password left, a credential belonging to + * somebody else, a counter going backwards — because each of them is a way in + * or a way to be locked out. + */ + +const RP_ID = 'files.example.test'; +const ORIGIN = 'https://files.example.test'; + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'passkeys-service-' }); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + const addUser = db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, '["user"]', ?, ?)` + ); + addUser.run('u1', 'someone@example.com', 'someone', 'Someone', now, now); + addUser.run('u2', 'other@example.com', 'other', 'Other', now, now); + + const passkeys = currentEnv.requireFresh('src/services/users/passkeys'); + return { db, passkeys }; +}; + +/** A password on an account, of the kind `hasPassword` looks for. */ +const givePassword = (db, userId) => + db + .prepare( + `INSERT INTO auth_methods (id, user_id, method_type, password_hash, password_algo, created_at) + VALUES (?, ?, 'local_password', 'not-a-real-hash', 'bcrypt', ?)` + ) + .run(`m-${userId}`, userId, new Date().toISOString()); + +/** Register one through the service, the way the route does. */ +const enrol = async ({ passkeys, userId = 'u1', name, ...options }) => { + const started = await passkeys.beginRegistration({ + userId, + account: 'someone@example.com', + rpId: RP_ID, + rpName: 'Files', + }); + const credential = createCredential({ + challenge: started.challenge, + rpId: RP_ID, + origin: ORIGIN, + ...options, + }); + const passkey = await passkeys.finishRegistration({ + userId, + name, + response: { + attestationObject: credential.attestationObject.toString('base64url'), + clientDataJSON: credential.clientDataJSON.toString('base64url'), + }, + expected: { challenge: started.challenge, origins: [ORIGIN], rpId: RP_ID }, + }); + return { credential, passkey }; +}; + +const useIt = async ({ passkeys, credential, signCount = 1 }) => { + const started = passkeys.beginAuthentication({ rpId: RP_ID }); + const assertion = signAssertion({ + credential, + challenge: started.challenge, + rpId: RP_ID, + origin: ORIGIN, + signCount, + }); + return passkeys.finishAuthentication({ + response: { + id: credential.credentialId.toString('base64url'), + authenticatorData: assertion.authenticatorData.toString('base64url'), + clientDataJSON: assertion.clientDataJSON.toString('base64url'), + signature: assertion.signature.toString('base64url'), + }, + expected: { challenge: started.challenge, origins: [ORIGIN], rpId: RP_ID }, + }); +}; + +describe('keeping a passkey', () => { + it('finds the account it belongs to when it is used', async () => { + const { passkeys } = await seed(); + const { credential } = await enrol({ passkeys }); + + expect(await useIt({ passkeys, credential })).toMatchObject({ + userId: 'u1', + userVerified: true, + }); + }); + + it('remembers the counter, so the next signature has to beat it', async () => { + const { passkeys, db } = await seed(); + const { credential } = await enrol({ passkeys }); + + await useIt({ passkeys, credential, signCount: 30 }); + + expect(db.prepare('SELECT sign_count FROM passkeys').get().sign_count).toBe(30); + await expect(useIt({ passkeys, credential, signCount: 30 })).rejects.toThrow(/used before/); + await expect(useIt({ passkeys, credential, signCount: 29 })).rejects.toThrow(/used before/); + expect(await useIt({ passkeys, credential, signCount: 31 })).toMatchObject({ userId: 'u1' }); + }); + + it('refuses a credential another account already holds', async () => { + const { passkeys } = await seed(); + const { credential } = await enrol({ passkeys, userId: 'u1' }); + + await expect( + enrol({ passkeys, userId: 'u2', credentialId: credential.credentialId }) + ).rejects.toThrow(/already in use/); + }); + + it('says so when the account already holds it', async () => { + const { passkeys } = await seed(); + const { credential } = await enrol({ passkeys }); + + await expect( + enrol({ passkeys, userId: 'u1', credentialId: credential.credentialId }) + ).rejects.toThrow(/already on your account/); + }); + + it('keeps a name readable, and falls back to one that counts', async () => { + const { passkeys } = await seed(); + + await enrol({ passkeys, name: `a${String.fromCharCode(9)}b` }); + const second = await enrol({ passkeys, name: ' ' }); + const third = await enrol({ passkeys, name: 'x'.repeat(200) }); + + const held = await passkeys.listPasskeys('u1'); + expect(held.map((p) => p.name)).toEqual( + expect.arrayContaining(['a b', 'Passkey 2', 'x'.repeat(60)]) + ); + expect(second.passkey.name).toBe('Passkey 2'); + expect(third.passkey.name).toHaveLength(60); + }); +}); + +describe('taking one away', () => { + it('refuses the last one when it is the whole way in', async () => { + const { passkeys } = await seed(); + const { passkey } = await enrol({ passkeys }); + + expect(await passkeys.deletePasskey({ userId: 'u1', id: passkey.id })).toEqual({ + removed: false, + reason: 'last-way-in', + }); + expect(await passkeys.countPasskeys('u1')).toBe(1); + }); + + it('allows it once there is another one', async () => { + const { passkeys } = await seed(); + const { passkey } = await enrol({ passkeys }); + await enrol({ passkeys }); + + expect(await passkeys.deletePasskey({ userId: 'u1', id: passkey.id })).toEqual({ + removed: true, + }); + expect(await passkeys.countPasskeys('u1')).toBe(1); + }); + + it('allows the last one when a password can still open the account', async () => { + const { passkeys, db } = await seed(); + givePassword(db, 'u1'); + const { passkey } = await enrol({ passkeys }); + + expect(await passkeys.hasPassword('u1')).toBe(true); + expect(await passkeys.deletePasskey({ userId: 'u1', id: passkey.id })).toEqual({ + removed: true, + }); + }); + + it('will not take one from an account that does not hold it', async () => { + const { passkeys } = await seed(); + const { passkey } = await enrol({ passkeys, userId: 'u1' }); + await enrol({ passkeys, userId: 'u1' }); + + expect(await passkeys.deletePasskey({ userId: 'u2', id: passkey.id })).toEqual({ + removed: false, + reason: 'missing', + }); + expect(await passkeys.countPasskeys('u1')).toBe(2); + }); + + it('will not rename one that belongs to another account', async () => { + const { passkeys } = await seed(); + const { passkey } = await enrol({ passkeys, userId: 'u1', name: 'Mine' }); + + expect( + await passkeys.renamePasskey({ userId: 'u2', id: passkey.id, name: 'Yours' }) + ).toBeNull(); + expect((await passkeys.listPasskeys('u1'))[0].name).toBe('Mine'); + }); + + it('takes them all when an administrator hands an account back', async () => { + const { passkeys } = await seed(); + await enrol({ passkeys }); + await enrol({ passkeys }); + + expect(await passkeys.deleteAllPasskeys('u1')).toBe(2); + expect(await passkeys.listPasskeys('u1')).toEqual([]); + }); + + it('goes with the account it belonged to', async () => { + const { passkeys, db } = await seed(); + await enrol({ passkeys }); + + db.prepare('DELETE FROM users WHERE id = ?').run('u1'); + + expect(db.prepare('SELECT COUNT(*) AS total FROM passkeys').get().total).toBe(0); + }); +}); diff --git a/backend/tests/services/password-change-sessions.test.js b/backend/tests/services/password-change-sessions.test.js new file mode 100644 index 000000000..53ac05fa6 --- /dev/null +++ b/backend/tests/services/password-change-sessions.test.js @@ -0,0 +1,134 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What a password change leaves signed in. + * + * Changing a password is what someone does when they think it leaked, and the + * promise made in the administrator's guide is that it signs the account out + * everywhere but where the change was made. Sessions opened by signing in here + * carry the account id and were already ended; the ones the identity provider + * opened carry its tokens instead, and stayed open for as long as they lasted — + * thirty days by default — with whoever had the password still inside. + * + * This is the whole chain: the account's provider identities are read from + * `auth_methods`, the id token of each session is read back, and the two are + * matched. The store is exercised on its own in tests/utils; here it is the + * joining up that is at stake, because that is what nobody had done. + */ + +const ISSUER = 'https://idp.example'; + +let envContext; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +const idToken = (claims) => { + const part = (value) => Buffer.from(JSON.stringify(value)).toString('base64url'); + return `${part({ alg: 'RS256' })}.${part(claims)}.signature`; +}; + +/** A row as express-openid-connect writes it: the token set under `data`. */ +const providerSession = (sub, issuer = ISSUER) => ({ + cookie: { maxAge: 60_000 }, + header: { iat: 1, uat: 1, exp: 2 }, + data: { id_token: idToken({ iss: issuer, sub }), access_token: 'opaque' }, +}); + +const build = async () => { + envContext = await setupTestEnv({ tag: 'password-change-sessions-' }); + // Before the service: it loads the store lazily, and both must end up with + // the same sessions.db. + const { localStore } = envContext.requireFresh('src/utils/sessionStore'); + const users = envContext.requireFresh('src/services/users'); + const db = await envContext.requireFresh('src/services/db').getDb(); + return { users, db, localStore }; +}; + +const linkProviderIdentity = (db, userId, sub, issuer = ISSUER) => { + db.prepare( + `INSERT INTO auth_methods (id, user_id, method_type, provider_issuer, provider_sub, provider_name, created_at) + VALUES (?, ?, 'oidc', ?, ?, 'OIDC', ?)` + ).run(`auth-${sub}`, userId, issuer, sub, new Date().toISOString()); +}; + +const openSessions = (localStore) => + localStore.db + .prepare('SELECT sid FROM sessions ORDER BY sid') + .all() + .map((row) => row.sid); + +describe('changing a password', () => { + it('ends the sessions the identity provider opened for that account, and no others', async () => { + const { users, db, localStore } = await build(); + const alice = await users.createLocalUser({ + email: 'alice@example.com', + password: 'secret123', + username: 'alice', + displayName: 'Alice', + }); + const bob = await users.createLocalUser({ + email: 'bob@example.com', + password: 'secret123', + username: 'bob', + displayName: 'Bob', + }); + linkProviderIdentity(db, alice.id, 'alice-1'); + linkProviderIdentity(db, bob.id, 'bob-1'); + localStore.set('alice-sso', providerSession('alice-1')); + localStore.set('alice-here', providerSession('alice-1')); + localStore.set('alice-password', { cookie: { maxAge: 60_000 }, localUserId: alice.id }); + localStore.set('bob-sso', providerSession('bob-1')); + + await users.changeLocalPassword({ + userId: alice.id, + currentPassword: 'secret123', + newPassword: 'newpass456', + keepSessionId: 'alice-here', + }); + + expect(openSessions(localStore)).toEqual(['alice-here', 'bob-sso']); + }, 30_000); + + /** + * The administrator's reset makes the same promise, from the other side: the + * person whose password was reset is signed out everywhere. + */ + it('ends them when an administrator resets the password too', async () => { + const { users, db, localStore } = await build(); + const alice = await users.createLocalUser({ + email: 'alice@example.com', + password: 'secret123', + username: 'alice', + displayName: 'Alice', + }); + linkProviderIdentity(db, alice.id, 'alice-1'); + localStore.set('alice-sso', providerSession('alice-1')); + + await users.setLocalPasswordAdmin({ userId: alice.id, newPassword: 'newpass456' }); + + expect(openSessions(localStore)).toEqual([]); + }, 30_000); + + /** + * Giving a password to an account that had none ends nothing: no session was + * opened with it, and the person is in the middle of using one of them. + */ + it('leaves the provider sessions alone when the account is given its first password', async () => { + const { users, db, localStore } = await build(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('user-sso', 'sso@example.com', 1, 'sso', 'Sso', '["user"]', ?, ?)` + ).run(now, now); + linkProviderIdentity(db, 'user-sso', 'sso-1'); + localStore.set('sso-open', providerSession('sso-1')); + + await users.setLocalPasswordAdmin({ userId: 'user-sso', newPassword: 'newpass456' }); + + expect(openSessions(localStore)).toEqual(['sso-open']); + }, 30_000); +}); diff --git a/backend/tests/services/path-bindings-case.test.js b/backend/tests/services/path-bindings-case.test.js new file mode 100644 index 000000000..b9c4fe692 --- /dev/null +++ b/backend/tests/services/path-bindings-case.test.js @@ -0,0 +1,140 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A folder, and another whose name differs from it only by case. + * + * On a Linux volume `Docs` and `docs` are two folders. What the database ties + * to a path under one of them was found with `LIKE 'Docs/%'`, which SQLite + * matches without regard to case: deleting `Docs` dropped the favorites and + * the share links of `docs/…`, and renaming it re-pointed them at `Papers/…`. + * For a share that is worse than a broken link — `Papers/report.pdf` can be a + * different file from the one that was shared. + * + * Asked of the services rather than through the routes, because this machine's + * disk may not hold both folders at once, and the defect is in the query. + */ + +let env; +let db; +let bindings; +let shares; + +const PATHS = ['Docs', 'Docs/projet', 'Docs/sub/deep', 'docs/autre', 'Docs2/c', 'Docs.txt']; + +beforeEach(async () => { + env = await setupTestEnv({ tag: 'path-case-' }); + db = await env.requireFresh('src/services/db').getDb(); + bindings = env.requireFresh('src/services/pathBindingsService'); + shares = env.requireFresh('src/services/sharesService'); + + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('alice', 'alice@example.com', 1, 'alice', 'Alice', '["user"]', ?, ?)` + ).run(now, now); + + PATHS.forEach((folder, index) => { + db.prepare( + `INSERT INTO favorites (id, user_id, path, label, created_at, updated_at, position) + VALUES (?, 'alice', ?, 'x', ?, ?, ?)` + ).run(`fav-${index}`, folder, now, now, index); + db.prepare( + "INSERT INTO recent_destinations (user_id, path, used_at) VALUES ('alice', ?, ?)" + ).run(folder, now); + db.prepare( + `INSERT INTO folder_preferences (user_id, path, sort_by, sort_order, view_mode, updated_at) + VALUES ('alice', ?, 'name', 'asc', 'list', ?)` + ).run(folder, now); + }); +}); + +afterEach(async () => { + if (env) await env.cleanup(); + env = null; +}); + +const pathsIn = (table, column = 'path') => + db + .prepare(`SELECT ${column} AS value FROM ${table} ORDER BY ${column}`) + .all() + .map((row) => row.value); + +const shareAt = (sourcePath) => + shares.createShare({ ownerId: 'alice', sourceSpace: 'volume', sourcePath }); + +const sourceOf = (share) => + db.prepare('SELECT source_path FROM shares WHERE id = ?').pluck().get(share.id); + +describe('renaming a folder', () => { + it('carries what pointed into it, and nothing from the folder spelt otherwise', async () => { + await bindings.movePath('Docs', 'Papers'); + + const expected = [ + 'Docs.txt', + 'Docs2/c', + 'Papers', + 'Papers/projet', + 'Papers/sub/deep', + 'docs/autre', + ]; + expect(pathsIn('favorites')).toEqual(expected); + expect(pathsIn('recent_destinations')).toEqual(expected); + expect(pathsIn('folder_preferences')).toEqual(expected); + }); + + it('leaves a share of the other folder on the file that was shared', async () => { + const theirs = await shareAt('docs/report.pdf'); + const ours = await shareAt('Docs/report.pdf'); + + await bindings.movePath('Docs', 'Papers'); + + // Re-pointed at `Papers/report.pdf`, the link would open the file that + // used to be `Docs/report.pdf` — not the one its owner shared. + expect(sourceOf(theirs)).toBe('docs/report.pdf'); + expect(sourceOf(ours)).toBe('Papers/report.pdf'); + }); +}); + +describe('deleting a folder', () => { + it('forgets what pointed into it, and nothing from the folder spelt otherwise', async () => { + await bindings.forgetPath('Docs', { includeChildren: true }); + + const expected = ['Docs.txt', 'Docs2/c', 'docs/autre']; + expect(pathsIn('favorites')).toEqual(expected); + expect(pathsIn('recent_destinations')).toEqual(expected); + expect(pathsIn('folder_preferences')).toEqual(expected); + }); + + it('keeps the share links of the other folder', async () => { + const theirs = await shareAt('docs/report.pdf'); + const ours = await shareAt('Docs/report.pdf'); + + await bindings.forgetPath('Docs', { includeChildren: true }); + + expect(sourceOf(theirs)).toBe('docs/report.pdf'); + expect(sourceOf(ours)).toBeUndefined(); + }); + + it('counts, and so deletes, only the shares that are inside it', async () => { + await shareAt('Docs'); + await shareAt('Docs/report.pdf'); + await shareAt('Docs/sub/deep/plan.pdf'); + await shareAt('docs/report.pdf'); + await shareAt('Docs2/c.pdf'); + await shareAt('Docs.txt'); + const target = { sourceSpace: 'volume', sourcePath: 'Docs', includeChildren: true }; + const expected = ['Docs', 'Docs/report.pdf', 'Docs/sub/deep/plan.pdf']; + + // What the confirmation counts, and what the deletion then removes. + const listed = await shares.getSharesForSourceTargets([target]); + expect(listed.map((share) => share.sourcePath).sort()).toEqual(expected); + const byTarget = await shares.getSharesBySourceTarget([target]); + expect( + byTarget + .get('volume:Docs') + .map((share) => share.sourcePath) + .sort() + ).toEqual(expected); + }); +}); diff --git a/backend/tests/services/path-bindings.test.js b/backend/tests/services/path-bindings.test.js new file mode 100644 index 000000000..73c7fc7d1 --- /dev/null +++ b/backend/tests/services/path-bindings.test.js @@ -0,0 +1,199 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import request from 'supertest'; +import { createTestApp, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * What the database ties to a path has to follow that path, or forget it. + * + * Files move and disappear, and the rows pointing at them did not follow: a + * favorite outlived the folder it named, a share kept pointing at a path that + * no longer existed, and a folder's sort order was inherited by whatever folder + * happened to be created at the same place next. Deleting cleaned up the + * favorites of whoever deleted and nobody else's, which is the part that made + * it a bug rather than an omission — these are other people's rows. + */ + +describe('path bindings', () => { + let env; + + const setup = async () => { + env = await setupTestEnv({ + tag: 'path-bindings-', + modules: [ + 'src/services/db', + 'src/services/pathBindingsService', + 'src/services/settingsService', + 'src/services/accessManager', + 'src/routes/files', + 'src/middleware/errorHandler', + ], + }); + + await fs.mkdir(path.join(env.volumeDir, 'Projects', 'reports'), { recursive: true }); + await fs.writeFile(path.join(env.volumeDir, 'Projects', 'reports', 'q1.txt'), 'contents'); + await fs.mkdir(path.join(env.volumeDir, 'Archive'), { recursive: true }); + }; + + const appFor = (user) => { + const routes = env.requireFresh('src/routes/files'); + const { errorHandler } = env.requireFresh('src/middleware/errorHandler'); + return createTestApp({ router: routes, mountPath: '/api', user, errorHandler }); + }; + + /** Two people who both care about the same folder. */ + const givenBothUsersCareAbout = async (folderPath) => { + const db = await env.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + + for (const userId of ['alice', 'bob']) { + // favorites references users(id); these have to exist first. + db.prepare( + `INSERT OR IGNORE INTO users + (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, '["user"]', ?, ?)` + ).run(userId, `${userId}@example.com`, userId, userId, now, now); + + db.prepare( + `INSERT INTO favorites (id, user_id, path, label, icon, created_at, updated_at, position) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + `fav-${userId}-${folderPath}`, + userId, + folderPath, + 'Reports', + 'outline:StarIcon', + now, + now, + 0 + ); + + db.prepare('INSERT INTO recent_destinations (user_id, path, used_at) VALUES (?, ?, ?)').run( + userId, + folderPath, + now + ); + + db.prepare( + `INSERT INTO folder_preferences (user_id, path, sort_by, sort_order, view_mode, updated_at) + VALUES (?, ?, ?, ?, ?, ?)` + ).run(userId, folderPath, 'name', 'asc', 'list', now); + } + }; + + const countFor = async (table, column, value) => { + const db = await env.requireFresh('src/services/db').getDb(); + return db.prepare(`SELECT COUNT(*) AS count FROM ${table} WHERE ${column} = ?`).get(value) + .count; + }; + + afterEach(async () => { + if (env) { + await env.cleanup(); + env = null; + } + }); + + it('forgets everything tied to a deleted folder, for every user', async () => { + // The bug this replaces: deleting cleaned up the favorites of whoever + // pressed delete, leaving everyone else pointing at a folder that is gone. + await setup(); + await givenBothUsersCareAbout('Projects/reports'); + + const deleted = await request(appFor({ id: 'alice', roles: ['admin'] })) + .delete('/api/files') + .send({ items: [{ name: 'reports', path: 'Projects' }] }); + expect(deleted.status).toBe(200); + + expect(await countFor('favorites', 'path', 'Projects/reports')).toBe(0); + expect(await countFor('recent_destinations', 'path', 'Projects/reports')).toBe(0); + expect(await countFor('folder_preferences', 'path', 'Projects/reports')).toBe(0); + }); + + it('forgets what pointed inside a deleted folder too', async () => { + await setup(); + await givenBothUsersCareAbout('Projects/reports/q1'); + + // The folder holding it, not the volume: a volume is a mount, and the + // application refuses to delete one (nxzai/NextExplorer#409). + await request(appFor({ id: 'alice', roles: ['admin'] })) + .delete('/api/files') + .send({ items: [{ name: 'reports', path: 'Projects' }], permanent: true }); + + expect(await countFor('favorites', 'path', 'Projects/reports/q1')).toBe(0); + expect(await countFor('folder_preferences', 'path', 'Projects/reports/q1')).toBe(0); + }); + + it('follows a renamed folder rather than being left behind', async () => { + await setup(); + await givenBothUsersCareAbout('Projects/reports'); + + const renamed = await request(appFor({ id: 'alice', roles: ['admin'] })) + .post('/api/files/rename') + .send({ path: 'Projects', name: 'reports', newName: 'quarterly' }); + expect(renamed.status).toBe(200); + + expect(await countFor('favorites', 'path', 'Projects/reports')).toBe(0); + expect(await countFor('favorites', 'path', 'Projects/quarterly')).toBe(2); + expect(await countFor('folder_preferences', 'path', 'Projects/quarterly')).toBe(2); + expect(await countFor('recent_destinations', 'path', 'Projects/quarterly')).toBe(2); + }); + + it('carries what was inside a renamed folder with it', async () => { + // A favorite two levels down still names the same folder afterwards. + await setup(); + await givenBothUsersCareAbout('Projects/reports/q1'); + + await request(appFor({ id: 'alice', roles: ['admin'] })) + .post('/api/files/rename') + .send({ path: 'Projects', name: 'reports', newName: 'quarterly' }); + + expect(await countFor('favorites', 'path', 'Projects/reports/q1')).toBe(0); + expect(await countFor('favorites', 'path', 'Projects/quarterly/q1')).toBe(2); + }); + + it('follows a moved folder', async () => { + await setup(); + await givenBothUsersCareAbout('Projects/reports'); + + const moved = await request(appFor({ id: 'alice', roles: ['admin'] })) + .post('/api/files/move') + .send({ items: [{ name: 'reports', path: 'Projects' }], destination: 'Archive' }); + expect(moved.status).toBe(200); + + expect(await countFor('favorites', 'path', 'Projects/reports')).toBe(0); + expect(await countFor('favorites', 'path', 'Archive/reports')).toBe(2); + expect(await countFor('folder_preferences', 'path', 'Archive/reports')).toBe(2); + }); + + it('leaves the original alone when a folder is copied', async () => { + // A copy is a new folder that nobody has bookmarked yet; the original keeps + // everything that pointed at it. + await setup(); + await givenBothUsersCareAbout('Projects/reports'); + + const copied = await request(appFor({ id: 'alice', roles: ['admin'] })) + .post('/api/files/copy') + .send({ items: [{ name: 'reports', path: 'Projects' }], destination: 'Archive' }); + expect(copied.status).toBe(200); + + expect(await countFor('favorites', 'path', 'Projects/reports')).toBe(2); + expect(await countFor('favorites', 'path', 'Archive/reports')).toBe(0); + }); + + it('does not touch a folder whose name merely starts the same', async () => { + // Deleting "Projects/reports" must not take "Projects/reports-archive" + // with it — prefix matching without the separator would. + await setup(); + await givenBothUsersCareAbout('Projects/reports'); + await givenBothUsersCareAbout('Projects/reports-archive'); + + await request(appFor({ id: 'alice', roles: ['admin'] })) + .delete('/api/files') + .send({ items: [{ name: 'reports', path: 'Projects' }] }); + + expect(await countFor('favorites', 'path', 'Projects/reports')).toBe(0); + expect(await countFor('favorites', 'path', 'Projects/reports-archive')).toBe(2); + }); +}); diff --git a/backend/tests/services/performance-diagnostics.test.js b/backend/tests/services/performance-diagnostics.test.js new file mode 100644 index 000000000..2e3b7c010 --- /dev/null +++ b/backend/tests/services/performance-diagnostics.test.js @@ -0,0 +1,153 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The periodic record of what the process is costing. + * + * It exists for the case nobody can reproduce: an installation that goes slow after + * hours, on storage nobody here has, with a load nobody here makes. The only useful + * answer is what the process was costing at the time, so the sampler reports CPU, + * resident memory as the cgroup sees it, event-loop delay, and the queues that grow. + * + * What is worth testing is not the numbers — they are the machine's — but the three + * decisions around them: that it says nothing at all unless it was asked for, that it + * then reports only the intervals that look wrong, and that it can be told to report + * every one. A diagnostic that logs on every interval by accident is a diagnostic that + * fills a disk. + */ + +let env; + +afterEach(async () => { + vi.restoreAllMocks(); + if (env) { + env.requireFresh('src/services/performanceDiagnostics').stop(); + await env.cleanup(); + } + env = null; +}); + +const load = async (extraEnv = {}) => { + env = await setupTestEnv({ tag: 'perf-diagnostics-', env: extraEnv }); + // The logger first, and spied on before the service is loaded: the service keeps + // whichever logger it was given at require time, so spying on a fresh one afterwards + // watches an object nothing writes to. + const logger = env.requireFresh('src/utils/logger'); + const said = vi.spyOn(logger, 'info'); + const diagnostics = env.requireFresh('src/services/performanceDiagnostics'); + return { diagnostics, said }; +}; + +/** Every message a logger spy was given, as one string. */ +const messages = (spy) => spy.mock.calls.map((call) => String(call[1] ?? call[0])).join('\n'); + +describe('the performance record', () => { + it('is silent unless somebody asked for it', async () => { + const { diagnostics, said } = await load(); + + diagnostics.start(); + + expect(messages(said)).not.toContain('Performance diagnostics'); + }); + + it('says what it will watch, and by which thresholds, when it is on', async () => { + const { diagnostics, said } = await load({ + PERFORMANCE_DIAGNOSTICS_ENABLED: 'true', + PERFORMANCE_DIAGNOSTICS_INTERVAL_MS: '60000', + PERFORMANCE_DIAGNOSTICS_CPU_THRESHOLD: '90', + }); + + diagnostics.start(); + + expect(messages(said)).toContain('Performance diagnostics enabled'); + const announced = said.mock.calls.find(([, message]) => /enabled/.test(String(message)))[0]; + expect(announced.intervalMs).toBe(60000); + expect(announced.cpuThreshold).toBe(90); + }); + + it('holds an interval below its floor to the default, rather than sampling constantly', async () => { + // What an emptied or mistyped field sends. A sampler on a 1 ms interval costs more + // than whatever it was meant to diagnose. + const { diagnostics, said } = await load({ + PERFORMANCE_DIAGNOSTICS_ENABLED: 'true', + PERFORMANCE_DIAGNOSTICS_INTERVAL_MS: '1', + }); + + diagnostics.start(); + + const announced = said.mock.calls.find(([, message]) => /enabled/.test(String(message)))[0]; + expect(announced.intervalMs).toBe(15000); + }); + + it('reports nothing of an interval that looks ordinary', async () => { + const { diagnostics, said } = await load({ + PERFORMANCE_DIAGNOSTICS_ENABLED: 'true', + // Thresholds nothing here will reach. + PERFORMANCE_DIAGNOSTICS_CPU_THRESHOLD: '100000', + PERFORMANCE_DIAGNOSTICS_RSS_THRESHOLD_MB: '100000', + PERFORMANCE_DIAGNOSTICS_EVENT_LOOP_DELAY_MS: '100000', + }); + + diagnostics.start(); + await vi.waitFor(() => expect(messages(said)).toContain('enabled')); + await new Promise((resolve) => setTimeout(resolve, 50)); + + const records = said.mock.calls.filter(([, message]) => message === 'Performance diagnostics'); + expect(records).toEqual([]); + }); + + it('reports one that passes a threshold, and says which kind of interval it was', async () => { + const { diagnostics, said } = await load({ + PERFORMANCE_DIAGNOSTICS_ENABLED: 'true', + // A memory threshold of nothing: every interval is past it. + PERFORMANCE_DIAGNOSTICS_RSS_THRESHOLD_MB: '1', + }); + + diagnostics.start(); + + await vi.waitFor(() => { + const records = said.mock.calls.filter( + ([, message]) => message === 'Performance diagnostics' + ); + expect(records.length).toBeGreaterThan(0); + expect(records[0][0].reason).toBe('resource-pressure'); + }); + }); + + it('reports every interval when it is told to', async () => { + const { diagnostics, said } = await load({ + PERFORMANCE_DIAGNOSTICS_ENABLED: 'true', + PERFORMANCE_DIAGNOSTICS_LOG_EVERY_INTERVAL: 'true', + PERFORMANCE_DIAGNOSTICS_CPU_THRESHOLD: '100000', + PERFORMANCE_DIAGNOSTICS_RSS_THRESHOLD_MB: '100000', + PERFORMANCE_DIAGNOSTICS_EVENT_LOOP_DELAY_MS: '100000', + }); + + diagnostics.start(); + + await vi.waitFor(() => { + const records = said.mock.calls.filter( + ([, message]) => message === 'Performance diagnostics' + ); + expect(records.length).toBeGreaterThan(0); + // Nothing was under pressure: it is reporting because it was asked to. + expect(records[0][0].reason).toBe('interval'); + }); + }); + + it('samples the machine rather than guessing at it', async () => { + const { diagnostics } = await load({ PERFORMANCE_DIAGNOSTICS_ENABLED: 'true' }); + + const snapshot = await diagnostics.sample(); + + // `toMb` rounds, and this process is small enough to round to zero on some + // machines, so what is asserted is that the numbers came from somewhere rather + // than what they are. + expect(typeof snapshot.memoryMb.rss).toBe('number'); + expect(snapshot.memoryMb.heapTotal).toBeGreaterThan(0); + // And the queues each answered, or said they had nothing to answer with. + expect(snapshot).toHaveProperty('resources'); + expect(snapshot).toHaveProperty('cpuPercent'); + }); +}); diff --git a/backend/tests/services/personal-folder-reservation.test.js b/backend/tests/services/personal-folder-reservation.test.js new file mode 100644 index 000000000..173f19304 --- /dev/null +++ b/backend/tests/services/personal-folder-reservation.test.js @@ -0,0 +1,153 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A deleted account's personal folder is not handed to the next account. + * + * Deleting an account removed its row and nothing else: `_users/` stayed + * on disk with what it held, its trash and its versions, and the name was free + * again. With `USER_FOLDER_NAME_ORDER=username,id` — what the environment + * reference recommends — the next account called bob claimed `bob`, and with it + * the previous bob's files. Reproduced before this change. + * + * The name now stays reserved while the folder is on disk; removing or renaming + * that folder on the server frees it. The rows that only described the account + * go with it. + */ + +let envContext; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +const build = async () => { + envContext = await setupTestEnv({ + tag: 'personal-folder-reservation-', + env: { USER_FOLDER_NAME_ORDER: 'username,id', USER_DIR_ENABLED: 'true' }, + }); + const dbModule = envContext.requireFresh('src/services/db'); + const db = await dbModule.getDb(); + const { claimPersonalFolderName } = envContext.requireFresh('src/services/personalFolders'); + const { deleteUser } = envContext.requireFresh('src/services/users/management'); + const userRoot = path.join(envContext.volumeDir, '_users'); + return { db, dbModule, claimPersonalFolderName, deleteUser, userRoot }; +}; + +const addUser = (db, { id, username, createdAt = new Date().toISOString() }) => { + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, '["user"]', ?, ?)` + ).run(id, `${id}@example.com`, username, username, createdAt, createdAt); + return db.prepare('SELECT * FROM users WHERE id = ?').get(id); +}; + +const count = (db, sql, ...args) => + db + .prepare(sql) + .pluck() + .get(...args); + +describe('the folder name of a deleted account', () => { + it('is not given to the next account that derives it while its folder is on disk', async () => { + const { db, claimPersonalFolderName, deleteUser, userRoot } = await build(); + expect(claimPersonalFolderName(db, addUser(db, { id: 'bob-1', username: 'bob' }))).toBe('bob'); + fs.mkdirSync(path.join(userRoot, 'bob'), { recursive: true }); + fs.writeFileSync(path.join(userRoot, 'bob', 'payslip.pdf'), 'private'); + + await deleteUser({ userId: 'bob-1' }); + const claimed = claimPersonalFolderName(db, addUser(db, { id: 'bob-2', username: 'bob' })); + + expect(claimed).not.toBe('bob'); + expect(claimed).toBe('bob-2'); + expect(fs.readFileSync(path.join(userRoot, 'bob', 'payslip.pdf'), 'utf8')).toBe('private'); + }); + + it('is given out again once its folder has been removed from the disk', async () => { + const { db, claimPersonalFolderName, deleteUser, userRoot } = await build(); + claimPersonalFolderName(db, addUser(db, { id: 'bob-1', username: 'bob' })); + fs.mkdirSync(path.join(userRoot, 'bob'), { recursive: true }); + await deleteUser({ userId: 'bob-1' }); + + fs.rmSync(path.join(userRoot, 'bob'), { recursive: true }); + const claimed = claimPersonalFolderName(db, addUser(db, { id: 'bob-2', username: 'bob' })); + + expect(claimed).toBe('bob'); + expect(count(db, 'SELECT COUNT(*) FROM personal_folder_reservations')).toBe(0); + }); + + it('reserves nothing for an account that never had a folder name', async () => { + const { db, deleteUser } = await build(); + addUser(db, { id: 'ann-1', username: 'ann' }); + + await deleteUser({ userId: 'ann-1' }); + + expect(count(db, 'SELECT COUNT(*) FROM personal_folder_reservations')).toBe(0); + }); +}); + +describe('deleting an account', () => { + const seedRows = (db, userId) => { + const now = new Date().toISOString(); + db.prepare( + "INSERT INTO folder_preferences (user_id, path, sort_by, sort_order, view_mode, updated_at) VALUES (?, 'Docs', 'name', 'asc', 'list', ?)" + ).run(userId, now); + db.prepare( + "INSERT INTO recent_destinations (user_id, path, used_at) VALUES (?, 'Docs', ?)" + ).run(userId, now); + db.prepare('INSERT INTO auth_locks (key, failed_count, locked_until) VALUES (?, 3, NULL)').run( + userId + ); + }; + + it('removes the rows that only described it, and leaves everyone else their own', async () => { + const { db, deleteUser } = await build(); + addUser(db, { id: 'gone', username: 'gone' }); + addUser(db, { id: 'stays', username: 'stays' }); + seedRows(db, 'gone'); + seedRows(db, 'stays'); + + await deleteUser({ userId: 'gone' }); + + for (const table of ['folder_preferences', 'recent_destinations']) { + expect(count(db, `SELECT COUNT(*) FROM ${table} WHERE user_id = 'gone'`), table).toBe(0); + expect(count(db, `SELECT COUNT(*) FROM ${table} WHERE user_id = 'stays'`), table).toBe(1); + } + expect(count(db, "SELECT COUNT(*) FROM auth_locks WHERE key = 'gone'")).toBe(0); + expect(count(db, "SELECT COUNT(*) FROM auth_locks WHERE key = 'stays'")).toBe(1); + }); + + it('removes, at the upgrade, the rows accounts deleted before it left behind', async () => { + const { db, dbModule, deleteUser } = await build(); + addUser(db, { id: 'stays', username: 'stays' }); + addUser(db, { id: 'deleted-long-ago', username: 'old' }); + seedRows(db, 'stays'); + seedRows(db, 'deleted-long-ago'); + // Deleted the way every release before this one did: the row alone. + db.prepare("DELETE FROM users WHERE id = 'deleted-long-ago'").run(); + // Back to before the reservations table existed here, which is 22: the + // numbering is this repository's, not the fork's. + db.prepare("UPDATE meta SET value = '22' WHERE key = 'schema_version'").run(); + dbModule.closeDb(); + void deleteUser; + + const reopened = await envContext.requireFresh('src/services/db').getDb(); + + expect(count(reopened, "SELECT value FROM meta WHERE key = 'schema_version'")).toBe('23'); + for (const table of ['folder_preferences', 'recent_destinations']) { + expect( + count(reopened, `SELECT COUNT(*) FROM ${table} WHERE user_id = 'deleted-long-ago'`) + ).toBe(0); + expect(count(reopened, `SELECT COUNT(*) FROM ${table} WHERE user_id = 'stays'`)).toBe(1); + } + // Sign-in locks are left alone: those older releases wrote are keyed by the name + // that was typed, not by an account id, so which account one belonged to + // cannot be told, and an upgrade must not lift a lock it cannot place. + expect(count(reopened, 'SELECT COUNT(*) FROM auth_locks')).toBe(2); + }); +}); diff --git a/backend/tests/services/prepared-statements.test.js b/backend/tests/services/prepared-statements.test.js new file mode 100644 index 000000000..e86f59c55 --- /dev/null +++ b/backend/tests/services/prepared-statements.test.js @@ -0,0 +1,56 @@ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * better-sqlite3 compiles the SQL on every prepare() call. Services that run + * once per item were recompiling the same statements thousands of times: a CPU + * profile of a 3000-file delete put prepare() at the top of the applied work, + * ahead of the filesystem calls it exists to support. + */ + +let currentEnv; +afterEach(async () => { + vi.restoreAllMocks(); + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +describe('Prepared statement cache', () => { + it('compiles a given statement once per database handle', async () => { + const env = await setupTestEnv({ + tag: 'prepared-cache-', + modules: ['src/config/env', 'src/config/index', 'src/services/db'], + }); + currentEnv = env; + + const { getDb, prepared } = env.requireFresh('src/services/db'); + const db = await getDb(); + const spy = vi.spyOn(db, 'prepare'); + + const sql = 'SELECT * FROM shares WHERE source_space = ? AND source_path = ?'; + const first = prepared(db, sql); + for (let i = 0; i < 500; i += 1) prepared(db, sql); + + expect(spy).toHaveBeenCalledTimes(1); + // And it is the same statement, not a lookalike. + expect(prepared(db, sql)).toBe(first); + }); + + it('keeps different statements apart', async () => { + const env = await setupTestEnv({ + tag: 'prepared-cache-distinct-', + modules: ['src/config/env', 'src/config/index', 'src/services/db'], + }); + currentEnv = env; + + const { getDb, prepared } = env.requireFresh('src/services/db'); + const db = await getDb(); + + const a = prepared(db, 'SELECT * FROM shares WHERE id = ?'); + const b = prepared(db, 'SELECT * FROM shares WHERE owner_id = ?'); + + expect(a).not.toBe(b); + }); +}); diff --git a/backend/tests/services/raw-preview-cache-cleanup.test.js b/backend/tests/services/raw-preview-cache-cleanup.test.js new file mode 100644 index 000000000..dedee2b47 --- /dev/null +++ b/backend/tests/services/raw-preview-cache-cleanup.test.js @@ -0,0 +1,390 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { substituteModule } from '../helpers/substitute-module.js'; + +/** + * What the RAW preview cache throws away. + * + * An embedded preview is a full-size JPEG copied out of a RAW file, for the + * viewer and on the way to the photo's thumbnail. Nothing removed one: the key + * includes the RAW file's modification time, so every edit of a photo left the + * previous preview behind, and a crash left its temporary file. It is bounded + * now by the thumbnails' rules and settings, with a file limit of its own. + */ + +// Captured before any test fakes the timers, so real time can still be waited on. +const realSetTimeout = globalThis.setTimeout; +const pause = (ms) => new Promise((resolve) => realSetTimeout(resolve, ms)); + +const SERVICE_FILE = fileURLToPath( + new URL('../../src/services/rawPreviewService.js', import.meta.url) +); +const MINUTE = 60 * 1000; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; +const sha1 = (n) => String(n).padStart(40, 'a'); +const preview = (n, version = 1) => `v${version}-${sha1(n)}.jpg`; +const tempOf = (name) => `${name}.tmp-4242-1700000000000`; + +let currentEnv = null; +let releaseHeldExtraction = null; +let restoreModule = null; + +const setup = async (env = {}) => { + currentEnv = await setupTestEnv({ tag: 'raw-preview-cleanup-', env }); + const service = currentEnv.requireFresh('src/services/rawPreviewService'); + const dir = path.join(currentEnv.cacheDir, 'raw-previews'); + await fs.mkdir(dir, { recursive: true }); + return { service, dir }; +}; + +/** A cache entry, optionally aged. */ +const write = async (dir, name, { ageMs = 0 } = {}) => { + const file = path.join(dir, name); + await fs.writeFile(file, 'jpeg'); + if (ageMs > 0) { + const when = new Date(Date.now() - ageMs); + await fs.utimes(file, when, when); + } + return name; +}; + +const remaining = async (dir) => (await fs.readdir(dir)).sort(); + +const exists = (file) => + fs.access(file).then( + () => true, + () => false + ); + +afterEach(async () => { + // A held extraction would keep the service from ever settling. + releaseHeldExtraction?.(); + releaseHeldExtraction = null; + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } + vi.useRealTimers(); + restoreModule?.(); + restoreModule = null; +}); + +/** + * An exiftool whose preview extraction begins, and then waits to be told to + * finish. `started` resolves with the temporary path once it is on disk. + */ +const holdExtraction = () => { + let release; + const released = new Promise((resolve) => { + release = resolve; + }); + let reportStarted; + const started = new Promise((resolve) => { + reportStarted = resolve; + }); + + const exiftool = { + extractPreview: async (_input, output) => { + await fs.writeFile(output, 'half a preview'); + reportStarted(output); + await released; + await fs.writeFile(output, 'a preview'); + }, + extractThumbnail: async () => { + throw new Error('not reached'); + }, + extractJpgFromRaw: async () => { + throw new Error('not reached'); + }, + }; + + return { module: { exiftool }, started, release }; +}; + +describe('a RAW preview cache past its limit', () => { + it('loses its oldest previews first', async () => { + const { service, dir } = await setup({ RAW_PREVIEW_CACHE_MAX_FILES: '2' }); + // Alphabetical order is the reverse of age, so directory order cannot pass for it. + await write(dir, preview(4), { ageMs: 4 * HOUR }); + await write(dir, preview(3), { ageMs: 3 * HOUR }); + await write(dir, preview(2), { ageMs: 2 * HOUR }); + await write(dir, preview(1), { ageMs: 1 * HOUR }); + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual([preview(1), preview(2)]); + }); + + it('deletes no more than one batch at a time', async () => { + const { service, dir } = await setup({ + RAW_PREVIEW_CACHE_MAX_FILES: '1', + THUMBNAIL_CACHE_CLEANUP_BATCH_SIZE: '2', + }); + for (let i = 0; i < 6; i += 1) await write(dir, preview(i)); + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toHaveLength(4); + }); +}); + +describe('a RAW preview cache within its limit', () => { + it('is left entirely alone', async () => { + const { service, dir } = await setup({ RAW_PREVIEW_CACHE_MAX_FILES: '10' }); + const names = []; + for (let i = 0; i < 3; i += 1) names.push(await write(dir, preview(i), { ageMs: HOUR })); + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual(names.sort()); + }); +}); + +describe('a RAW preview nobody has needed for a long time', () => { + it('is removed once past the cache lifetime', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_TTL_DAYS: '1' }); + await write(dir, preview(1), { ageMs: 3 * DAY }); + await write(dir, preview(2)); + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual([preview(2)]); + }); + + /** A lifetime of zero is what turns the rule off, not what expires everything. */ + it('is kept when no lifetime is set', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_TTL_DAYS: '0' }); + await write(dir, preview(1), { ageMs: 365 * DAY }); + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual([preview(1)]); + }); +}); + +describe('a RAW preview from another cache version', () => { + it('is removed', async () => { + const { service, dir } = await setup(); + await write(dir, preview(1, 2)); + await write(dir, preview(2)); + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual([preview(2)]); + }); +}); + +describe('temporary files an extraction left behind', () => { + it('are removed once clearly abandoned', async () => { + const { service, dir } = await setup(); + await write(dir, preview(1), { ageMs: 2 * HOUR }); + await write(dir, tempOf(preview(2)), { ageMs: 2 * HOUR }); + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual([preview(1)]); + }); + + it('are kept while recent', async () => { + const { service, dir } = await setup(); + await write(dir, tempOf(preview(1)), { ageMs: 10 * MINUTE }); + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual([tempOf(preview(1))]); + }); + + it('neither count towards the limit nor are trimmed to meet it', async () => { + const { service, dir } = await setup({ RAW_PREVIEW_CACHE_MAX_FILES: '1' }); + const names = [ + await write(dir, preview(1)), + await write(dir, tempOf(preview(2))), + await write(dir, tempOf(preview(3))), + ]; + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual(names.sort()); + }); + + it('are left alone when the name is not one of ours', async () => { + const { service, dir } = await setup(); + const names = [ + await write(dir, 'photo.jpg.tmp-4242-1700000000000', { ageMs: 2 * HOUR }), + await write(dir, 'v1-nothexadecimal.jpg.tmp-4242-1700000000000', { ageMs: 2 * HOUR }), + ]; + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual(names.sort()); + }); + + it('are kept however old while their extraction is still going on', async () => { + const held = holdExtraction(); + releaseHeldExtraction = held.release; + restoreModule = substituteModule(SERVICE_FILE, 'exiftool-vendored', held.module); + const { service, dir } = await setup(); + const raw = path.join(currentEnv.volumeDir, 'photo.cr2'); + await fs.writeFile(raw, 'raw bytes'); + + const extraction = service.getRawPreviewJpegPath(raw); + const tempFile = await held.started; + const longAgo = new Date(Date.now() - 2 * HOUR); + await fs.utimes(tempFile, longAgo, longAgo); + + await service.cleanupRawPreviewCache(); + expect(await remaining(dir)).toContain(path.basename(tempFile)); + + held.release(); + const finalPath = await extraction; + expect(await remaining(dir)).toEqual([path.basename(finalPath)]); + }); +}); + +describe('a file in that directory that is not a RAW preview', () => { + it('is neither counted nor deleted', async () => { + const { service, dir } = await setup({ RAW_PREVIEW_CACHE_MAX_FILES: '1' }); + const names = [ + await write(dir, 'notes.txt'), + await write(dir, 'v1-nothexadecimal.jpg'), + await write(dir, `v1-${sha1(7)}.jpeg`), + await write(dir, preview(1)), + ]; + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual(names.sort()); + }); +}); + +describe('a RAW preview cache with the limit switched off', () => { + /** Zero lifts the limit on the count, and must not mean "delete everything". */ + it('keeps every current preview, however many', async () => { + const { service, dir } = await setup({ RAW_PREVIEW_CACHE_MAX_FILES: '0' }); + const names = []; + for (let i = 1; i < 6; i += 1) names.push(await write(dir, preview(i))); + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual(names.sort()); + }); + + /** + * It used to leave the directory unmanaged: previews of another version, + * those past their lifetime and abandoned temporary files stayed for good. + */ + it('still removes what is outdated, expired or abandoned', async () => { + const { service, dir } = await setup({ RAW_PREVIEW_CACHE_MAX_FILES: '0' }); + const kept = await write(dir, preview(1)); + await write(dir, preview(2, 2)); + await write(dir, tempOf(preview(3)), { ageMs: 2 * HOUR }); + await write(dir, preview(4), { ageMs: 40 * DAY }); + + await service.cleanupRawPreviewCache(); + + expect(await remaining(dir)).toEqual([kept]); + }); +}); + +describe('the RAW preview cleanup schedule', () => { + const env = { RAW_PREVIEW_CACHE_MAX_FILES: '1', THUMBNAIL_CACHE_CLEANUP_INTERVAL_MS: '60000' }; + + /** + * Move the clock on a minute at a time until `probe` holds, and say whether + * it did. Real time passes in between for the file operations to land. + */ + const advanceUntil = async (probe, minutes = 30) => { + for (let i = 0; i < minutes; i += 1) { + if (await probe()) return true; + await vi.advanceTimersByTimeAsync(MINUTE); + await pause(5); + } + return probe(); + }; + + it('trims the cache by itself, and goes on doing so', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const { dir } = await setup(env); + for (let i = 0; i < 3; i += 1) await write(dir, preview(i)); + + expect(await advanceUntil(async () => (await remaining(dir)).length === 1)).toBe(true); + + for (let i = 3; i < 6; i += 1) await write(dir, preview(i)); + expect(await advanceUntil(async () => (await remaining(dir)).length === 1)).toBe(true); + }); + + it('stays stopped once asked, even with a pass under way', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const { service, dir } = await setup(env); + for (let i = 0; i < 3; i += 1) await write(dir, preview(i)); + + const pass = service.cleanupRawPreviewCache(); + await service.stopRawPreviewWork(); + await pass; + expect(await remaining(dir)).toHaveLength(1); + + for (let i = 3; i < 6; i += 1) await write(dir, preview(i)); + expect(await advanceUntil(async () => (await remaining(dir)).length < 4)).toBe(false); + }); +}); + +describe('files outside the thumbnail and RAW preview directories', () => { + /** + * Names each cleanup would take, aged well past every rule, placed around the + * two directories: in the cache root, on the volume, above both, and in + * look-alike folders on the volume. The directories themselves are emptied, + * which is what says the cleanups ran at all. + */ + it('are never touched', async () => { + const limits = { RAW_PREVIEW_CACHE_MAX_FILES: '1', THUMBNAIL_CACHE_MAX_FILES: '1' }; + const { service: rawPreviews, dir: previewDir } = await setup(limits); + const thumbnails = currentEnv.requireFresh('src/services/thumbnailService'); + const thumbnailDir = path.join(currentEnv.cacheDir, 'thumbnails'); + await fs.mkdir(thumbnailDir, { recursive: true }); + + const tempted = [ + `${sha1(9)}.webp`, + `v3-${sha1(9)}.webp`, + tempOf(`v3-${sha1(9)}.webp`), + preview(9), + tempOf(preview(9)), + ]; + const places = [ + currentEnv.cacheDir, + currentEnv.volumeDir, + currentEnv.tmpRoot, + path.join(currentEnv.volumeDir, 'thumbnails'), + path.join(currentEnv.volumeDir, 'raw-previews'), + ]; + const planted = []; + for (const place of places) { + await fs.mkdir(place, { recursive: true }); + for (const name of tempted) { + await write(place, name, { ageMs: 40 * DAY }); + planted.push(path.join(place, name)); + } + } + + await write(thumbnailDir, `${sha1(1)}.webp`, { ageMs: 40 * DAY }); + await write(thumbnailDir, `v3-${sha1(2)}.webp`, { ageMs: 40 * DAY }); + await write(thumbnailDir, tempOf(`v3-${sha1(3)}.webp`), { ageMs: 40 * DAY }); + await write(previewDir, preview(1), { ageMs: 40 * DAY }); + await write(previewDir, preview(2), { ageMs: 40 * DAY }); + await write(previewDir, tempOf(preview(3)), { ageMs: 40 * DAY }); + + await thumbnails.cleanupThumbnailCache(); + await rawPreviews.cleanupRawPreviewCache(); + + const missing = []; + for (const file of planted) { + if (!(await exists(file))) missing.push(file); + } + expect(missing).toEqual([]); + expect(await remaining(thumbnailDir)).toEqual([]); + expect(await remaining(previewDir)).toEqual([]); + }); +}); diff --git a/backend/tests/services/raw-preview-exiftool.test.js b/backend/tests/services/raw-preview-exiftool.test.js new file mode 100644 index 000000000..99d40aeca --- /dev/null +++ b/backend/tests/services/raw-preview-exiftool.test.js @@ -0,0 +1,175 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { createRequire } from 'node:module'; +import { substituteModule } from '../helpers/substitute-module.js'; + +// The decision below is a pure function on the service; requiring it here +// rather than through the env harness keeps these cases free of a filesystem. +const { chooseExiftoolPath, EXIFTOOL_CANDIDATES } = createRequire(import.meta.url)( + '../../src/services/rawPreviewService.js' +); + +/** + * Which ExifTool reads a RAW file. + * + * The archive brings its own, which is 23 MB of Perl and the right default — + * one dependency less to explain, and the version this was tested against. + * Somebody installing outside a container may already have ExifTool and would + * rather not carry a second copy (#9), so `EXIFTOOL_PATH` points at theirs. + * + * What is held here is only which one is used: the library is the same, and + * everything after it is the same code. + */ + +const SERVICE_FILE = fileURLToPath( + new URL('../../src/services/rawPreviewService.js', import.meta.url) +); + +let currentEnv = null; +let restoreModule = null; + +afterEach(async () => { + if (restoreModule) { + restoreModule(); + restoreModule = null; + } + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +/** + * A stand-in for `exiftool-vendored` that says which of the two was asked. + * + * Neither extracts anything: what the caller does with a RAW file it cannot + * read is the same either way, and is not what this is about. + */ +const fakeVendored = () => { + const asked = []; + const built = []; + + const refuse = (which) => async () => { + asked.push(which); + throw new Error('nothing to extract'); + }; + + const bundled = { + extractPreview: refuse('bundled'), + extractThumbnail: refuse('bundled'), + extractJpgFromRaw: refuse('bundled'), + end: async () => {}, + }; + + class ExifTool { + constructor(options) { + built.push(options); + this.extractPreview = refuse('machine'); + this.extractThumbnail = refuse('machine'); + this.extractJpgFromRaw = refuse('machine'); + this.end = async () => {}; + } + } + + return { asked, built, module: { exiftool: bundled, ExifTool } }; +}; + +const askFor = async (env = {}) => { + currentEnv = await setupTestEnv({ tag: 'raw-preview-exiftool-', env }); + const fake = fakeVendored(); + restoreModule = substituteModule(SERVICE_FILE, 'exiftool-vendored', fake.module); + + const service = currentEnv.requireFresh('src/services/rawPreviewService'); + const raw = path.join(currentEnv.volumeDir, 'photo.cr2'); + await fs.writeFile(raw, Buffer.from('not really a raw file')); + + // It will fail — nothing here extracts anything. Which ExifTool it asked is + // the answer being looked for. + await expect(service.getRawPreviewJpegPath(raw)).rejects.toThrow(); + await service.stopRawPreviewWork?.(); + return fake; +}; + +describe('which ExifTool reads a RAW file', () => { + it('uses the one in the archive when nothing says otherwise', async () => { + const fake = await askFor(); + + expect(fake.built).toEqual([]); + expect(fake.asked).toContain('bundled'); + expect(fake.asked).not.toContain('machine'); + }); + + it("uses the machine's when EXIFTOOL_PATH names one", async () => { + const fake = await askFor({ EXIFTOOL_PATH: '/usr/bin/exiftool' }); + + expect(fake.built).toEqual([{ exiftoolPath: '/usr/bin/exiftool' }]); + expect(fake.asked).toContain('machine'); + expect(fake.asked).not.toContain('bundled'); + }); + + it('treats a blank setting as no setting at all', async () => { + const fake = await askFor({ EXIFTOOL_PATH: ' ' }); + + // A variable left empty in a configuration file is somebody who did not + // choose, not somebody who chose the empty path. + expect(fake.built).toEqual([]); + expect(fake.asked).toContain('bundled'); + }); +}); + +/** + * And which one it settles on when nobody said. + * + * The minimal archive leaves the 21 MB of Perl behind so a distribution can + * supply it, and until now that meant `apt install libimage-exiftool-perl` + * plus a variable nobody was told about (#9). The rule is read here on its + * own, because the probing around it is filesystem and the order is the part + * that can be wrong. + */ +describe('choosing an ExifTool when nothing named one', () => { + const choose = ({ named = '', vendored = false, present = [] } = {}) => + chooseExiftoolPath({ + named, + vendored, + candidates: EXIFTOOL_CANDIDATES, + runnable: (candidate) => present.includes(candidate), + }); + + it('takes what EXIFTOOL_PATH names, before anything else', () => { + expect( + choose({ named: '/opt/mine/exiftool', vendored: true, present: ['/usr/bin/exiftool'] }) + ).toBe('/opt/mine/exiftool'); + }); + + it('keeps the bundled one when it travelled', () => { + // The tested version, and the one the full archive carries: a machine that + // also has its own does not get quietly switched to it. + expect(choose({ vendored: true, present: ['/usr/bin/exiftool'] })).toBe(''); + }); + + it("falls to the machine's own when the bundled one is not there", () => { + expect(choose({ vendored: false, present: ['/usr/bin/exiftool'] })).toBe('/usr/bin/exiftool'); + }); + + it('prefers the first candidate over a later one', () => { + expect( + choose({ vendored: false, present: ['/usr/local/bin/exiftool', '/usr/bin/exiftool'] }) + ).toBe('/usr/bin/exiftool'); + }); + + it('says nothing rather than something wrong when there is none', () => { + // The caller then has no ExifTool, which is a supported state: no RAW + // metadata, and everything else carries on. + expect(choose({ vendored: false, present: [] })).toBe(''); + }); + + it('looks where a distribution puts it, and not on PATH', () => { + // The PATH a service inherits is whatever started it, and this one may be + // running as root — the same reason ffmpegRunner resolves absolute paths. + expect(EXIFTOOL_CANDIDATES.every((candidate) => candidate.startsWith('/'))).toBe(true); + expect(EXIFTOOL_CANDIDATES).toContain('/usr/bin/exiftool'); + }); +}); diff --git a/backend/tests/services/rename-entry.test.js b/backend/tests/services/rename-entry.test.js new file mode 100644 index 000000000..ded089c43 --- /dev/null +++ b/backend/tests/services/rename-entry.test.js @@ -0,0 +1,252 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Renaming is one of the two paths that write to disk after authorising, and + * neither had a test. Reading it found nothing wrong — the parent, the item, + * the new name and the target are each checked, in that order — which is + * exactly the kind of code where a regression is silent. + */ + +let envContext; +let renameEntry; +let context; +let plainContext; + +const volumePath = (...parts) => path.join(envContext.volumeDir, ...parts); + +const exists = async (target) => + fs + .access(target) + .then(() => true) + .catch(() => false); + +beforeEach(async () => { + envContext = await setupTestEnv({ tag: 'rename-entry-' }); + ({ renameEntry } = envContext.requireFresh('src/services/renameService')); + + const dbService = envContext.requireFresh('src/services/db'); + const db = await dbService.getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, '["admin"]', ?, ?)` + ).run('admin-1', 'admin@example.com', 'admin', 'Admin', now, now); + + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, '["user"]', ?, ?)` + ).run('user-1', 'user@example.com', 'user', 'User', now, now); + + context = { user: { id: 'admin-1', email: 'admin@example.com', roles: ['admin'] } }; + plainContext = { user: { id: 'user-1', email: 'user@example.com', roles: ['user'] } }; + + await fs.mkdir(volumePath('Documents'), { recursive: true }); + await fs.writeFile(volumePath('Documents', 'report.txt'), 'contents'); +}); + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('renaming an entry', () => { + it('moves it and says where it ended up', async () => { + const result = await renameEntry({ + context, + parentRelative: 'Documents', + currentName: 'report.txt', + newName: 'summary.txt', + }); + + expect(result.changed).toBe(true); + expect(result.name).toBe('summary.txt'); + expect(result.relativePath).toBe('Documents/summary.txt'); + expect(result.previousAbsolutePath).toBe(volumePath('Documents', 'report.txt')); + expect(await exists(volumePath('Documents', 'summary.txt'))).toBe(true); + expect(await exists(volumePath('Documents', 'report.txt'))).toBe(false); + expect(await fs.readFile(volumePath('Documents', 'summary.txt'), 'utf8')).toBe('contents'); + }); + + // The editor renames the document it has open, and needs to know whether + // anything moved so it can keep its session pointing at the file. + it('treats renaming to the same name as nothing to do', async () => { + const result = await renameEntry({ + context, + parentRelative: 'Documents', + currentName: 'report.txt', + newName: 'report.txt', + }); + + expect(result.changed).toBe(false); + expect(result.name).toBe('report.txt'); + expect(await exists(volumePath('Documents', 'report.txt'))).toBe(true); + }); + + it('refuses a name that is already taken', async () => { + await fs.writeFile(volumePath('Documents', 'taken.txt'), 'someone else'); + + await expect( + renameEntry({ + context, + parentRelative: 'Documents', + currentName: 'report.txt', + newName: 'taken.txt', + }) + ).rejects.toMatchObject({ statusCode: 409 }); + + // Neither file moved. + expect(await fs.readFile(volumePath('Documents', 'taken.txt'), 'utf8')).toBe('someone else'); + expect(await exists(volumePath('Documents', 'report.txt'))).toBe(true); + }); + + // A name with a separator in it is the caller's mistake, not the server's. + // Answering 500 sent everyone looking in the wrong place, the logs included. + it('answers a name with a path in it as a bad request', async () => { + for (const newName of ['../escape.txt', 'sub/report.txt', '..']) { + await expect( + renameEntry({ + context, + parentRelative: 'Documents', + currentName: 'report.txt', + newName, + }) + ).rejects.toMatchObject({ statusCode: 400 }); + } + + expect(await exists(volumePath('Documents', 'report.txt'))).toBe(true); + }); + + it('refuses an empty or missing name', async () => { + for (const newName of ['', null, undefined, 42]) { + await expect( + renameEntry({ + context, + parentRelative: 'Documents', + currentName: 'report.txt', + newName, + }) + ).rejects.toMatchObject({ statusCode: 400 }); + } + }); + + it('refuses when the original name is missing', async () => { + await expect( + renameEntry({ context, parentRelative: 'Documents', currentName: '', newName: 'a.txt' }) + ).rejects.toMatchObject({ statusCode: 400 }); + }); + + it('says so when there is nothing to rename', async () => { + await expect( + renameEntry({ + context, + parentRelative: 'Documents', + currentName: 'not-here.txt', + newName: 'anything.txt', + }) + ).rejects.toMatchObject({ statusCode: 404 }); + }); + + it('renames a folder with everything in it', async () => { + await fs.mkdir(volumePath('Documents', 'Project', 'inner'), { recursive: true }); + await fs.writeFile(volumePath('Documents', 'Project', 'inner', 'deep.txt'), 'deep'); + + const result = await renameEntry({ + context, + parentRelative: 'Documents', + currentName: 'Project', + newName: 'Archive', + }); + + expect(result.changed).toBe(true); + expect(await fs.readFile(volumePath('Documents', 'Archive', 'inner', 'deep.txt'), 'utf8')).toBe( + 'deep' + ); + }); +}); + +describe('what a rename must not walk past', () => { + const readOnly = async (relativePath, { recursive = true } = {}) => { + const accessControl = envContext.requireFresh('src/services/accessControlService'); + await accessControl.setRules([{ path: relativePath, permissions: 'ro', recursive }]); + }; + + // An administrator passes through a read-only rule on purpose — they are the + // one who set it. Everyone else is stopped by it. + it('refuses to rename inside a read-only folder', async () => { + await readOnly('Documents'); + + await expect( + renameEntry({ + context: plainContext, + parentRelative: 'Documents', + currentName: 'report.txt', + newName: 'summary.txt', + }) + ).rejects.toMatchObject({ statusCode: 403 }); + + expect(await exists(volumePath('Documents', 'report.txt'))).toBe(true); + expect(await exists(volumePath('Documents', 'summary.txt'))).toBe(false); + }); + + // Renaming is a write to the folder as much as to the item: the name lives + // in the directory. A rule that covers the folder alone still stops it, and + // it is the only case where the item's own permission would say yes. + it('refuses when only the folder itself is read-only', async () => { + await readOnly('Documents', { recursive: false }); + + await expect( + renameEntry({ + context: plainContext, + parentRelative: 'Documents', + currentName: 'report.txt', + newName: 'summary.txt', + }) + ).rejects.toMatchObject({ statusCode: 403 }); + + expect(await exists(volumePath('Documents', 'report.txt'))).toBe(true); + }); + + // The name being taken is checked too, not only the one being left. A rule + // can name a single path, and renaming *into* it is a write to it. + it('refuses when the name it would take is read-only', async () => { + await readOnly('Documents/summary.txt', { recursive: false }); + + await expect( + renameEntry({ + context: plainContext, + parentRelative: 'Documents', + currentName: 'report.txt', + newName: 'summary.txt', + }) + ).rejects.toMatchObject({ statusCode: 403 }); + + expect(await exists(volumePath('Documents', 'report.txt'))).toBe(true); + expect(await exists(volumePath('Documents', 'summary.txt'))).toBe(false); + }); +}); + +describe('what follows a renamed folder', () => { + it('takes its favourites with it', async () => { + await fs.mkdir(volumePath('Documents', 'Project'), { recursive: true }); + + const favorites = envContext.requireFresh('src/services/favoritesService'); + await favorites.addFavorite('admin-1', { + path: 'Documents/Project', + label: 'Project', + }); + + await renameEntry({ + context, + parentRelative: 'Documents', + currentName: 'Project', + newName: 'Archive', + }); + + const after = await favorites.getFavorites('admin-1'); + expect(after.map((entry) => entry.path)).toContain('Documents/Archive'); + expect(after.map((entry) => entry.path)).not.toContain('Documents/Project'); + }); +}); diff --git a/backend/tests/services/request-user.test.js b/backend/tests/services/request-user.test.js new file mode 100644 index 000000000..df301ea66 --- /dev/null +++ b/backend/tests/services/request-user.test.js @@ -0,0 +1,250 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Who the application decides you are. + * + * This is where an account is looked up, and where it is given the personal + * folder name it keeps for the rest of its life. It had the lowest coverage of + * anything in the personal-folder feature — forty-four per cent of statements, + * twenty-eight of branches — and the whole OIDC half of it was untouched. + * + * That half held a defect. An account whose row does not exist yet gets a user + * object assembled from the provider's claims, and it carried no folder name at + * all. A name absent is a name derived, and `USER_FOLDER_NAME_ORDER` — in the + * order the reference recommends for reusing /home — puts `username` first, + * which two identities from two providers can share. The claim mechanism that + * exists to prevent exactly that cannot run without a row to write to. + */ + +const MODULES = [ + 'src/config/env', + 'src/config/index', + 'src/services/db', + 'src/services/users/requestUser', +]; + +const OIDC_ENV = { + AUTH_ENABLED: 'true', + OIDC_ENABLED: 'true', + OIDC_ISSUER: 'https://idp.example', + OIDC_CLIENT_ID: 'nextexplorer', + USER_DIR_ENABLED: 'true', +}; + +let envContext; + +const build = async (env = {}) => { + envContext = await setupTestEnv({ + tag: 'request-user-', + env: { ...OIDC_ENV, ...env }, + modules: MODULES, + }); + const { getRequestUser } = envContext.requireFresh('src/services/users/requestUser'); + const db = await envContext.requireFresh('src/services/db').getDb(); + return { getRequestUser, db }; +}; + +const seedAccount = (db, { id = 'user-1', username = 'someone', folderName = null } = {}) => { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, personal_folder_name, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, '["user"]', ?, ?, ?)` + ).run(id, `${username}@example.com`, username, username, folderName, now, now); + return id; +}; + +const linkOidc = (db, { userId = 'user-1', sub = 'sub-1' } = {}) => { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO auth_methods (id, user_id, method_type, provider_issuer, provider_sub, provider_name, created_at) + VALUES (?, ?, 'oidc', 'https://idp.example', ?, 'OIDC', ?)` + ).run(`auth-${sub}`, userId, sub, now); +}; + +const oidcRequest = (claims) => ({ + oidc: { isAuthenticated: () => true, user: claims }, +}); + +afterEach(async () => { + if (envContext) { + await envContext.cleanup(); + envContext = null; + } +}); + +describe('a request that already carries a user', () => { + it('is answered with that user', async () => { + const { getRequestUser } = await build(); + const user = { id: 'synthetic', roles: ['admin'] }; + + expect(await getRequestUser({ user })).toBe(user); + }); + + it('is not answered by an object with no id', async () => { + const { getRequestUser, db } = await build(); + seedAccount(db, { folderName: 'someone' }); + + const result = await getRequestUser({ user: {}, session: { localUserId: 'user-1' } }); + + expect(result.id).toBe('user-1'); + }); +}); + +describe('a local session', () => { + it('finds the account', async () => { + const { getRequestUser, db } = await build(); + seedAccount(db, { folderName: 'someone' }); + + const user = await getRequestUser({ session: { localUserId: 'user-1' } }); + + expect(user).toMatchObject({ id: 'user-1', provider: 'local' }); + }); + + /** The claim happens here for an account created any other way. */ + it('gives an account without a folder name one, and keeps it', async () => { + const { getRequestUser, db } = await build(); + seedAccount(db, { folderName: null }); + + const first = await getRequestUser({ session: { localUserId: 'user-1' } }); + const second = await getRequestUser({ session: { localUserId: 'user-1' } }); + + expect(first.personalFolderName).toBeTruthy(); + expect(second.personalFolderName).toBe(first.personalFolderName); + }); + + it('leaves a name it already holds alone', async () => { + const { getRequestUser, db } = await build(); + seedAccount(db, { folderName: 'chosen-long-ago' }); + + const user = await getRequestUser({ session: { localUserId: 'user-1' } }); + + expect(user.personalFolderName).toBe('chosen-long-ago'); + }); +}); + +describe('an OIDC session for an account that exists', () => { + it('finds it through its provider subject', async () => { + const { getRequestUser, db } = await build(); + seedAccount(db, { folderName: 'someone' }); + linkOidc(db); + + const user = await getRequestUser(oidcRequest({ sub: 'sub-1' })); + + expect(user).toMatchObject({ + id: 'user-1', + provider: 'oidc', + oidcIssuer: 'https://idp.example', + }); + }); + + it('takes the picture from the claims when the account has none', async () => { + const { getRequestUser, db } = await build(); + seedAccount(db, { folderName: 'someone' }); + linkOidc(db); + + const user = await getRequestUser( + oidcRequest({ sub: 'sub-1', picture: ' https://idp.example/me.png ' }) + ); + + expect(user.avatarUrl).toBe('https://idp.example/me.png'); + }); + + it('ignores a picture claim that is only whitespace', async () => { + const { getRequestUser, db } = await build(); + seedAccount(db, { folderName: 'someone' }); + linkOidc(db); + + const user = await getRequestUser(oidcRequest({ sub: 'sub-1', picture: ' ' })); + + expect(user.avatarUrl).toBeFalsy(); + }); + + it('claims a folder name for it if it has none yet', async () => { + const { getRequestUser, db } = await build(); + seedAccount(db, { folderName: null }); + linkOidc(db); + + const user = await getRequestUser(oidcRequest({ sub: 'sub-1' })); + + expect(user.personalFolderName).toBeTruthy(); + }); +}); + +describe('an OIDC session for an account with no row yet', () => { + const claims = { + sub: 'sub-unsynced', + email: 'Someone@Example.com', + preferred_username: 'someone', + name: 'Someone', + }; + + it('is answered from the claims', async () => { + const { getRequestUser } = await build(); + + const user = await getRequestUser(oidcRequest(claims)); + + expect(user).toMatchObject({ id: 'oidc:sub-unsynced', provider: 'oidc', username: 'someone' }); + }); + + it('normalises the email', async () => { + const { getRequestUser } = await build(); + + const user = await getRequestUser(oidcRequest(claims)); + + expect(user.email).toBe('someone@example.com'); + }); + + /** + * The defect this file was written for. Without a name of its own the folder + * is derived, and `username` is the first candidate in the order the + * documentation recommends — so two identities sharing a preferred username + * would be handed the same directory, with no row for the claim to protect. + */ + it('carries a folder name of its own', async () => { + const { getRequestUser } = await build(); + + const user = await getRequestUser(oidcRequest(claims)); + + expect(user.personalFolderName).toBeTruthy(); + }); + + it('takes that name from the subject, not from the username', async () => { + const { getRequestUser } = await build(); + + const user = await getRequestUser(oidcRequest(claims)); + + expect(user.personalFolderName).toContain('sub-unsynced'); + expect(user.personalFolderName).not.toBe('someone'); + }); + + it('gives two identities sharing a username two different folders', async () => { + const { getRequestUser } = await build(); + + const first = await getRequestUser(oidcRequest({ ...claims, sub: 'sub-a' })); + const second = await getRequestUser(oidcRequest({ ...claims, sub: 'sub-b' })); + + expect(first.personalFolderName).not.toBe(second.personalFolderName); + }); + + /** With auto-creation off, an unknown subject is nobody rather than somebody. */ + it('is nobody when accounts are not created automatically', async () => { + const { getRequestUser } = await build({ OIDC_AUTO_CREATE_USERS: 'false' }); + + expect(await getRequestUser(oidcRequest(claims))).toBeNull(); + }); +}); + +describe('a request with nothing to go on', () => { + it('is nobody', async () => { + const { getRequestUser } = await build(); + + expect(await getRequestUser({})).toBeNull(); + }); + + it('is nobody when there is no request at all', async () => { + const { getRequestUser } = await build(); + + expect(await getRequestUser(undefined)).toBeNull(); + }); +}); diff --git a/backend/tests/services/search-index-forget-folder.test.js b/backend/tests/services/search-index-forget-folder.test.js new file mode 100644 index 000000000..a38e6d9ae --- /dev/null +++ b/backend/tests/services/search-index-forget-folder.test.js @@ -0,0 +1,145 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Forgetting a folder: what the index does when one is excluded from Settings, + * deleted, or moved out of the volume. + * + * It used to go row by row in one go, each delete its own commit, on the only + * thread the server has. Excluding a large folder froze the application for as + * long as that took — reported in #11 from an instance indexing a network + * share, and measured here at two seconds for fifty thousand files on a fast + * machine. And the folder was found with `LIKE`, which ignores case: `Archive` + * took `archive` with it. + */ + +let envContext; +let db; +let store; + +const build = async () => { + envContext = await setupTestEnv({ tag: 'search-index-forget-' }); + db = await envContext.requireFresh('src/services/indexDb').getIndexDb(); + store = envContext.requireFresh('src/services/searchIndexStore'); +}; + +const put = (path, text) => store.upsertDocument(db, { path, mtimeMs: Date.now(), size: 10, text }); + +const paths = () => + db + .prepare('SELECT path FROM search_documents ORDER BY path') + .all() + .map((row) => row.path); + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('forgetting a folder', () => { + it('takes the folder and everything under it, and nothing beside it', async () => { + await build(); + put('Archive', undefined); + put('Archive/a.txt', 'pangolin'); + put('Archive/sub/b.txt', 'pangolin'); + // Beside it, each sharing something with it. + put('Archive2/c.txt', 'pangolin'); + put('Archive-old/d.txt', 'pangolin'); + put('Archive.txt', 'pangolin'); + put('Current/Archive/e.txt', 'pangolin'); + // Another folder on a Linux volume, not the same one spelt differently. + put('archive/f.txt', 'pangolin'); + + const removed = await store.removeUnder(db, 'Archive'); + + expect(removed).toBe(3); + expect(paths()).toEqual([ + 'Archive-old/d.txt', + 'Archive.txt', + 'Archive2/c.txt', + 'Current/Archive/e.txt', + 'archive/f.txt', + ]); + // The words went with the rows: nothing answers from a forgotten file. + expect(store.search(db, 'pangolin').sort()).toEqual([ + 'Archive-old/d.txt', + 'Archive.txt', + 'Archive2/c.txt', + 'Current/Archive/e.txt', + 'archive/f.txt', + ]); + }); + + it('takes a single file by its own path', async () => { + await build(); + put('Docs/a.txt', 'pangolin'); + put('Docs/a.txt.bak', 'pangolin'); + + expect(await store.removeUnder(db, 'Docs/a.txt')).toBe(1); + expect(paths()).toEqual(['Docs/a.txt.bak']); + }); + + it('reads the characters of a name literally', async () => { + await build(); + put('100%_done/x.txt', 'pangolin'); + put('100X_done/y.txt', 'pangolin'); + + expect(await store.removeUnder(db, '100%_done')).toBe(1); + expect(paths()).toEqual(['100X_done/y.txt']); + }); + + it('forgets nothing when given no folder', async () => { + await build(); + put('Docs/a.txt', 'pangolin'); + + expect(await store.removeUnder(db, '')).toBe(0); + expect(paths()).toEqual(['Docs/a.txt']); + }); +}); + +describe('the server while a folder is forgotten', () => { + it('goes on answering between two batches', async () => { + await build(); + db.transaction(() => { + for (let index = 0; index < 2500; index += 1) put(`Archive/file-${index}.txt`, 'pangolin'); + })(); + + // A turn of the event loop asked for after the removal has started: if + // it runs before the removal is over, anything else waiting — a listing, + // a download, another account — would have been answered too. + let finished = false; + const removal = Promise.resolve(store.removeUnder(db, 'Archive', { batchSize: 500 })).then( + (count) => { + finished = true; + return count; + } + ); + let answeredMeanwhile = null; + setImmediate(() => { + answeredMeanwhile = !finished; + }); + + expect(await removal).toBe(2500); + expect(answeredMeanwhile).toBe(true); + expect(paths()).toEqual([]); + }); + + it('pauses once between each two batches, and not after the last', async () => { + await build(); + db.transaction(() => { + for (let index = 0; index < 2500; index += 1) put(`Archive/file-${index}.txt`, undefined); + })(); + let pauses = 0; + + const removed = await store.removeUnder(db, 'Archive', { + batchSize: 1000, + pause: async () => { + pauses += 1; + }, + }); + + expect(removed).toBe(2500); + // 1000, 1000, then 500: the short batch says there is nothing left. + expect(pauses).toBe(2); + }); +}); diff --git a/backend/tests/services/search-index-move-folder.test.js b/backend/tests/services/search-index-move-folder.test.js new file mode 100644 index 000000000..cfd7c9f4a --- /dev/null +++ b/backend/tests/services/search-index-move-folder.test.js @@ -0,0 +1,178 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Following a folder the application moved or renamed. + * + * Its rows were found with `LIKE`, which ignores case: moving `Docs` carried + * `docs/…` along, another folder on a Linux volume, listed afterwards under a + * path that does not exist. And the parent written for each row ended in a + * slash, which no parent does: a search from the moved folder missed the files + * directly in it, and the next pass took `Papers/` for a folder that was gone, + * forgot every row of it, and read the whole folder again on the pass after. + */ + +let envContext; +let db; +let store; + +const build = async () => { + envContext = await setupTestEnv({ tag: 'search-index-move-' }); + db = await envContext.requireFresh('src/services/indexDb').getIndexDb(); + store = envContext.requireFresh('src/services/searchIndexStore'); +}; + +const put = (relativePath, text, isDirectory = false) => + store.upsertDocument(db, { + path: relativePath, + mtimeMs: Date.now(), + size: 10, + text, + isDirectory, + }); + +const rows = () => + db.prepare('SELECT path, dir, name_fold AS name FROM search_documents ORDER BY path').all(); + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('moving a folder', () => { + it('takes the folder and everything under it, and nothing beside it', async () => { + await build(); + put('Docs', undefined, true); + put('Docs/a.txt', 'pangolin'); + put('Docs/sub', undefined, true); + put('Docs/sub/b.txt', 'pangolin'); + // Beside it, each sharing something with it. + put('Docs2/c.txt', 'pangolin'); + put('Docs.txt', 'pangolin'); + put('Other/Docs/e.txt', 'pangolin'); + // Another folder on a Linux volume, not the same one spelt differently. + put('docs/d.txt', 'pangolin'); + + expect(store.movePath(db, 'Docs', 'Papers')).toBe(4); + + expect(rows().map((row) => row.path)).toEqual([ + 'Docs.txt', + 'Docs2/c.txt', + 'Other/Docs/e.txt', + 'Papers', + 'Papers/a.txt', + 'Papers/sub', + 'Papers/sub/b.txt', + 'docs/d.txt', + ]); + // The words moved with the rows. + expect(store.search(db, 'pangolin').sort()).toEqual([ + 'Docs.txt', + 'Docs2/c.txt', + 'Other/Docs/e.txt', + 'Papers/a.txt', + 'Papers/sub/b.txt', + 'docs/d.txt', + ]); + }); + + it('gives each row the parent it now has, and the folder its new name', async () => { + await build(); + put('Docs', undefined, true); + put('Docs/a.txt', 'pangolin'); + put('Docs/sub', undefined, true); + put('Docs/sub/deep/b.txt', 'pangolin'); + + store.movePath(db, 'Docs', 'Archive/2026/Papers'); + + expect(rows()).toEqual([ + { path: 'Archive/2026/Papers', dir: 'Archive/2026', name: 'papers' }, + { path: 'Archive/2026/Papers/a.txt', dir: 'Archive/2026/Papers', name: 'a.txt' }, + { path: 'Archive/2026/Papers/sub', dir: 'Archive/2026/Papers', name: 'sub' }, + { + path: 'Archive/2026/Papers/sub/deep/b.txt', + dir: 'Archive/2026/Papers/sub/deep', + name: 'b.txt', + }, + ]); + }); + + it('moves a single file by its own path', async () => { + await build(); + put('Docs/a.txt', 'pangolin'); + put('Docs/a.txt.bak', 'pangolin'); + + expect(store.movePath(db, 'Docs/a.txt', 'Docs/b.txt')).toBe(1); + expect(rows()).toEqual([ + { path: 'Docs/a.txt.bak', dir: 'Docs', name: 'a.txt.bak' }, + { path: 'Docs/b.txt', dir: 'Docs', name: 'b.txt' }, + ]); + }); + + it('is found from the folder it moved to', async () => { + await build(); + put('Docs', undefined, true); + put('Docs/a.txt', 'pangolin'); + + store.movePath(db, 'Docs', 'Papers'); + + // By name and by contents, asked from inside the folder, as a search + // started there asks. + expect([...store.iterateNameCandidates(db, { base: 'Papers', literal: 'a.txt' })]).toEqual([ + 'Papers/a.txt', + ]); + expect( + store.searchRanked(db, 'pangolin', 10, { base: 'Papers' }).map((row) => row.path) + ).toEqual(['Papers/a.txt']); + }); +}); + +describe('the pass after a move', () => { + it('neither forgets what moved nor reads it again', async () => { + await build(); + const volume = envContext.volumeDir; + await fs.mkdir(path.join(volume, 'Docs', 'sub'), { recursive: true }); + await fs.writeFile(path.join(volume, 'Docs', 'a.txt'), 'le pangolin\n'); + await fs.writeFile(path.join(volume, 'Docs', 'sub', 'b.txt'), 'un autre pangolin\n'); + const { indexTree } = envContext.requireFresh('src/services/searchIndexer'); + await indexTree({ db, rootAbs: volume, cpuPercent: 100 }); + + // What the application does on a rename: the disk, then the index. + await fs.rename(path.join(volume, 'Docs'), path.join(volume, 'Papers')); + store.movePath(db, 'Docs', 'Papers'); + const pass = await indexTree({ db, rootAbs: volume, cpuPercent: 100 }); + + expect(pass.removed).toBe(0); + expect(pass.indexed).toBe(0); + expect(store.search(db, 'pangolin').sort()).toEqual(['Papers/a.txt', 'Papers/sub/b.txt']); + }); +}); + +describe('an index a move had already written wrongly', () => { + it('has its folders put right when it is opened, once', async () => { + await build(); + // Opening it has already looked, and says so. + expect( + db + .prepare('SELECT value FROM meta WHERE key = ?') + .pluck() + .get('search_index_moved_dirs_repaired') + ).toBe('1'); + + put('Papers/a.txt', 'pangolin'); + put('Papers/sub/b.txt', 'pangolin'); + // What the old move left behind. + db.prepare("UPDATE search_documents SET dir = dir || '/'").run(); + db.prepare('DELETE FROM meta WHERE key = ?').run('search_index_moved_dirs_repaired'); + + expect(store.repairMovedDirs(db)).toBe(2); + expect(rows().map((row) => row.dir)).toEqual(['Papers', 'Papers/sub']); + + // Not again: nothing writes such a parent any more, and looking is a scan + // of the whole table. + db.prepare("UPDATE search_documents SET dir = dir || '/'").run(); + expect(store.repairMovedDirs(db)).toBe(0); + }); +}); diff --git a/backend/tests/services/search-index-view.test.js b/backend/tests/services/search-index-view.test.js new file mode 100644 index 000000000..7c626ac08 --- /dev/null +++ b/backend/tests/services/search-index-view.test.js @@ -0,0 +1,125 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The way back from a row of the index to the name a reader uses. + * + * The queries already stay inside the folder asked about, so from the route + * none of the refusals below can be reached — which is exactly why they are + * tested here: they are what stands between a mistake in a query and a file + * handed to somebody who was never shown the folder it is in. + */ + +let envContext; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +const load = async () => { + envContext = await setupTestEnv({ tag: 'search-index-view-' }); + return envContext.requireFresh('src/services/searchIndexView'); +}; + +const neverIgnored = () => false; + +describe('the way back from a row', () => { + it('names a row under the folder in the words of the reader', async () => { + const { createIndexView } = await load(); + const view = createIndexView({ + base: 'Docs/partage', + logicalBase: 'share/abc', + isIgnoredName: neverIgnored, + }); + + expect(view.toLogical('Docs/partage/sous/rapport.txt')).toBe('share/abc/sous/rapport.txt'); + expect(view.toAbsolute('Docs/partage/sous/rapport.txt')).toBe( + path.join(envContext.volumeDir, 'Docs/partage/sous/rapport.txt') + ); + }); + + it('is the identity for the volume searched under its own name', async () => { + const { createIndexView } = await load(); + const view = createIndexView({ base: '', logicalBase: '', isIgnoredName: neverIgnored }); + + expect(view.toLogical('Docs/rapport.txt')).toBe('Docs/rapport.txt'); + }); + + it('refuses whatever is not strictly under the folder', async () => { + const { createIndexView } = await load(); + const view = createIndexView({ + base: 'Docs/partage', + logicalBase: 'share/abc', + isIgnoredName: neverIgnored, + }); + + for (const row of [ + 'Docs/partage-bis/rapport.txt', // the same letters, another folder + 'Docs/hors.txt', + 'Docs/partage', // the folder itself, where the reader stands + 'Docs/partage/', + 'Docs/partage//double.txt', + 'Autre/Docs/partage/x.txt', + '', + null, + ]) { + expect(view.toLogical(row)).toBeNull(); + } + }); + + it('refuses a row inside a folder the search never enters', async () => { + const { createIndexView } = await load(); + const view = createIndexView({ + base: 'Docs', + logicalBase: 'Docs', + isIgnoredName: (name) => name === '_users' || name.startsWith('.'), + }); + + expect(view.toLogical('Docs/_users/x.txt')).toBeNull(); + expect(view.toLogical('Docs/.cache/x.txt')).toBeNull(); + // The entry itself is judged by its name elsewhere, with the reader's + // setting for hidden files; only the folders on the way are refused here. + expect(view.toLogical('Docs/.profile')).toBe('Docs/.profile'); + }); + + // An assigned volume or its share can hold the personal folders without the + // reader having any claim on them. The name check above would stop `_users`; + // this is the check that knows where the personal folders actually are. + it("refuses somebody's personal folder reached from outside it", async () => { + const { createIndexView } = await load(); + const view = createIndexView({ base: '', logicalBase: 'Tout', isIgnoredName: neverIgnored }); + + expect(view.toLogical('_users/bob/secret.txt')).toBeNull(); + expect(view.toLogical('Public/ouvert.txt')).toBe('Tout/Public/ouvert.txt'); + }); + + it('answers inside a personal folder searched from inside it', async () => { + const { createIndexView } = await load(); + const view = createIndexView({ + base: '_users/bob', + logicalBase: 'personal', + isIgnoredName: neverIgnored, + baseInPersonalRoot: true, + }); + + expect(view.toLogical('_users/bob/notes/a.txt')).toBe('personal/notes/a.txt'); + expect(view.toLogical('_users/bobby/a.txt')).toBeNull(); + }); +}); + +describe('where a folder sits in the volume', () => { + it('is found through a link, and not outside the volume', async () => { + const { volumePathOf } = await load(); + const volume = envContext.volumeDir; + await fs.mkdir(path.join(volume, 'Docs', 'vrai'), { recursive: true }); + await fs.symlink(path.join(volume, 'Docs', 'vrai'), path.join(volume, 'Docs', 'lien')); + + expect(await volumePathOf(volume)).toBe(''); + expect(await volumePathOf(path.join(volume, 'Docs', 'lien'))).toBe('Docs/vrai'); + expect(await volumePathOf(envContext.configDir)).toBeNull(); + expect(await volumePathOf(path.join(volume, 'absent'))).toBeNull(); + }); +}); diff --git a/backend/tests/services/search-index.test.js b/backend/tests/services/search-index.test.js index e85a7420d..e91f8c955 100644 --- a/backend/tests/services/search-index.test.js +++ b/backend/tests/services/search-index.test.js @@ -134,7 +134,6 @@ describe('being interruptible', () => { await build(); await fs.mkdir(volumePath('Docs'), { recursive: true }); for (let index = 0; index < 60; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(volumePath('Docs', `file-${index}.txt`), `document ${index} pangolin\n`); } }); @@ -245,7 +244,6 @@ describe('how much it holds at once', () => { // Six documents of two megabytes each: counted in documents that is one // batch, counted in bytes it cannot be. for (let index = 0; index < 6; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile( volumePath('Docs', `big-${index}.txt`), `pangolin ${'lorem ipsum dolor sit amet '.repeat(80000)}` @@ -339,7 +337,6 @@ describe('what a pass costs', () => { // it costs the machine, so time is what it has to be paid in. it('stands aside on elapsed time, not on how many files went by', async () => { for (let index = 0; index < 40; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(volumePath('Docs', `note-${index}.txt`), `pangolin ${index}\n`); } @@ -381,7 +378,6 @@ describe('what a pass costs', () => { // large volume is where the two gigabytes came from. it('compiles its queries once, not once per document', async () => { for (let index = 0; index < 40; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(volumePath('Docs', `note-${index}.txt`), `pangolin ${index}\n`); } @@ -408,7 +404,6 @@ describe('what a pass costs', () => { // end the indexing that happens to be running at the same moment. it('does not stop on one reading that another task caused', async () => { for (let index = 0; index < 40; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(volumePath('Docs', `note-${index}.txt`), `pangolin ${index}\n`); } @@ -432,7 +427,6 @@ describe('what a pass costs', () => { // costs; this one holds when a belief turns out to be wrong. it('stops rather than let the process grow without end', async () => { for (let index = 0; index < 40; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(volumePath('Docs', `note-${index}.txt`), `pangolin ${index}\n`); } @@ -480,7 +474,6 @@ describe('how much runs at once', () => { it('reads one announced file at a time, however many are announced', async () => { for (let index = 0; index < 30; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(volumePath('Docs', `note-${index}.txt`), `pangolin ${index}\n`); } @@ -561,12 +554,14 @@ describe('forgetting what is gone', () => { }); // Nothing walks a folder that is not there, so nothing asks what it held. + // Three rows go, not two: a folder has a row of its own now, so that somebody + // can find it by its name without knowing what is inside it. it('forgets a folder that was removed outright', async () => { await fs.rm(volumePath('Docs', 'Notes'), { recursive: true }); const result = await indexAll(); - expect(result.removed).toBe(2); + expect(result.removed).toBe(3); expect(store.search(db, 'pangolin')).toEqual(['Docs/kept.txt']); }); @@ -575,7 +570,6 @@ describe('forgetting what is gone', () => { it('keeps the deletions it was sure of when it is cut short', async () => { await fs.rm(volumePath('Docs', 'Notes', 'one.txt')); for (let index = 0; index < 60; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(volumePath('Docs', `filler-${index}.txt`), `pangolin ${index}\n`); } @@ -609,16 +603,14 @@ describe('saying why a file was read again', () => { await fs.mkdir(volumePath('Docs', 'Churn'), { recursive: true }); const settled = new Date(1_700_000_000_000); for (let index = 0; index < 8; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(volumePath('Docs', 'Churn', `c-${index}.txt`), `pangolin ${index}\n`); - // eslint-disable-next-line no-await-in-loop + await fs.utimes(volumePath('Docs', 'Churn', `c-${index}.txt`), settled, settled); } await indexAll(); const moved = new Date(1_700_000_060_000); for (let index = 0; index < 8; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.utimes(volumePath('Docs', 'Churn', `c-${index}.txt`), moved, moved); } @@ -667,9 +659,8 @@ describe('saying why a file was read again', () => { // report exists to tell apart from a signal. const before = new Date(1_700_000_000_000); for (let index = 0; index < 6; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.writeFile(volumePath('Docs', 'Bruyant', `n-${index}.txt`), `pangolin ${index}\n`); - // eslint-disable-next-line no-await-in-loop + await fs.utimes(volumePath('Docs', 'Bruyant', `n-${index}.txt`), before, before); } await indexAll(); @@ -678,7 +669,6 @@ describe('saying why a file was read again', () => { // signature of storage that rounds, and of nothing else. const shifted = new Date(1_700_000_120_000); for (let index = 0; index < 6; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.utimes(volumePath('Docs', 'Bruyant', `n-${index}.txt`), shifted, shifted); } diff --git a/backend/tests/services/settings-without-json.test.js b/backend/tests/services/settings-without-json.test.js new file mode 100644 index 000000000..d9416513d --- /dev/null +++ b/backend/tests/services/settings-without-json.test.js @@ -0,0 +1,130 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import Database from 'better-sqlite3'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Settings live in app.db, and only there. + * + * They used to be mirrored into app-config.json by one save path and read back + * from it whenever app.db could not be read. The screens save through another + * path, so the file stopped following the settings — and a read that failed ran + * with whatever the file held, usually no access rules at all: reproduced, a + * folder hidden by a rule answered `rw` for as long as the read kept failing. + * The file is still read once by the migrations that carry very old settings + * into app.db; nothing at runtime reads or writes it. + */ + +let envContext; + +afterEach(async () => { + vi.restoreAllMocks(); + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +const setup = async () => { + envContext = await setupTestEnv({ tag: 'settings-without-json-' }); + const db = await envContext.requireFresh('src/services/db').getDb(); + const accessControl = envContext.requireFresh('src/services/accessControlService'); + const settings = envContext.requireFresh('src/services/settingsService'); + return { db, accessControl, settings, file: path.join(envContext.configDir, 'app-config.json') }; +}; + +/** + * Make the read of the system settings fail, the way a damaged database does. + * + * Only that read: the branding is read from the same table, and failing both + * would let a fallback in the system settings hide behind the branding's own + * failure. + */ +const breakSettingsReads = (db) => { + const proto = Object.getPrototypeOf(db.prepare('SELECT 1')); + for (const method of ['get', 'all']) { + const original = proto[method]; + vi.spyOn(proto, method).mockImplementation(function (...args) { + if (/FROM system_settings WHERE category = \?\s*$/.test(this.source)) { + throw Object.assign(new Error('database disk image is malformed'), { + code: 'SQLITE_CORRUPT', + }); + } + return original.apply(this, args); + }); + } +}; + +describe('access rules when the settings cannot be read', () => { + it('refuse to answer rather than let a hidden folder open', async () => { + const { db, accessControl } = await setup(); + await accessControl.setRules([{ path: 'Secret', recursive: true, permissions: 'hidden' }]); + expect(await accessControl.getPermissionForPath('Secret/plan.pdf')).toBe('hidden'); + + breakSettingsReads(db); + + await expect(accessControl.getPermissionForPath('Secret/plan.pdf')).rejects.toThrow( + /malformed/ + ); + }); + + it('do not fall back to an app-config.json left on disk, whatever it says', async () => { + const { db, accessControl, file } = await setup(); + await accessControl.setRules([{ path: 'Secret', recursive: true, permissions: 'hidden' }]); + fs.writeFileSync( + file, + JSON.stringify({ version: 4, settings: { access: { rules: [] } }, favorites: [] }) + ); + + breakSettingsReads(db); + + await expect(accessControl.getPermissionForPath('Secret/plan.pdf')).rejects.toThrow(); + }); +}); + +describe('app-config.json at runtime', () => { + it('is not created by reading the settings of a new installation', async () => { + const { settings, accessControl, file } = await setup(); + + await settings.getPublicSettings(); + await settings.getSettings(); + await accessControl.getRules(); + + expect(fs.existsSync(file)).toBe(false); + }); + + it('is not written by saving settings, and one already there is left as it was', async () => { + const { accessControl, settings, file } = await setup(); + const before = JSON.stringify({ version: 4, settings: {}, favorites: [] }); + fs.writeFileSync(file, before); + + await accessControl.setRules([{ path: 'Secret', recursive: true, permissions: 'ro' }]); + await settings.setSettings({ thumbnails: { quality: 60 } }); + + expect(fs.readFileSync(file, 'utf8')).toBe(before); + expect(await accessControl.getPermissionForPath('Secret/a.txt')).toBe('ro'); + }); + + it('gives the sign-in page the default branding when none was saved', async () => { + const { settings } = await setup(); + + const { branding } = await settings.getPublicSettings(); + + expect(branding).toEqual(expect.objectContaining({ appName: expect.any(String) })); + }); + + it('keeps the default branding when the saved one cannot be parsed', async () => { + const { db, settings } = await setup(); + db.prepare( + "INSERT INTO system_settings (id, category, key, value, updated_at) VALUES ('b', 'branding', 'branding', '{not json', ?)" + ).run(new Date().toISOString()); + + const { branding } = await settings.getPublicSettings(); + + expect(branding).toEqual(expect.objectContaining({ appName: expect.any(String) })); + }); +}); + +// Database is imported for its prototype only through the connection above. +void Database; diff --git a/backend/tests/services/settings.test.js b/backend/tests/services/settings.test.js index cce8f1696..ebbf26e37 100644 --- a/backend/tests/services/settings.test.js +++ b/backend/tests/services/settings.test.js @@ -1,11 +1,7 @@ import { describe, it, expect } from 'vitest'; import { setupTestEnv } from '../helpers/env-test-utils.js'; -const SETTINGS_MODULES = [ - 'src/services/storage/jsonStorage', - 'src/services/settingsService', - 'src/services/db', -]; +const SETTINGS_MODULES = ['src/services/settingsService', 'src/services/db']; const createSettingsContext = async () => { const envContext = await setupTestEnv({ @@ -29,6 +25,8 @@ describe('Settings Service', () => { expect(settings.thumbnails.size).toBe(200); expect(settings.thumbnails.quality).toBe(70); expect(settings.thumbnails.concurrency).toBe(10); + expect(settings.uploads.chunkedEnabled).toBe(false); + expect(settings.uploads.chunkSizeBytes).toBe(8 * 1024 * 1024); } finally { await envContext.cleanup(); } @@ -36,18 +34,15 @@ describe('Settings Service', () => { }); describe('setSettings', () => { - it('should sanitize thumbnails and filter access rules', async () => { + it('should sanitize thumbnails and uploads, and tidy a rule path', async () => { const { envContext, settingsService } = await createSettingsContext(); try { const payload = { thumbnails: { size: 5000, quality: 150, concurrency: -2 }, access: { - rules: [ - { path: '/Projects', permissions: 'ro', recursive: true }, - { path: 'uploads', permissions: 'invalid', recursive: false }, - { path: '../bad', permissions: 'hidden' }, - ], + rules: [{ path: '/Projects', permissions: 'ro', recursive: true }], }, + uploads: { chunkedEnabled: true, chunkSizeBytes: 512 }, }; const updated = await settingsService.setSettings(payload); @@ -56,9 +51,42 @@ describe('Settings Service', () => { expect(updated.thumbnails.quality).toBe(100); expect(updated.thumbnails.concurrency).toBe(1); expect(updated.thumbnails.enabled).toBe(true); - expect(updated.access.rules.length).toBe(2); + expect(updated.access.rules.length).toBe(1); expect(updated.access.rules[0].path).toBe('Projects'); - expect(updated.access.rules[1].permissions).toBe('rw'); + expect(updated.uploads.chunkedEnabled).toBe(true); + expect(updated.uploads.chunkSizeBytes).toBe(1024 * 1024); + } finally { + await envContext.cleanup(); + } + }); + + /** + * A number out of its bounds is brought within them, because every value in + * the range means the same kind of thing. A rule is not like that: there is + * no nearest valid folder for `../bad`, and the nearest valid permissions + * for a misspelt `readonly` used to be `rw` — the opposite of what was + * meant. Both are answered instead, and nothing is stored. + */ + it('should refuse an access rule it cannot store rather than repair it', async () => { + const { envContext, settingsService } = await createSettingsContext(); + try { + await settingsService.setSettings({ + access: { rules: [{ path: 'Projects', permissions: 'ro', recursive: true }] }, + }); + + await expect( + settingsService.setSettings({ + access: { rules: [{ path: 'uploads', permissions: 'invalid', recursive: false }] }, + }) + ).rejects.toThrow(/is not one of the permissions/); + await expect( + settingsService.setSettings({ + access: { rules: [{ path: '../bad', permissions: 'hidden' }] }, + }) + ).rejects.toThrow(/Traversal outside the volume root/); + + const { access } = await settingsService.getSystemSettings(); + expect(access.rules).toEqual([expect.objectContaining({ path: 'Projects' })]); } finally { await envContext.cleanup(); } @@ -79,17 +107,143 @@ describe('Settings Service', () => { ).run('user-1', 'user-1@example.com', 1, 'user-1', 'User 1', '["user"]', now, now); await settingsService.setUserSetting('user-1', 'showSidebarFavorites', false); - await settingsService.setUserSetting('user-1', 'showSidebarShares', 0); - await settingsService.setUserSetting('user-1', 'showSidebarTools', 'yes'); + await settingsService.setUserSetting('user-1', 'showSidebarShares', true); + // Whether a .md file opens in the editor rather than the preview (#347) + // is a per-user choice, and a boolean like the others. + await settingsService.setUserSetting('user-1', 'markdownOpensInEditor', true); + + // Anything that is not a boolean is not an answer, and is not stored: + // `Boolean('yes')` used to store true and `Boolean(0)` false, in place + // of what the person had chosen. + expect( + await settingsService.setUserSetting('user-1', 'showSidebarShares', 0) + ).toBeUndefined(); + expect( + await settingsService.setUserSetting('user-1', 'showSidebarTools', 'yes') + ).toBeUndefined(); const settings = await settingsService.getUserSettings('user-1'); expect(settings.showSidebarFavorites).toBe(false); - expect(settings.showSidebarShares).toBe(false); - expect(settings.showSidebarTools).toBe(true); + expect(settings.showSidebarShares).toBe(true); + // Never stored, so the client's own default is what applies. + expect(settings.showSidebarTools).toBeUndefined(); + expect(settings.markdownOpensInEditor).toBe(true); + } finally { + await envContext.cleanup(); + } + }); + }); + + describe('folder sorts', () => { + it('keeps every folder a user has set a preference on', async () => { + // The cap existed because these lived in one JSON blob, rewritten whole + // on every change: past a hundred folders the oldest was silently + // forgotten. As rows there is nothing to cap, and nothing to forget. + const { envContext, settingsService, dbService } = await createSettingsContext(); + try { + const db = await dbService.getDb(); + const now = new Date().toISOString(); + db.prepare( + ` + INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ` + ).run('user-1', 'user-1@example.com', 1, 'user-1', 'User 1', '["user"]', now, now); + + for (let index = 0; index < 150; index += 1) { + await settingsService.setUserFolderSort('user-1', `Projects/folder-${index}`, { + by: 'customColumn', + order: 'desc', + }); + } + + const settings = await settingsService.getUserSettings('user-1'); + + expect(Object.keys(settings.folderSorts)).toHaveLength(150); + expect(settings.folderSorts['Projects/folder-0']).toMatchObject({ + by: 'customColumn', + order: 'desc', + }); } finally { await envContext.cleanup(); } }); + + it('keeps a folder sort and its view side by side', async () => { + // One row carries both, so setting one must not wipe the other. + const { envContext, settingsService, dbService } = await createSettingsContext(); + try { + const db = await dbService.getDb(); + const now = new Date().toISOString(); + db.prepare( + ` + INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ` + ).run('user-1', 'user-1@example.com', 1, 'user-1', 'User 1', '["user"]', now, now); + + await settingsService.setUserFolderSort('user-1', 'Photos', { by: 'name', order: 'asc' }); + await settingsService.setUserFolderView('user-1', 'Photos', { mode: 'photos' }); + + const settings = await settingsService.getUserSettings('user-1'); + + expect(settings.folderSorts.Photos).toMatchObject({ by: 'name', order: 'asc' }); + expect(settings.folderViews.Photos).toMatchObject({ mode: 'photos' }); + } finally { + await envContext.cleanup(); + } + }); + }); +}); + +/** + * A path written into the compose file has to reach the page that shows it. + * + * It did not: the server sent it and the browser dropped it, because the + * settings store copies system settings field by field and nobody added the + * new one. Both halves are covered now — this end, and the store's own test. + */ +describe('exclusions that come from the environment', () => { + it('reports the search index exclusions the environment set', async () => { + const envContext = await setupTestEnv({ + tag: 'settings-search-index-', + modules: [...SETTINGS_MODULES, 'src/services/searchIndexExclusions'], + env: { SEARCH_INDEX: 'true', SEARCH_INDEX_EXCLUDE: 'Stacks/docker, Sauvegardes/2024' }, + }); + try { + const settingsService = envContext.requireFresh('src/services/settingsService'); + const settings = await settingsService.getSettings(); + + expect(settings.searchIndex.environmentExcludedPaths).toEqual([ + 'Sauvegardes/2024', + 'Stacks/docker', + ]); + // The environment's list is not the administrator's, and neither is + // shown in place of the other. + expect(settings.searchIndex.excludedPaths).toEqual([]); + } finally { + await envContext.cleanup(); + } + }); + + it('keeps the two lists apart when an administrator adds one', async () => { + const envContext = await setupTestEnv({ + tag: 'settings-search-index-', + modules: [...SETTINGS_MODULES, 'src/services/searchIndexExclusions'], + env: { SEARCH_INDEX: 'true', SEARCH_INDEX_EXCLUDE: 'Stacks/docker' }, + }); + try { + const settingsService = envContext.requireFresh('src/services/settingsService'); + await settingsService.setSystemSetting('system', 'searchIndex', { + excludedPaths: ['Photos/RAW'], + }); + + const settings = await settingsService.getSettings(); + expect(settings.searchIndex.excludedPaths).toEqual(['Photos/RAW']); + expect(settings.searchIndex.environmentExcludedPaths).toEqual(['Stacks/docker']); + } finally { + await envContext.cleanup(); + } }); }); diff --git a/backend/tests/services/terminal-session-gate.test.js b/backend/tests/services/terminal-session-gate.test.js index 2fd304be9..e8b0d8d5d 100644 --- a/backend/tests/services/terminal-session-gate.test.js +++ b/backend/tests/services/terminal-session-gate.test.js @@ -21,7 +21,10 @@ import { setupTestEnv } from '../helpers/env-test-utils.js'; let currentEnv; const load = async () => { - currentEnv = await setupTestEnv({ tag: 'terminal-gate-', modules: ['src/services/terminalService'] }); + currentEnv = await setupTestEnv({ + tag: 'terminal-gate-', + modules: ['src/services/terminalService'], + }); return currentEnv.requireFresh('src/services/terminalService'); }; diff --git a/backend/tests/services/text-editor-encoding.test.js b/backend/tests/services/text-editor-encoding.test.js new file mode 100644 index 000000000..1b4b30fa0 --- /dev/null +++ b/backend/tests/services/text-editor-encoding.test.js @@ -0,0 +1,171 @@ +import { describe, it, expect } from 'vitest'; + +const { + detectTextEncoding, + decodeText, + encodeText, +} = require('../../src/services/textEditorService'); + +/** + * The encoding a text file is written in. + * + * A 3.5 MB text file was refused with "this file appears to be binary and + * cannot be opened in the text editor", which is both wrong and impossible to + * act on. The file was UTF-16: every ASCII character carries a zero byte + * alongside it, and a zero byte is exactly what the binary test looks for. + * + * That is not an exotic case. PowerShell's `Out-File` wrote UTF-16LE by default + * until PowerShell 6 and Windows Notepad still offers it as "Unicode", so a log + * or an export from a Windows machine is very often UTF-16. + */ + +const utf16le = (text, { bom = false } = {}) => { + const body = Buffer.from(text, 'utf16le'); + return bom ? Buffer.concat([Buffer.from([0xff, 0xfe]), body]) : body; +}; + +const utf16be = (text, { bom = false } = {}) => { + const body = Buffer.from(text, 'utf16le').swap16(); + return bom ? Buffer.concat([Buffer.from([0xfe, 0xff]), body]) : body; +}; + +const utf8 = (text, { bom = false } = {}) => { + const body = Buffer.from(text, 'utf8'); + return bom ? Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), body]) : body; +}; + +const LOG = 'Nom de la machine : POSTE-042\r\nStatut : à jour\r\n'.repeat(40); + +describe('what a file is written in', () => { + it('is UTF-8 when nothing says otherwise', () => { + expect(detectTextEncoding(utf8(LOG))).toEqual({ encoding: 'utf8', bom: false }); + }); + + it('is what the mark says, when there is one', () => { + expect(detectTextEncoding(utf8(LOG, { bom: true }))).toEqual({ + encoding: 'utf8', + bom: true, + }); + expect(detectTextEncoding(utf16le(LOG, { bom: true }))).toEqual({ + encoding: 'utf16le', + bom: true, + }); + expect(detectTextEncoding(utf16be(LOG, { bom: true }))).toEqual({ + encoding: 'utf16be', + bom: true, + }); + }); + + /** Plenty of tools write UTF-16 without a mark, and it is still UTF-16. */ + it('is UTF-16 when the zero bytes fall where UTF-16 puts them', () => { + expect(detectTextEncoding(utf16le(LOG))).toEqual({ encoding: 'utf16le', bom: false }); + expect(detectTextEncoding(utf16be(LOG))).toEqual({ encoding: 'utf16be', bom: false }); + }); + + /** A zero on both sides of the pairs is not one encoding read wrongly. */ + it('is not UTF-16 for something that merely holds zeros', () => { + const binary = Buffer.from([0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x02, 0x00]); + + expect(detectTextEncoding(binary).encoding).toBe('utf8'); + }); + + /** + * Two bytes are "A" in UTF-16BE and a zero followed by "A" in UTF-8, and + * nothing in them settles which. Too little to go on is answered by the + * ordinary reading, which then refuses it for the zero it holds. + */ + it('is UTF-8 for a file too short to show a pattern', () => { + expect(detectTextEncoding(Buffer.from([0x00, 0x41])).encoding).toBe('utf8'); + }); + + /** Three bytes of a PNG header would reach the ratio on their own. */ + it('is not UTF-16 for a handful of bytes that happen to alternate', () => { + const pngHead = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0x1a, 0x0a]); + + expect(detectTextEncoding(pngHead).encoding).toBe('utf8'); + }); + + /** + * In real UTF-16 the half beside the zero is a character. A zero paired with + * a control byte is a file that merely contains zeros. + */ + it('is not UTF-16 when the other half of each pair is not text', () => { + const alternating = Buffer.alloc(64); + for (let index = 0; index < alternating.length; index += 2) alternating[index + 1] = 0x03; + + expect(detectTextEncoding(alternating).encoding).toBe('utf8'); + }); + + /** A file cut mid-character must not answer with a stack trace. */ + it('reads a big-endian file that stops in the middle of a character', () => { + const truncated = Buffer.concat([ + Buffer.from([0xfe, 0xff]), + Buffer.from('Bonjour', 'utf16le').swap16(), + Buffer.from([0x00]), + ]); + + expect(decodeText(truncated, detectTextEncoding(truncated))).toBe('Bonjour'); + }); + + it('is UTF-8 for nothing at all', () => { + expect(detectTextEncoding(Buffer.alloc(0))).toEqual({ encoding: 'utf8', bom: false }); + }); +}); + +describe('reading it back', () => { + it.each([ + ['UTF-8', utf8], + ['UTF-8 with a mark', (text) => utf8(text, { bom: true })], + ['UTF-16LE', utf16le], + ['UTF-16LE with a mark', (text) => utf16le(text, { bom: true })], + ['UTF-16BE', utf16be], + ['UTF-16BE with a mark', (text) => utf16be(text, { bom: true })], + ])('gives back the words of a file written in %s', (_name, write) => { + const buffer = write(LOG); + + expect(decodeText(buffer, detectTextEncoding(buffer))).toBe(LOG); + }); + + /** The mark is not part of what somebody typed, and must not reach them. */ + it('leaves no mark at the start of the text', () => { + const buffer = utf16le('Bonjour', { bom: true }); + + expect(decodeText(buffer, detectTextEncoding(buffer)).charCodeAt(0)).not.toBe(0xfeff); + }); +}); + +describe('writing it back', () => { + it.each([ + ['UTF-8', utf8], + ['UTF-8 with a mark', (text) => utf8(text, { bom: true })], + ['UTF-16LE', utf16le], + ['UTF-16LE with a mark', (text) => utf16le(text, { bom: true })], + ['UTF-16BE', utf16be], + ['UTF-16BE with a mark', (text) => utf16be(text, { bom: true })], + ])('keeps a file written in %s exactly as it was', (_name, write) => { + const original = write(LOG); + + const rewritten = encodeText( + decodeText(original, detectTextEncoding(original)), + detectTextEncoding(original) + ); + + expect(rewritten).toEqual(original); + }); + + /** + * A UTF-16 log saved back as UTF-8 would halve in size and read perfectly + * well here, while breaking whatever wrote it — a script reading it with a + * fixed encoding, an import expecting the mark it left. + */ + it('does not quietly convert a UTF-16 file to UTF-8', () => { + const rewritten = encodeText('Bonjour', { encoding: 'utf16le', bom: true }); + + expect(rewritten.subarray(0, 2)).toEqual(Buffer.from([0xff, 0xfe])); + expect(rewritten.length).toBe(2 + 'Bonjour'.length * 2); + }); + + it('writes a new file in UTF-8, with nothing in front of it', () => { + expect(encodeText('Bonjour')).toEqual(Buffer.from('Bonjour', 'utf8')); + }); +}); diff --git a/backend/tests/services/thumbnail-cache-cleanup.test.js b/backend/tests/services/thumbnail-cache-cleanup.test.js new file mode 100644 index 000000000..871b62056 --- /dev/null +++ b/backend/tests/services/thumbnail-cache-cleanup.test.js @@ -0,0 +1,462 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { randomUUID } from 'node:crypto'; +import { fileURLToPath } from 'node:url'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { substituteModule } from '../helpers/substitute-module.js'; + +/** + * What the thumbnail cache throws away. + * + * It is the only thing standing between a cache and a full disk, and it deletes + * files — so both halves are worth stating. Keeping too much fills the volume; + * deleting too much sends every thumbnail back through ffmpeg, which is the + * cost the cache exists to avoid. + * + * None of it was covered, because it is reached only through timers. + */ + +let currentEnv; +let releaseHeldWrite = null; +let restoreModule = null; + +const SERVICE_FILE = fileURLToPath( + new URL('../../src/services/thumbnailService.js', import.meta.url) +); +const HOUR = 60 * 60 * 1000; +const CURRENT = 'v3-'; +const OLD = 'v2-'; +const sha1 = (n) => String(n).padStart(40, 'a'); +/** How releases up to 2.0.3 named a thumbnail: the key, and no version. */ +const legacy = (n) => `${sha1(n)}.webp`; +/** A thumbnail's temporary name as it is written now, and as 2.0.x wrote it. */ +const tempOf = (name) => `${name}.tmp-4242-${Date.now()}-${randomUUID()}`; +const legacyTempOf = (name) => `${name}.tmp-4242-${Date.now()}`; + +const setup = async (env = {}) => { + currentEnv = await setupTestEnv({ + tag: 'thumb-cleanup-', + env: { THUMBNAILS: 'true', ...env }, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/services/ffmpegRunner', + 'src/services/thumbnailService', + ], + }); + + const service = currentEnv.requireFresh('src/services/thumbnailService'); + const { directories } = currentEnv.requireFresh('src/config/index'); + await fs.mkdir(directories.thumbnails, { recursive: true }); + return { service, dir: directories.thumbnails }; +}; + +/** A cache entry, optionally aged. */ +const write = async (dir, name, { ageMs = 0 } = {}) => { + const file = path.join(dir, name); + await fs.writeFile(file, 'webp'); + if (ageMs > 0) { + const when = new Date(Date.now() - ageMs); + await fs.utimes(file, when, when); + } + return name; +}; + +const remaining = async (dir) => (await fs.readdir(dir)).sort(); + +afterEach(async () => { + // A write held open would keep the queue from ever going idle. + releaseHeldWrite?.(); + releaseHeldWrite = null; + if (currentEnv) { + const service = currentEnv.loaded?.('src/services/thumbnailService'); + try { + await service?.stopThumbnailWork?.(); + } catch (_) { + // Nothing in flight. + } + await currentEnv.cleanup(); + currentEnv = null; + } + restoreModule?.(); + restoreModule = null; +}); + +/** + * A sharp whose file writes begin, and then wait to be told to finish. + * + * `started` resolves with the temporary path once the partial file is on disk. + */ +const holdThumbnailWrite = () => { + let release; + const released = new Promise((resolve) => { + release = resolve; + }); + let reportStarted; + const started = new Promise((resolve) => { + reportStarted = resolve; + }); + + const pipeline = { + rotate: () => pipeline, + resize: () => pipeline, + webp: () => pipeline, + toFile: async (file) => { + await fs.writeFile(file, 'half a thumbnail'); + reportStarted(file); + await released; + await fs.writeFile(file, 'a thumbnail'); + }, + }; + const sharp = Object.assign(() => pipeline, { + concurrency: () => 1, + cache: () => ({}), + counters: () => ({}), + }); + + return { sharp, started, release }; +}; + +const eventually = async (probe) => { + for (let attempt = 0; attempt < 300; attempt += 1) { + const value = await probe(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error('the condition never held'); +}; + +describe('entries from an older cache version', () => { + it('are removed', async () => { + const { service, dir } = await setup(); + await write(dir, `${OLD}${sha1(1)}.webp`); + await write(dir, `${CURRENT}${sha1(2)}.webp`); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual([`${CURRENT}${sha1(2)}.webp`]); + }); +}); + +describe('thumbnails named before the version prefix existed', () => { + /** + * Releases up to 2.0.3 wrote `.webp`. The cleanup only knew the + * versioned name, so these were neither counted nor ever removed. + */ + it('are removed', async () => { + const { service, dir } = await setup(); + await write(dir, legacy(1)); + await write(dir, legacy(2)); + await write(dir, `${CURRENT}${sha1(3)}.webp`); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual([`${CURRENT}${sha1(3)}.webp`]); + }); +}); + +describe('temporary files a write left behind', () => { + /** The aged thumbnail kept alongside says the age rule is the temporaries' alone. */ + it('are removed once clearly abandoned, under either naming', async () => { + const { service, dir } = await setup(); + const kept = await write(dir, `${CURRENT}${sha1(1)}.webp`, { ageMs: 2 * HOUR }); + await write(dir, tempOf(`${CURRENT}${sha1(1)}.webp`), { ageMs: 2 * HOUR }); + await write(dir, legacyTempOf(legacy(2)), { ageMs: 2 * HOUR }); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual([kept]); + }); + + it('are kept while recent', async () => { + const { service, dir } = await setup(); + const names = [ + await write(dir, tempOf(`${CURRENT}${sha1(1)}.webp`), { ageMs: 10 * 60 * 1000 }), + await write(dir, legacyTempOf(`${CURRENT}${sha1(2)}.webp`)), + ]; + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual(names.sort()); + }); + + it('neither count towards the limit nor are trimmed to meet it', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_MAX_FILES: '2' }); + const names = []; + for (let i = 0; i < 2; i += 1) names.push(await write(dir, `${CURRENT}${sha1(i)}.webp`)); + for (let i = 10; i < 13; i += 1) { + names.push(await write(dir, tempOf(`${CURRENT}${sha1(i)}.webp`))); + } + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual(names.sort()); + }); + + /** + * Removing them must not use up the trim the limit calls for. Taking the + * larger of "removable" and "over the limit" stopped two thumbnails short. + */ + it('are removed on top of the trim a cache past its limit needs', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_MAX_FILES: '2' }); + for (let i = 0; i < 4; i += 1) await write(dir, `${CURRENT}${sha1(i)}.webp`); + for (let i = 10; i < 12; i += 1) { + await write(dir, tempOf(`${CURRENT}${sha1(i)}.webp`), { ageMs: 2 * HOUR }); + } + + await service.cleanupThumbnailCache(); + + const left = await remaining(dir); + expect(left.filter((name) => name.includes('.tmp-'))).toEqual([]); + expect(left).toHaveLength(2); + }); + + it('are left alone when the name is not one of ours', async () => { + const { service, dir } = await setup(); + const names = [ + await write(dir, 'notes.txt.tmp-4242-1700000000000', { ageMs: 2 * HOUR }), + await write(dir, 'v3-nothexadecimal.webp.tmp-4242-1700000000000', { ageMs: 2 * HOUR }), + ]; + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual(names.sort()); + }); + + /** + * The queues stop waiting for a job after thirty seconds and the job goes on, + * so a write that slow is exactly the one whose temporary file looks + * abandoned. Taking it would fail the rename it is still heading for. + */ + it('are kept however old while their write is still going on', async () => { + const held = holdThumbnailWrite(); + releaseHeldWrite = held.release; + restoreModule = substituteModule(SERVICE_FILE, 'sharp', held.sharp); + const { service, dir } = await setup(); + const source = path.join(currentEnv.volumeDir, 'photo.jpg'); + await fs.writeFile(source, 'a photo'); + + await service.queueThumbnailGeneration(source); + const tempFile = await held.started; + const longAgo = new Date(Date.now() - 2 * HOUR); + await fs.utimes(tempFile, longAgo, longAgo); + + await service.cleanupThumbnailCache(); + expect(await remaining(dir)).toContain(path.basename(tempFile)); + + held.release(); + const thumbnail = await eventually(async () => + (await remaining(dir)).find((name) => /^v3-[a-f0-9]{40}\.webp$/.test(name)) + ); + expect(await remaining(dir)).toEqual([thumbnail]); + }); +}); + +describe('entries nobody has looked at for a long time', () => { + it('are removed once past their lifetime', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_TTL_DAYS: '1' }); + await write(dir, `${CURRENT}${sha1(1)}.webp`, { ageMs: 3 * 24 * 60 * 60 * 1000 }); + await write(dir, `${CURRENT}${sha1(2)}.webp`); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual([`${CURRENT}${sha1(2)}.webp`]); + }); + + /** A lifetime of zero is what turns the rule off, not what expires everything. */ + it('are kept when no lifetime is set', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_TTL_DAYS: '0' }); + await write(dir, `${CURRENT}${sha1(1)}.webp`, { ageMs: 365 * 24 * 60 * 60 * 1000 }); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toHaveLength(1); + }); +}); + +describe('a cache that has grown past its limit', () => { + it('is brought back under it', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_MAX_FILES: '2' }); + for (let i = 0; i < 5; i += 1) await write(dir, `${CURRENT}${sha1(i)}.webp`); + + await service.cleanupThumbnailCache(); + + expect((await remaining(dir)).length).toBeLessThanOrEqual(2); + }); + + it('deletes no more than one batch at a time', async () => { + const { service, dir } = await setup({ + THUMBNAIL_CACHE_MAX_FILES: '1', + THUMBNAIL_CACHE_CLEANUP_BATCH_SIZE: '2', + }); + for (let i = 0; i < 6; i += 1) await write(dir, `${CURRENT}${sha1(i)}.webp`); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toHaveLength(4); + }); +}); + +describe('a file in that directory that is not a thumbnail', () => { + /** + * The name pattern says what belongs to this cache. It used to decide which + * entries were expired or outdated and then be dropped for the overflow trim, + * which took every file in the directory — so anything else living there both + * counted towards the limit and could be deleted to satisfy it. + */ + it('is not deleted to make room', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_MAX_FILES: '1' }); + await write(dir, 'please-keep-me.txt'); + for (let i = 0; i < 4; i += 1) await write(dir, `${CURRENT}${sha1(i)}.webp`); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toContain('please-keep-me.txt'); + }); + + it('does not count towards the limit', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_MAX_FILES: '3' }); + await write(dir, 'notes.txt'); + await write(dir, 'other.log'); + for (let i = 0; i < 3; i += 1) await write(dir, `${CURRENT}${sha1(i)}.webp`); + + await service.cleanupThumbnailCache(); + + const left = await remaining(dir); + expect(left.filter((name) => name.endsWith('.webp'))).toHaveLength(3); + }); + + it('is left alone even when it looks nearly right', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_MAX_FILES: '1' }); + await write(dir, 'v3-nothexadecimal.webp'); + for (let i = 0; i < 3; i += 1) await write(dir, `${CURRENT}${sha1(i)}.webp`); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toContain('v3-nothexadecimal.webp'); + }); +}); + +describe('a cache within its limits', () => { + it('is left entirely alone', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_MAX_FILES: '10' }); + const names = []; + for (let i = 0; i < 3; i += 1) names.push(await write(dir, `${CURRENT}${sha1(i)}.webp`)); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual(names.sort()); + }); +}); + +describe('a cache with the limit switched off', () => { + /** Zero lifts the limit on the count, and must not mean "delete everything". */ + it('keeps every current thumbnail, however many', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_MAX_FILES: '0' }); + const names = []; + for (let i = 0; i < 4; i += 1) names.push(await write(dir, `${CURRENT}${sha1(i)}.webp`)); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual(names.sort()); + }); + + /** + * It used to leave the directory unmanaged: another version's thumbnails, + * those past their lifetime and abandoned temporary files stayed for good. + */ + it('still removes what is outdated, expired or abandoned', async () => { + const { service, dir } = await setup({ + THUMBNAIL_CACHE_MAX_FILES: '0', + THUMBNAIL_CACHE_TTL_DAYS: '1', + }); + const kept = await write(dir, `${CURRENT}${sha1(1)}.webp`); + await write(dir, `${OLD}${sha1(2)}.webp`); + await write(dir, legacy(3)); + await write(dir, `${CURRENT}${sha1(4)}.webp`, { ageMs: 3 * 24 * HOUR }); + await write(dir, tempOf(`${CURRENT}${sha1(5)}.webp`), { ageMs: 2 * HOUR }); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual([kept]); + }); +}); + +describe('two cleanups asked for at once', () => { + it('run as one', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_MAX_FILES: '1' }); + for (let i = 0; i < 4; i += 1) await write(dir, `${CURRENT}${sha1(i)}.webp`); + + const [first, second] = await Promise.all([ + service.cleanupThumbnailCache(), + service.cleanupThumbnailCache(), + ]); + + expect(first).toBe(second); + }); +}); + +describe('a cache past its limit, trimmed', () => { + it('gives up its oldest thumbnails first, whatever order the directory lists them in', async () => { + const { service, dir } = await setup({ THUMBNAIL_CACHE_MAX_FILES: '2' }); + // Listed alphabetically, the newest come first. + const newest = await write(dir, `${CURRENT}${sha1(0)}.webp`, { ageMs: 1 * HOUR }); + const newer = await write(dir, `${CURRENT}${sha1(1)}.webp`, { ageMs: 2 * HOUR }); + await write(dir, `${CURRENT}${sha1(2)}.webp`, { ageMs: 3 * HOUR }); + await write(dir, `${CURRENT}${sha1(3)}.webp`, { ageMs: 4 * HOUR }); + + await service.cleanupThumbnailCache(); + + expect(await remaining(dir)).toEqual([newest, newer].sort()); + }); +}); + +describe('the thumbnail cleanup schedule', () => { + const MINUTE = 60 * 1000; + // Captured before any test fakes the timers, so real time can still be waited on. + const realSetTimeout = setTimeout; + const pause = (ms) => new Promise((resolve) => realSetTimeout(resolve, ms)); + const env = { THUMBNAIL_CACHE_MAX_FILES: '1', THUMBNAIL_CACHE_CLEANUP_INTERVAL_MS: '60000' }; + + /** Move the clock on a minute at a time until `probe` holds, and say whether it did. */ + const advanceUntil = async (probe, minutes = 30) => { + for (let i = 0; i < minutes; i += 1) { + if (await probe()) return true; + await vi.advanceTimersByTimeAsync(MINUTE); + await pause(5); + } + return probe(); + }; + + afterEach(() => { + vi.useRealTimers(); + }); + + it('trims the cache by itself, and goes on doing so with no thumbnail generated', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const { dir } = await setup(env); + for (let i = 0; i < 3; i += 1) await write(dir, `${CURRENT}${sha1(i)}.webp`); + + expect(await advanceUntil(async () => (await remaining(dir)).length === 1)).toBe(true); + + // Nothing is generated from here on: only the clock can bring the next pass. + for (let i = 3; i < 6; i += 1) await write(dir, `${OLD}${sha1(i)}.webp`); + expect(await advanceUntil(async () => (await remaining(dir)).length === 1)).toBe(true); + }); + + it('stays stopped once asked, even with a pass under way', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const { service, dir } = await setup(env); + for (let i = 0; i < 3; i += 1) await write(dir, `${CURRENT}${sha1(i)}.webp`); + + const pass = service.cleanupThumbnailCache(); + await service.stopThumbnailWork(); + // The pass under way has finished by the time the stop returns. + expect(await remaining(dir)).toHaveLength(1); + await pass; + + for (let i = 3; i < 6; i += 1) await write(dir, `${OLD}${sha1(i)}.webp`); + expect(await advanceUntil(async () => (await remaining(dir)).length < 4)).toBe(false); + }); +}); diff --git a/backend/tests/services/thumbnail-ffmpeg-ceiling.test.js b/backend/tests/services/thumbnail-ffmpeg-ceiling.test.js new file mode 100644 index 000000000..61d690eff --- /dev/null +++ b/backend/tests/services/thumbnail-ffmpeg-ceiling.test.js @@ -0,0 +1,194 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; +import { fileURLToPath } from 'node:url'; +import PQueue from 'p-queue'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { substituteModule } from '../helpers/substitute-module.js'; + +/** + * An ffmpeg that never exits. + * + * The queues time out and free the slot, and that is all they do: the job goes + * on, on purpose, so that a thumbnail still being written is not started a + * second time. Nothing else ever ended the run. A process wedged on a file it + * cannot decode therefore held that one file's thumbnail in flight until the + * server was restarted — every later request found the file already in flight + * and waited behind a promise that would never settle. + * + * The ceiling is the only thing that ends such a run. Both places that start + * ffmpeg get it: a video, and a HEIC photo. + * + * It is cut to a second here, and the queues to a few milliseconds, so that + * the order of the two can be looked at rather than waited out. + */ + +const SERVICE_FILE = fileURLToPath( + new URL('../../src/services/thumbnailService.js', import.meta.url) +); +const QUEUE_TIMEOUT_MS = 40; +const CEILING_MS = 1000; + +/** The service's queues, with their timeout cut to a few milliseconds. */ +class QuickTimeoutQueue extends PQueue { + constructor(options = {}) { + super(options.timeout ? { ...options, timeout: QUEUE_TIMEOUT_MS } : options); + } +} + +let ctx; +const restores = []; + +afterEach(async () => { + if (ctx) { + const service = ctx.loaded?.('src/services/thumbnailService'); + try { + await service?.stopThumbnailWork?.(); + } catch (_) { + // Nothing in flight. + } + await ctx.cleanup(); + ctx = null; + } + while (restores.length) restores.pop()(); +}); + +/** An ffmpeg that starts, writes nothing, and never exits. */ +const wedgedFfmpeg = () => { + const signals = []; + let runs = 0; + const runner = { + ffmpegPath: '/fake/ffmpeg', + ffprobePath: '/fake/ffprobe', + hasFfmpeg: () => true, + hasFfprobe: () => true, + probe: async () => ({ format: { duration: 8 } }), + run: () => { + runs += 1; + const child = new EventEmitter(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = (signal) => signals.push(signal); + return child; + }, + }; + return { runner, signals, runs: () => runs }; +}; + +/** + * A sharp whose write cannot finish on its own: ffmpeg sends nothing, so the + * only thing that can settle it is the pipeline being torn down. + */ +const neverFinishingSharp = () => { + const pipeline = () => { + const stream = new PassThrough(); + stream.resume(); + return Object.assign(stream, { + rotate: () => stream, + resize: () => stream, + webp: () => stream, + toFile: () => + new Promise((_resolve, reject) => { + stream.on('error', reject); + stream.on('close', () => reject(new Error('the pipeline was torn down'))); + }), + }); + }; + return Object.assign(pipeline, { + concurrency: () => 1, + cache: () => ({}), + counters: () => ({}), + }); +}; + +const setup = async (substitutes) => { + ctx = await setupTestEnv({ + tag: 'thumb-ceiling-', + env: { THUMBNAILS: 'true', THUMBNAIL_FFMPEG_TIMEOUT_MS: String(CEILING_MS) }, + }); + for (const [request, exports] of substitutes) { + restores.push(substituteModule(SERVICE_FILE, request, exports)); + } + return ctx.requireFresh('src/services/thumbnailService'); +}; + +const pastTheQueueTimeout = () => + new Promise((resolve) => setTimeout(resolve, QUEUE_TIMEOUT_MS * 5)); + +const eventually = async (probe) => { + for (let attempt = 0; attempt < 200; attempt += 1) { + const value = await probe(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error('the condition never held'); +}; + +const startedOn = async (service, source) => { + await service.queueThumbnailGeneration(source); + await eventually(async () => service.getDiagnosticsSnapshot().counts.activeExternalProcesses > 0); +}; + +describe('an ffmpeg that never exits', () => { + it('is left alone while the queue gives up on it, then killed at the ceiling', async () => { + const ffmpeg = wedgedFfmpeg(); + const service = await setup([ + ['sharp', neverFinishingSharp()], + ['p-queue', { default: QuickTimeoutQueue }], + ['./ffmpegRunner', ffmpeg.runner], + ]); + const source = path.join(ctx.volumeDir, 'clip.mp4'); + await fs.writeFile(source, 'a video'); + + await startedOn(service, source); + await pastTheQueueTimeout(); + + // The queues have stopped waiting, and the run is untouched: the file is + // still in flight behind it, and no second ffmpeg is started for it. + const { queues, counts } = service.getDiagnosticsSnapshot(); + expect(queues.thumbnail.pending).toBe(0); + expect(queues.video.pending).toBe(0); + expect(counts.inflight).toBe(1); + expect(ffmpeg.signals).toEqual([]); + await expect(service.queueThumbnailGeneration(source)).resolves.toMatchObject({ + pending: true, + queued: true, + }); + expect(ffmpeg.runs()).toBe(1); + + // And then the ceiling ends it. + await eventually(async () => ffmpeg.signals.length > 0); + expect(ffmpeg.signals).toContain('SIGKILL'); + + // The file is no longer held: it is a failure now, remembered for its ten + // minutes and asked again after them, rather than in flight until a restart. + await eventually(async () => service.getDiagnosticsSnapshot().counts.inflight === 0); + expect(service.getDiagnosticsSnapshot().counts.failedCache).toBe(1); + await expect(service.queueThumbnailGeneration(source)).resolves.toMatchObject({ + pending: false, + queued: false, + }); + }); + + it('is killed at the ceiling for a HEIC photo too', async () => { + const ffmpeg = wedgedFfmpeg(); + const service = await setup([ + ['sharp', neverFinishingSharp()], + ['p-queue', { default: QuickTimeoutQueue }], + ['./ffmpegRunner', ffmpeg.runner], + ]); + const source = path.join(ctx.volumeDir, 'photo.heic'); + await fs.writeFile(source, 'a photo'); + + await startedOn(service, source); + await pastTheQueueTimeout(); + expect(ffmpeg.signals).toEqual([]); + + await eventually(async () => ffmpeg.signals.length > 0); + expect(ffmpeg.signals).toContain('SIGKILL'); + await eventually(async () => service.getDiagnosticsSnapshot().counts.inflight === 0); + expect(service.getDiagnosticsSnapshot().counts.failedCache).toBe(1); + }); +}); diff --git a/backend/tests/services/thumbnail-queue-timeout.test.js b/backend/tests/services/thumbnail-queue-timeout.test.js new file mode 100644 index 000000000..22a1ff737 --- /dev/null +++ b/backend/tests/services/thumbnail-queue-timeout.test.js @@ -0,0 +1,207 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; +import { fileURLToPath } from 'node:url'; +import PQueue from 'p-queue'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; +import { substituteModule } from '../helpers/substitute-module.js'; + +/** + * A thumbnail its queue has stopped waiting for. + * + * The queues give up on a job after thirty seconds and free its slot, and the + * job goes on: a large photo on a slow disk, a video ffmpeg takes its time + * over. The file used to be forgotten at that moment, so the next listing that + * asked for its thumbnail started the same work a second time beside the first + * — two sharp pipelines, or two ffmpeg processes, for one thumbnail. + * + * The timeout is shortened here, and the writes are held open, so the moment + * after the queue gives up can be looked at rather than waited for. + */ + +const SERVICE_FILE = fileURLToPath( + new URL('../../src/services/thumbnailService.js', import.meta.url) +); +const QUEUE_TIMEOUT_MS = 40; + +/** The service's queues, with their timeout cut to a few milliseconds. */ +class QuickTimeoutQueue extends PQueue { + constructor(options = {}) { + super(options.timeout ? { ...options, timeout: QUEUE_TIMEOUT_MS } : options); + } +} + +let ctx; +let releaseHeldWrites = null; +const restores = []; + +afterEach(async () => { + // A write held open would keep the queues from ever going idle. + releaseHeldWrites?.(); + releaseHeldWrites = null; + if (ctx) { + const service = ctx.loaded?.('src/services/thumbnailService'); + try { + await service?.stopThumbnailWork?.(); + } catch (_) { + // Nothing in flight. + } + await ctx.cleanup(); + ctx = null; + } + while (restores.length) restores.pop()(); +}); + +/** + * A sharp whose file writes begin, count themselves, and then wait to be told + * to finish. Its pipeline is a stream, so ffmpeg's output can be piped into it. + */ +const holdWrites = () => { + let release; + const released = new Promise((resolve) => { + release = resolve; + }); + let reportStarted; + const started = new Promise((resolve) => { + reportStarted = resolve; + }); + let writes = 0; + + const pipeline = () => { + const stream = new PassThrough(); + stream.resume(); + return Object.assign(stream, { + rotate: () => stream, + resize: () => stream, + webp: () => stream, + toFile: async (file) => { + writes += 1; + await fs.writeFile(file, 'half a thumbnail'); + reportStarted(file); + await released; + await fs.writeFile(file, 'a thumbnail'); + }, + }); + }; + const sharp = Object.assign(pipeline, { + concurrency: () => 1, + cache: () => ({}), + counters: () => ({}), + }); + + releaseHeldWrites = release; + return { sharp, started, release, writes: () => writes }; +}; + +/** An ffmpeg that starts, counts itself, and never finishes on its own. */ +const fakeFfmpeg = () => { + let runs = 0; + const runner = { + ffmpegPath: '/fake/ffmpeg', + ffprobePath: '/fake/ffprobe', + hasFfmpeg: () => true, + hasFfprobe: () => true, + probe: async () => ({ format: { duration: 8 } }), + run: () => { + runs += 1; + const child = new EventEmitter(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = () => {}; + return child; + }, + }; + return { runner, runs: () => runs }; +}; + +const setup = async (substitutes) => { + ctx = await setupTestEnv({ tag: 'thumb-timeout-', env: { THUMBNAILS: 'true' } }); + // After the environment has cleared the application modules, so the service + // required next is the one that receives these. + for (const [request, exports] of substitutes) { + restores.push(substituteModule(SERVICE_FILE, request, exports)); + } + return ctx.requireFresh('src/services/thumbnailService'); +}; + +const pastTheTimeout = () => new Promise((resolve) => setTimeout(resolve, QUEUE_TIMEOUT_MS * 5)); + +const eventually = async (probe) => { + for (let attempt = 0; attempt < 300; attempt += 1) { + const value = await probe(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error('the condition never held'); +}; + +describe('a thumbnail its queue has stopped waiting for', () => { + it('is not started a second time while it is still being written', async () => { + const held = holdWrites(); + const service = await setup([ + ['sharp', held.sharp], + ['p-queue', { default: QuickTimeoutQueue }], + ]); + const source = path.join(ctx.volumeDir, 'photo.jpg'); + await fs.writeFile(source, 'a photo'); + + await service.queueThumbnailGeneration(source); + await held.started; + await pastTheTimeout(); + // The queue has given up on it: its slot is free, and the write goes on. + expect(service.getDiagnosticsSnapshot().queues.thumbnail.pending).toBe(0); + + await expect(service.queueThumbnailGeneration(source)).resolves.toMatchObject({ + pending: true, + queued: true, + }); + await pastTheTimeout(); + expect(held.writes()).toBe(1); + + held.release(); + const thumbnail = await eventually(async () => { + const answer = await service.queueThumbnailGeneration(source); + return answer.thumbnail; + }); + expect(thumbnail).toMatch(/^\/static\/thumbnails\/v3-[a-f0-9]+\.webp$/); + expect(held.writes()).toBe(1); + }); + + it('is not given a second ffmpeg while the first is still running, for a video', async () => { + const held = holdWrites(); + const ffmpeg = fakeFfmpeg(); + const service = await setup([ + ['sharp', held.sharp], + ['p-queue', { default: QuickTimeoutQueue }], + ['./ffmpegRunner', ffmpeg.runner], + ]); + const source = path.join(ctx.volumeDir, 'clip.mp4'); + await fs.writeFile(source, 'a video'); + + await service.queueThumbnailGeneration(source); + await held.started; + await pastTheTimeout(); + // Both queues have given up on it: the thumbnail's and the video's. + const { queues } = service.getDiagnosticsSnapshot(); + expect(queues.thumbnail.pending).toBe(0); + expect(queues.video.pending).toBe(0); + + await expect(service.queueThumbnailGeneration(source)).resolves.toMatchObject({ + pending: true, + queued: true, + }); + await pastTheTimeout(); + expect(ffmpeg.runs()).toBe(1); + + held.release(); + const thumbnail = await eventually(async () => { + const answer = await service.queueThumbnailGeneration(source); + return answer.thumbnail; + }); + expect(thumbnail).toMatch(/^\/static\/thumbnails\/v3-[a-f0-9]+\.webp$/); + expect(ffmpeg.runs()).toBe(1); + expect(held.writes()).toBe(1); + }); +}); diff --git a/backend/tests/services/thumbnail-service.test.js b/backend/tests/services/thumbnail-service.test.js new file mode 100644 index 000000000..2b9a81eac --- /dev/null +++ b/backend/tests/services/thumbnail-service.test.js @@ -0,0 +1,47 @@ +import path from 'path'; +import { createRequire } from 'module'; +import { describe, expect, it } from 'vitest'; + +const require = createRequire(import.meta.url); +const { directories } = require('../../src/config'); +const { isThumbnailCachePath } = require('../../src/services/thumbnailService'); + +describe('Thumbnail Service', () => { + describe('isThumbnailCachePath', () => { + it('detects files inside the thumbnail cache directory', () => { + const thumbnailPath = path.join( + directories.thumbnails, + 'v2-0123456789abcdef0123456789abcdef01234567.webp' + ); + + expect(isThumbnailCachePath(thumbnailPath)).toBe(true); + }); + + it('detects generated thumbnail artifacts even when the cache is exposed through another path', () => { + const aliasedPath = path.join( + directories.volume, + 'cache', + 'thumbnails', + 'v2-0123456789abcdef0123456789abcdef01234567.webp' + ); + + expect(isThumbnailCachePath(aliasedPath)).toBe(true); + }); + + it('detects current v3 generated thumbnail artifacts', () => { + const thumbnailPath = path.join( + directories.volume, + 'media', + 'v3-0123456789abcdef0123456789abcdef01234567.webp' + ); + + expect(isThumbnailCachePath(thumbnailPath)).toBe(true); + }); + + it('does not block regular webp files', () => { + const regularImage = path.join(directories.volume, 'photos', 'cover.webp'); + + expect(isThumbnailCachePath(regularImage)).toBe(false); + }); + }); +}); diff --git a/backend/tests/services/transfer-engines.test.js b/backend/tests/services/transfer-engines.test.js new file mode 100644 index 000000000..d4d5a0343 --- /dev/null +++ b/backend/tests/services/transfer-engines.test.js @@ -0,0 +1,332 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The same promises, kept by both engines. + * + * Copying and deleting have two implementations — `rsync` and `rm` on one side, + * streams and `fs.rm` on the other — and which one runs used to be decided by + * the platform, at the moment the module loaded. Each was then only ever + * exercised where it was chosen: the native path could not run on a developer's + * macOS machine, and the JavaScript path could not run on the Linux that CI + * runs. Nobody ran both, and nothing named the setting: `FILE_TRANSFER_ENGINE` + * appeared once in the whole repository, in the line that read it. + * + * The engine is asked for now rather than assumed, so these run the same facts + * through both. Where they disagree, one of them is wrong — and the point of + * writing them side by side is that the disagreement is visible rather than + * dependent on who ran the suite. + * + * `rm -rf` is what the native delete is, spawned detached. Thirty lines of it + * had never been executed by a test on any machine that measured coverage. + */ + +let currentEnv; + +const ENGINES = ['native', 'stream']; + +const setup = async (engine) => { + currentEnv = await setupTestEnv({ + tag: `transfer-${engine}-`, + env: { FILE_TRANSFER_ENGINE: engine }, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/services/fileTransferService', + 'src/services/accessManager', + 'src/utils/pathUtils', + ], + }); + + const service = currentEnv.requireFresh('src/services/fileTransferService'); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('admin', 'admin@example.com', 1, 'admin', 'Admin', '["admin"]', ?, ?)` + ).run(now, now); + + // Inside a volume, not in the list of volumes: a folder at the top is a + // volume, and the application refuses to rename, move or delete one + // (nxzai/NextExplorer#409). + const volume = path.join(currentEnv.volumeDir, 'Nvm'); + await fs.mkdir(volume, { recursive: true }); + + return { service, volume, user: { id: 'admin', roles: ['admin'] } }; +}; + +/** A folder with something in it, and something in a folder inside it. */ +const seedTree = async (volume, name) => { + const root = path.join(volume, name); + await fs.mkdir(path.join(root, 'nested'), { recursive: true }); + await fs.writeFile(path.join(root, 'top.txt'), 'top'); + await fs.writeFile(path.join(root, 'nested', 'deep.txt'), 'deep'); + return root; +}; + +const exists = async (target) => + fs + .access(target) + .then(() => true) + .catch(() => false); + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +describe.each(ENGINES)('the %s engine', (engine) => { + it('is the one that was asked for', async () => { + const { service } = await setup(engine); + + expect(service.nativeTransferEnabled()).toBe(engine === 'native'); + }); + + it('reports which one it is running', async () => { + const { service } = await setup(engine); + + expect(service.getDiagnosticsSnapshot().nativeTransferEnabled).toBe(engine === 'native'); + }); + + /** + * These are about the engines that remove things from disk, so they ask for + * a permanent deletion: without it the entry would go to the trash, and no + * engine would run at all. + */ + it('deletes a folder and everything under it', async () => { + const { service, volume, user } = await setup(engine); + const root = await seedTree(volume, 'Doomed'); + + await service.deleteItems([{ path: 'Nvm', name: 'Doomed' }], { user, permanent: true }); + + expect(await exists(root)).toBe(false); + }); + + it('deletes a single file', async () => { + const { service, volume, user } = await setup(engine); + await fs.writeFile(path.join(volume, 'note.txt'), 'gone soon'); + + await service.deleteItems([{ path: 'Nvm', name: 'note.txt' }], { user, permanent: true }); + + expect(await exists(path.join(volume, 'note.txt'))).toBe(false); + }); + + it('leaves what it was not asked to delete', async () => { + const { service, volume, user } = await setup(engine); + await seedTree(volume, 'Doomed'); + await seedTree(volume, 'Spared'); + + await service.deleteItems([{ path: 'Nvm', name: 'Doomed' }], { user, permanent: true }); + + expect(await exists(path.join(volume, 'Spared', 'nested', 'deep.txt'))).toBe(true); + }); + + it('says what it deleted', async () => { + const { service, volume, user } = await setup(engine); + await seedTree(volume, 'Doomed'); + + const results = await service.deleteItems([{ path: 'Nvm', name: 'Doomed' }], { + user, + permanent: true, + }); + + expect(results).toEqual([expect.objectContaining({ status: 'deleted' })]); + }); + + /** + * A name beginning with a dash is an option to a command line and a filename + * to everybody else. The native path passes `--` before the path for exactly + * this; the JavaScript one never had the problem. + */ + it('deletes a folder whose name looks like an option', async () => { + const { service, volume, user } = await setup(engine); + const root = await seedTree(volume, '-rf-trap'); + + await service.deleteItems([{ path: 'Nvm', name: '-rf-trap' }], { user, permanent: true }); + + expect(await exists(root)).toBe(false); + }); + + it('copies a folder with everything under it', async () => { + const { service, volume, user } = await setup(engine); + await seedTree(volume, 'Source'); + await fs.mkdir(path.join(volume, 'Target'), { recursive: true }); + + const prep = await service.prepareTransfer( + [{ path: 'Nvm', name: 'Source' }], + 'Nvm/Target', + 'copy', + { + user, + } + ); + await service.executeTransfer(prep, 'copy', undefined, { user }); + + expect(await exists(path.join(volume, 'Target', 'Source', 'nested', 'deep.txt'))).toBe(true); + }); + + it('leaves the original where it was when copying', async () => { + const { service, volume, user } = await setup(engine); + await seedTree(volume, 'Source'); + await fs.mkdir(path.join(volume, 'Target'), { recursive: true }); + + const prep = await service.prepareTransfer( + [{ path: 'Nvm', name: 'Source' }], + 'Nvm/Target', + 'copy', + { + user, + } + ); + await service.executeTransfer(prep, 'copy', undefined, { user }); + + expect(await exists(path.join(volume, 'Source', 'top.txt'))).toBe(true); + }); + + it('copies the contents of a file, not just its name', async () => { + const { service, volume, user } = await setup(engine); + await fs.writeFile(path.join(volume, 'note.txt'), 'the actual bytes'); + await fs.mkdir(path.join(volume, 'Target'), { recursive: true }); + + const prep = await service.prepareTransfer( + [{ path: 'Nvm', name: 'note.txt' }], + 'Nvm/Target', + 'copy', + { + user, + } + ); + await service.executeTransfer(prep, 'copy', undefined, { user }); + + expect(await fs.readFile(path.join(volume, 'Target', 'note.txt'), 'utf8')).toBe( + 'the actual bytes' + ); + }); + + it('moves a folder rather than leaving it behind', async () => { + const { service, volume, user } = await setup(engine); + await seedTree(volume, 'Source'); + await fs.mkdir(path.join(volume, 'Target'), { recursive: true }); + + const prep = await service.prepareTransfer( + [{ path: 'Nvm', name: 'Source' }], + 'Nvm/Target', + 'move', + { + user, + } + ); + await service.executeTransfer(prep, 'move', undefined, { user }); + + expect(await exists(path.join(volume, 'Target', 'Source', 'top.txt'))).toBe(true); + expect(await exists(path.join(volume, 'Source'))).toBe(false); + }); + + /** Nothing to do is the caller's mistake, whichever engine would have done it. */ + it('refuses a transfer with no items', async () => { + const { service, user } = await setup(engine); + + await expect(service.prepareTransfer([], 'Nvm/Target', 'copy', { user })).rejects.toThrow( + /at least one item/i + ); + }); + + it('refuses a delete that was cancelled before it began', async () => { + const { service, volume, user } = await setup(engine); + await seedTree(volume, 'Doomed'); + const controller = new AbortController(); + controller.abort(); + + await expect( + service.deleteItems([{ path: 'Nvm', name: 'Doomed' }], { user, signal: controller.signal }) + ).rejects.toThrow(); + }); + + it('leaves the folder alone when the delete was cancelled before it began', async () => { + const { service, volume, user } = await setup(engine); + const root = await seedTree(volume, 'Doomed'); + const controller = new AbortController(); + controller.abort(); + + await service + .deleteItems([{ path: 'Nvm', name: 'Doomed' }], { user, signal: controller.signal }) + .catch(() => {}); + + expect(await exists(root)).toBe(true); + }); +}); + +/** + * When the native tool cannot be used at all. + * + * `--info=progress2` arrived in rsync 3.1. RHEL 7 ships 3.0.9 and macOS ships + * 2.6.9, and on either of those every copy failed with a raw usage error while + * a working implementation sat unused in the same file. The setting documented + * for exactly that case only helped somebody who already knew to reach for it, + * after their copies had failed. + * + * The distinction these pin is the one that makes falling back safe: a tool + * that could not start has written nothing, so the other implementation can + * begin cleanly. A tool that failed partway has written something, and + * resuming over it is not a recovery. + */ +describe('deciding whether a native tool is unusable', () => { + let isUnusable; + + const load = async () => { + if (!isUnusable) { + const { service } = await setup('stream'); + isUnusable = service.nativeToolIsUnusable; + } + return isUnusable; + }; + + it('says so when the binary is not installed', async () => { + const decide = await load(); + + expect(decide({ code: 'ENOENT', message: 'spawn rsync ENOENT' })).toBe(true); + }); + + it('says so when the tool is too old to understand the request', async () => { + const decide = await load(); + + expect(decide({ stderr: "rsync: unrecognized option `--info=progress2'" })).toBe(true); + }); + + it('accepts the other wordings the same refusal comes in', async () => { + const decide = await load(); + + expect(decide({ stderr: 'unknown option -- info' })).toBe(true); + expect(decide({ stderr: 'illegal option -- x' })).toBe(true); + }); + + /** + * The permission failure that a restricted ZFS dataset produces is a real + * attempt that got partway, and it already has its own retry. Reading it as + * "unusable" would throw away that handling and copy the tree twice. + */ + it('does not say so for a destination that refused a chmod', async () => { + const decide = await load(); + + expect(decide({ exitCode: 23, stderr: 'rsync: failed to set permissions on ...' })).toBe(false); + }); + + it('does not say so for a failure partway through', async () => { + const decide = await load(); + + expect(decide({ exitCode: 11, stderr: 'rsync: write failed: No space left on device' })).toBe( + false + ); + }); + + it('does not say so for an error carrying nothing to go on', async () => { + const decide = await load(); + + expect(decide({})).toBe(false); + expect(decide(null)).toBe(false); + }); +}); diff --git a/backend/tests/services/trash-cycle.test.js b/backend/tests/services/trash-cycle.test.js index 569289093..15cb06b30 100644 --- a/backend/tests/services/trash-cycle.test.js +++ b/backend/tests/services/trash-cycle.test.js @@ -172,7 +172,6 @@ const runSequence = async (envContext, seed) => { // A process that died holds nothing in flight. operations.inflight.clear(); for (const zone of store.listZones(db)) { - // eslint-disable-next-line no-await-in-loop await operations.recoverZone(zone); } }; @@ -395,7 +394,6 @@ const runSequence = async (envContext, seed) => { const context = `seed ${seed} after: ${log.slice(-8).join(' | ')}`; for (const volume of VOLUMES) { - // eslint-disable-next-line no-await-in-loop const onDisk = (await fs.readdir(abs(volume))).filter((name) => name !== '.nextexplorer'); const expected = [...live.keys()] .filter((relative) => volumeOf(relative) === volume) @@ -403,7 +401,6 @@ const runSequence = async (envContext, seed) => { expect(onDisk.sort(), context).toEqual(expected.sort()); } for (const [relative, entry] of live) { - // eslint-disable-next-line no-await-in-loop expect(await readEntry(abs(relative), entry.kind), context).toBe(entry.content); } @@ -412,7 +409,6 @@ const runSequence = async (envContext, seed) => { [...elsewhere.keys()].sort() ); for (const [name, entry] of elsewhere) { - // eslint-disable-next-line no-await-in-loop expect(await readEntry(abs(`${ELSEWHERE}/${name}`), entry.kind), context).toBe(entry.content); } @@ -424,23 +420,22 @@ const runSequence = async (envContext, seed) => { expect(row.state, context).toBe('trashed'); expect(row.originalPath, context).toBe(abs(item.original)); expect(row.size, context).toBe(item.size); - // eslint-disable-next-line no-await-in-loop + expect(await readEntry(zones.itemPaths(zone.root, row.id).payload, item.kind), context).toBe( item.content ); } for (const zone of store.listZones(db)) { - // eslint-disable-next-line no-await-in-loop expect((await verify.verifyZone(zone)).violations, context).toEqual([]); } }; for (let step = 0; step < STEPS; step += 1) { const name = random.pick(weighted); - // eslint-disable-next-line no-await-in-loop + const done = await actions[name](); if (done) log.push(done); - // eslint-disable-next-line no-await-in-loop + await compare(); } diff --git a/backend/tests/services/trash-restore-entry-column.test.js b/backend/tests/services/trash-restore-entry-column.test.js new file mode 100644 index 000000000..8e4783a21 --- /dev/null +++ b/backend/tests/services/trash-restore-entry-column.test.js @@ -0,0 +1,48 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The column an installation made before it existed. + * + * `trash_items` was created with the trash; `restore_entry` joined its + * definition a few commits later, when a restore could first be sent into a + * folder of somebody's choosing. `CREATE TABLE IF NOT EXISTS` adds nothing to a + * table that is already there, so every installation made in between has a + * `trash_items` without it — and a restore into a chosen folder writes it, so + * the first one fails on a column that is not there. + */ + +let env; + +beforeEach(async () => { + env = await setupTestEnv({ tag: 'trash-restore-entry-', modules: ['src/services/db'] }); +}); + +afterEach(async () => { + await env.cleanup(); +}); + +const columns = (db) => + db + .prepare('PRAGMA table_info(trash_items)') + .all() + .map((column) => column.name); + +describe('a trash table made before restore_entry existed', () => { + it('gains the column when the database is opened', async () => { + const dbService = env.requireFresh('src/services/db'); + const db = await dbService.getDb(); + expect(columns(db)).toContain('restore_entry'); + + // Put it back the way that installation's table looks: SQLite can drop a + // column, so the table is rebuilt exactly as it was before it joined. + db.exec('ALTER TABLE trash_items DROP COLUMN restore_entry'); + expect(columns(db)).not.toContain('restore_entry'); + await dbService.closeDb(); + + const reopened = await env.requireFresh('src/services/db').getDb(); + + expect(columns(reopened)).toContain('restore_entry'); + }); +}); diff --git a/backend/tests/services/trash-settings.test.js b/backend/tests/services/trash-settings.test.js index 0e08adbe9..20210a38a 100644 --- a/backend/tests/services/trash-settings.test.js +++ b/backend/tests/services/trash-settings.test.js @@ -126,16 +126,14 @@ describe('changing them', () => { expect(response.body.trash).toMatchObject({ retentionDays: 60, maxPercent: 20 }); }); - it('is not changed by anyone but an administrator', async () => { - // The settings route ignores every system section a non-admin sends, trash - // included: the request is accepted but nothing system-wide is written. + it('is refused to everyone else, with nothing written', async () => { const app = await buildApp({ id: 'user', roles: ['user'] }); const response = await request(app) .patch('/api/settings') .send({ trash: { enabled: false } }); - expect(response.body.trash).toBeUndefined(); + expect(response.status).toBe(403); const settingsService = envContext.requireFresh('src/services/settingsService'); expect((await settingsService.getSystemSettings()).trash.enabled).toBe(true); }); diff --git a/backend/tests/services/two-factor.test.js b/backend/tests/services/two-factor.test.js new file mode 100644 index 000000000..72dd18251 --- /dev/null +++ b/backend/tests/services/two-factor.test.js @@ -0,0 +1,268 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A second factor on a local account. + * + * What has to hold is mostly about what happens when things go wrong: a setup + * abandoned halfway leaves nobody locked out, a code is worth one login and + * not two, a recovery code is worth one use, and losing the key the secret was + * written under costs an authenticator rather than an account. + */ + +let currentEnv; + +afterEach(async () => { + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const seed = async () => { + currentEnv = await setupTestEnv({ tag: 'two-factor-' }); + const db = await currentEnv.requireFresh('src/services/db').getDb(); + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES ('u1','someone@example.com',1,'someone','Someone','["user"]', ?, ?)` + ).run(now, now); + + const twoFactor = currentEnv.requireFresh('src/services/users/twoFactor'); + const { totpCode } = currentEnv.requireFresh('src/utils/totp'); + return { db, twoFactor, totpCode }; +}; + +/** Signed up, confirmed, and holding its recovery codes. */ +const turnOn = async ({ twoFactor, totpCode }) => { + const { secret } = await twoFactor.beginEnrolment({ + userId: 'u1', + account: 'someone@example.com', + }); + const confirmed = await twoFactor.confirmEnrolment({ userId: 'u1', code: totpCode(secret) }); + expect(confirmed).not.toBeNull(); + return { secret, recoveryCodes: confirmed.recoveryCodes }; +}; + +describe('setting it up', () => { + it('hands out a secret and an address the phone can read, and turns nothing on yet', async () => { + const { twoFactor } = await seed(); + + const { secret, uri } = await twoFactor.beginEnrolment({ + userId: 'u1', + account: 'someone@example.com', + }); + + expect(secret).toMatch(/^[A-Z2-7]{32}$/); + expect(uri).toContain(`secret=${secret}`); + expect(uri).toContain('someone%40example.com'); + expect(await twoFactor.twoFactorRequired('u1')).toBe(false); + expect(await twoFactor.twoFactorStatus('u1')).toMatchObject({ enabled: false, pending: true }); + }); + + /** A page left open yesterday is not a second authenticator. */ + it('replaces an unconfirmed secret rather than keeping both', async () => { + const { twoFactor, totpCode } = await seed(); + const first = await twoFactor.beginEnrolment({ userId: 'u1', account: 'someone' }); + const second = await twoFactor.beginEnrolment({ userId: 'u1', account: 'someone' }); + + expect(second.secret).not.toBe(first.secret); + expect( + await twoFactor.confirmEnrolment({ userId: 'u1', code: totpCode(first.secret) }) + ).toBeNull(); + expect( + await twoFactor.confirmEnrolment({ userId: 'u1', code: totpCode(second.secret) }) + ).not.toBeNull(); + }); + + it('refuses a wrong code, and leaves it off', async () => { + const { twoFactor } = await seed(); + await twoFactor.beginEnrolment({ userId: 'u1', account: 'someone' }); + + expect(await twoFactor.confirmEnrolment({ userId: 'u1', code: '000000' })).toBeNull(); + expect(await twoFactor.twoFactorRequired('u1')).toBe(false); + }); + + it('turns it on with ten recovery codes, handed over once', async () => { + const { twoFactor, totpCode } = await seed(); + const { recoveryCodes } = await turnOn({ twoFactor, totpCode }); + + expect(recoveryCodes).toHaveLength(10); + expect(new Set(recoveryCodes).size).toBe(10); + for (const code of recoveryCodes) expect(code).toMatch(/^[A-HJ-NP-Z2-9]{5}-[A-HJ-NP-Z2-9]{5}$/); + expect(await twoFactor.twoFactorStatus('u1')).toMatchObject({ + enabled: true, + recoveryCodesLeft: 10, + }); + }); + + it('refuses to start again while it is on', async () => { + const { twoFactor, totpCode } = await seed(); + await turnOn({ twoFactor, totpCode }); + + await expect(twoFactor.beginEnrolment({ userId: 'u1', account: 'someone' })).rejects.toThrow(); + }); +}); + +describe('signing in with it', () => { + it('takes the code the phone shows', async () => { + const { twoFactor, totpCode } = await seed(); + const { secret } = await turnOn({ twoFactor, totpCode }); + + // A step later than the one the confirmation spent. + const at = Date.now() + 30_000; + const code = totpCode(secret, { at }); + expect(await twoFactor.verifySecondFactor({ userId: 'u1', code })).toMatchObject({ ok: true }); + }); + + /** + * Six digits are good for thirty seconds and for one sign-in. Read over a + * shoulder, or found in a proxy's log, they are already spent. + */ + it('refuses the same code twice', async () => { + const { twoFactor, totpCode } = await seed(); + const { secret } = await turnOn({ twoFactor, totpCode }); + const at = Date.now() + 30_000; + const code = totpCode(secret, { at }); + + expect(await twoFactor.verifySecondFactor({ userId: 'u1', code })).toMatchObject({ ok: true }); + expect(await twoFactor.verifySecondFactor({ userId: 'u1', code })).toMatchObject({ ok: false }); + }); + + it('refuses a wrong code', async () => { + const { twoFactor, totpCode } = await seed(); + await turnOn({ twoFactor, totpCode }); + + expect(await twoFactor.verifySecondFactor({ userId: 'u1', code: '000000' })).toMatchObject({ + ok: false, + }); + }); + + it('takes a recovery code, once, and says how many are left', async () => { + const { twoFactor, totpCode } = await seed(); + const { recoveryCodes } = await turnOn({ twoFactor, totpCode }); + const [paper] = recoveryCodes; + + expect(await twoFactor.verifySecondFactor({ userId: 'u1', code: paper })).toMatchObject({ + ok: true, + usedRecoveryCode: true, + recoveryCodesLeft: 9, + }); + expect(await twoFactor.verifySecondFactor({ userId: 'u1', code: paper })).toMatchObject({ + ok: false, + }); + }); + + /** Read off a printout, typed back in whatever case and spacing. */ + it('reads a recovery code however it was typed', async () => { + const { twoFactor, totpCode } = await seed(); + const { recoveryCodes } = await turnOn({ twoFactor, totpCode }); + const typed = recoveryCodes[0].toLowerCase().replace('-', ' '); + + expect(await twoFactor.verifySecondFactor({ userId: 'u1', code: typed })).toMatchObject({ + ok: true, + }); + }); + + /** + * A setup somebody walked away from is not a second factor. Nothing asks for + * one yet either, so this is the belt to that pair of braces: the code that + * checks a factor refuses a secret no phone ever confirmed. + */ + it('refuses a code for a secret that was never confirmed', async () => { + const { twoFactor, totpCode } = await seed(); + const { secret } = await twoFactor.beginEnrolment({ userId: 'u1', account: 'someone' }); + + expect( + await twoFactor.verifySecondFactor({ userId: 'u1', code: totpCode(secret) }) + ).toMatchObject({ ok: false }); + }); + + it('asks nothing of an account that never set it up', async () => { + const { twoFactor } = await seed(); + + expect(await twoFactor.twoFactorRequired('u1')).toBe(false); + expect(await twoFactor.verifySecondFactor({ userId: 'u1', code: '000000' })).toMatchObject({ + ok: false, + }); + }); + + /** + * The key beside the database is gone or was replaced. The secret cannot be + * read, so the phone is no help — and this is exactly the case recovery codes + * exist for, so they still let somebody in to set it up again. + */ + it('still takes a recovery code when the secret can no longer be read', async () => { + const { db, twoFactor, totpCode } = await seed(); + const { secret, recoveryCodes } = await turnOn({ twoFactor, totpCode }); + db.prepare('UPDATE totp_credentials SET secret = ? WHERE user_id = ?').run( + 'v1:AAAA:BBBB:CCCC', + 'u1' + ); + + const at = Date.now() + 30_000; + expect( + await twoFactor.verifySecondFactor({ userId: 'u1', code: totpCode(secret, { at }) }) + ).toMatchObject({ ok: false }); + expect( + await twoFactor.verifySecondFactor({ userId: 'u1', code: recoveryCodes[1] }) + ).toMatchObject({ ok: true }); + }); +}); + +describe('afterwards', () => { + it('draws new recovery codes, and the old ones stop working', async () => { + const { twoFactor, totpCode } = await seed(); + const { recoveryCodes } = await turnOn({ twoFactor, totpCode }); + + const fresh = await twoFactor.replaceRecoveryCodes('u1'); + + expect(fresh).toHaveLength(10); + expect( + await twoFactor.verifySecondFactor({ userId: 'u1', code: recoveryCodes[0] }) + ).toMatchObject({ ok: false }); + expect(await twoFactor.verifySecondFactor({ userId: 'u1', code: fresh[0] })).toMatchObject({ + ok: true, + }); + }); + + it('will not draw codes for an account that has it off', async () => { + const { twoFactor } = await seed(); + await expect(twoFactor.replaceRecoveryCodes('u1')).rejects.toThrow(); + }); + + it('takes it off, secret and codes both', async () => { + const { db, twoFactor, totpCode } = await seed(); + await turnOn({ twoFactor, totpCode }); + + expect(await twoFactor.disableTwoFactor('u1')).toBe(true); + + expect(await twoFactor.twoFactorRequired('u1')).toBe(false); + expect(db.prepare('SELECT COUNT(*) AS n FROM totp_credentials').get().n).toBe(0); + expect(db.prepare('SELECT COUNT(*) AS n FROM totp_recovery_codes').get().n).toBe(0); + }); +}); + +describe('what the database holds', () => { + it('keeps the secret unreadable in the row', async () => { + const { db, twoFactor, totpCode } = await seed(); + const { secret } = await turnOn({ twoFactor, totpCode }); + + const stored = db.prepare('SELECT secret FROM totp_credentials WHERE user_id = ?').get('u1'); + + expect(stored.secret).not.toContain(secret); + expect(stored.secret.startsWith('v1:')).toBe(true); + }); + + it('keeps recovery codes hashed', async () => { + const { db, twoFactor, totpCode } = await seed(); + const { recoveryCodes } = await turnOn({ twoFactor, totpCode }); + + const rows = db.prepare('SELECT code_hash FROM totp_recovery_codes').all(); + + for (const row of rows) { + expect(row.code_hash).toMatch(/^[0-9a-f]{64}$/); + expect(recoveryCodes).not.toContain(row.code_hash); + } + }); +}); diff --git a/backend/tests/services/upload-remnants.test.js b/backend/tests/services/upload-remnants.test.js new file mode 100644 index 000000000..abbefde80 --- /dev/null +++ b/backend/tests/services/upload-remnants.test.js @@ -0,0 +1,110 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A killed process leaves `holiday.mp4.uploading` behind, and nothing before + * this removed it. The sweep deletes a file it did not create, on the strength + * of a name — so what it leaves alone matters as much as what it takes. + */ + +let envContext; +let sweepStaleUploadRemnants; + +const HOUR = 60 * 60 * 1000; +const DAY = 24 * HOUR; + +const build = async () => { + envContext = await setupTestEnv({ tag: 'upload-remnants-test-' }); + ({ sweepStaleUploadRemnants } = envContext.requireFresh('src/services/uploadRemnants')); + return envContext.volumeDir; +}; + +/** A file last written `ageMs` ago. */ +const writeAged = async (dir, name, ageMs) => { + const target = path.join(dir, name); + await fs.writeFile(target, 'partial'); + const when = new Date(Date.now() - ageMs); + await fs.utimes(target, when, when); + return target; +}; + +const exists = async (target) => + fs + .access(target) + .then(() => true) + .catch(() => false); + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('sweeping the remains of interrupted uploads', () => { + it('removes one nothing has written to in a day', async () => { + const dir = await build(); + const remnant = await writeAged(dir, 'holiday.mp4.uploading', 2 * DAY); + + expect(await sweepStaleUploadRemnants(dir)).toBe(1); + expect(await exists(remnant)).toBe(false); + }); + + // An upload in flight writes to its temporary file continuously. Sweeping it + // would break the very thing the sweep runs alongside. + it('leaves an upload that is still running', async () => { + const dir = await build(); + const inFlight = await writeAged(dir, 'movie.mkv.uploading', 5 * 1000); + + expect(await sweepStaleUploadRemnants(dir)).toBe(0); + expect(await exists(inFlight)).toBe(true); + }); + + it('leaves ordinary files alone', async () => { + const dir = await build(); + const kept = [ + await writeAged(dir, 'notes.txt', 2 * DAY), + await writeAged(dir, 'uploading.txt', 2 * DAY), + await writeAged(dir, 'holiday.mp4', 2 * DAY), + await writeAged(dir, 'report.uploading.pdf', 2 * DAY), + ]; + + expect(await sweepStaleUploadRemnants(dir)).toBe(0); + for (const target of kept) { + expect(await exists(target)).toBe(true); + } + }); + + // Someone's own file, old and named unfortunately, is not ours to delete on + // sight — but it is not ours to keep for ever either. The threshold is the + // whole of the protection, so it has to be the threshold that decides. + it('takes a day to decide', async () => { + const dir = await build(); + const young = await writeAged(dir, 'yesterday.uploading', 23 * HOUR); + const old = await writeAged(dir, 'the-day-before.uploading', 25 * HOUR); + + expect(await sweepStaleUploadRemnants(dir)).toBe(1); + expect(await exists(young)).toBe(true); + expect(await exists(old)).toBe(false); + }); + + it('stays in the folder it was given', async () => { + const dir = await build(); + const nested = path.join(dir, 'holiday'); + await fs.mkdir(nested, { recursive: true }); + const deeper = await writeAged(nested, 'inner.mp4.uploading', 2 * DAY); + const directory = path.join(dir, 'a-folder.uploading'); + await fs.mkdir(directory, { recursive: true }); + + expect(await sweepStaleUploadRemnants(dir)).toBe(0); + expect(await exists(deeper)).toBe(true); + expect(await exists(directory)).toBe(true); + }); + + // The tidying before an upload must never be the reason it fails. + it('says nothing when the folder is not there', async () => { + const dir = await build(); + + await expect(sweepStaleUploadRemnants(path.join(dir, 'no-such-folder'))).resolves.toBe(0); + }); +}); diff --git a/backend/tests/services/upload-storage-guard.test.js b/backend/tests/services/upload-storage-guard.test.js new file mode 100644 index 000000000..9654bc2ef --- /dev/null +++ b/backend/tests/services/upload-storage-guard.test.js @@ -0,0 +1,117 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * A full volume is not only a failed upload: where `/config` shares the + * filesystem, SQLite stops being able to write and the application stops + * working for everyone. These check the guard refuses first — and, just as + * importantly, that it stays out of the way when it cannot know. + */ + +let envContext; + +const build = async (env = {}) => { + envContext = await setupTestEnv({ tag: 'upload-storage-test-', env }); + return envContext.requireFresh('src/services/uploadStorageGuard'); +}; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('refusing an upload that will not fit', () => { + it('refuses one larger than the filesystem', async () => { + const guard = await build(); + + await expect( + guard.ensureStorageAvailable( + envContext.volumeDir, + Number.MAX_SAFE_INTEGER, + 'destination storage' + ) + ).rejects.toThrow(/Not enough storage available in destination storage/); + }); + + it('answers 507, so a client knows not to retry', async () => { + const guard = await build(); + + const error = await guard + .ensureStorageAvailable(envContext.volumeDir, Number.MAX_SAFE_INTEGER, 'destination storage') + .then( + () => null, + (err) => err + ); + + expect(error).not.toBeNull(); + expect(error.statusCode).toBe(507); + expect(error.isOperational).toBe(true); + }); + + it('accepts an upload there is room for', async () => { + const guard = await build(); + + await expect( + guard.ensureStorageAvailable(envContext.volumeDir, 1024, 'destination storage') + ).resolves.toBeUndefined(); + }); + + // The reserve is the whole point: an upload that would fit exactly, leaving + // nothing for the database beside it, is the one that takes the instance + // down. Both sizes below are measured against one reading of the free space, + // and both sit tens of megabytes away from the boundary, so what separates + // them is the reserve rather than the disk moving under the test. + it('refuses one that fits only by eating into the reserve', async () => { + const guard = await build({ UPLOAD_STORAGE_RESERVE: '64M' }); + const available = await guard.getAvailableBytes(envContext.volumeDir); + expect(Number.isFinite(available)).toBe(true); + + const megabytes = (count) => count * 1024 * 1024; + + // Room on the disk, none left over. + await expect( + guard.ensureStorageAvailable( + envContext.volumeDir, + available - megabytes(32), + 'destination storage' + ) + ).rejects.toThrow(/including reserve/); + + // Room on the disk, and the reserve still free afterwards. + await expect( + guard.ensureStorageAvailable( + envContext.volumeDir, + available - megabytes(128), + 'destination storage' + ) + ).resolves.toBeUndefined(); + }); +}); + +describe('staying out of the way when it cannot know', () => { + it('says nothing when the size of what is coming is unknown', async () => { + const guard = await build(); + + for (const unknown of [null, undefined, Number.NaN, -1]) { + await expect( + guard.ensureStorageAvailable(envContext.volumeDir, unknown, 'destination storage') + ).resolves.toBeUndefined(); + } + }); + + // Refusing every upload on a filesystem we cannot measure would cost more + // than the risk it avoids. + it('says nothing when the filesystem cannot be measured', async () => { + const guard = await build(); + const notADirectory = path.join(envContext.volumeDir, 'a-file'); + await fs.writeFile(notADirectory, 'x'); + const unmeasurable = path.join(notADirectory, 'under', 'a', 'file'); + + expect(await guard.getAvailableBytes(unmeasurable)).toBeNull(); + await expect( + guard.ensureStorageAvailable(unmeasurable, Number.MAX_SAFE_INTEGER, 'destination storage') + ).resolves.toBeUndefined(); + }); +}); diff --git a/backend/tests/services/uploadFolderTargetService.test.js b/backend/tests/services/uploadFolderTargetService.test.js new file mode 100644 index 000000000..b5388b736 --- /dev/null +++ b/backend/tests/services/uploadFolderTargetService.test.js @@ -0,0 +1,87 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The name a picked folder takes in its destination. + * + * The destination is a real one under the volume, because the folder about to + * be created is authorized by its logical path before the mkdir: a name an + * administrator hid, or the zone's own, never reaches the disk. What that + * refusal leaves is covered from the routes, in `upload-landing.test.js`. + */ + +let envContext; +let service; + +const OWNER = { user: { id: 'test-user' } }; + +const build = async () => { + envContext = await setupTestEnv({ tag: 'upload-folder-target-' }); + service = envContext.requireFresh('src/services/uploadFolderTargetService'); + const destinationRoot = path.join(envContext.volumeDir, 'Inbox'); + await fs.mkdir(destinationRoot, { recursive: true }); + return destinationRoot; +}; + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; + service = null; +}); + +describe('folder upload target reservation', () => { + it('keeps a duplicate folder batch together under one available directory', async () => { + const destinationRoot = await build(); + await fs.mkdir(path.join(destinationRoot, 'photos')); + + const resolve = (relativePath, uploadBatchId) => + service.resolveFolderUploadRelativePath({ + relativePath, + destinationRoot, + logicalBase: 'Inbox', + context: OWNER, + uploadBatchId, + }); + + const first = await resolve('photos/2026/one.jpg', 'folder-upload-0001'); + const second = await resolve('photos/2026/two.jpg', 'folder-upload-0001'); + const nextBatch = await resolve('photos/2026/three.jpg', 'folder-upload-0002'); + + expect(first).toBe('photos (1)/2026/one.jpg'); + expect(second).toBe('photos (1)/2026/two.jpg'); + expect(nextBatch).toBe('photos (2)/2026/three.jpg'); + }); + + it('atomically reserves a distinct destination before folder files are queued', async () => { + const destinationRoot = await build(); + await fs.mkdir(path.join(destinationRoot, 'photos')); + + const targetRoots = await Promise.all( + Array.from({ length: 3 }, () => + service.reserveFolderUploadTarget({ + destinationRoot, + logicalBase: 'Inbox', + sourceRoot: 'photos', + context: OWNER, + }) + ) + ); + + expect(targetRoots.sort()).toEqual(['photos (1)', 'photos (2)', 'photos (3)']); + }); + + it('rejects a nested path as a folder root reservation', async () => { + const destinationRoot = await build(); + + await expect( + service.reserveFolderUploadTarget({ + destinationRoot, + logicalBase: 'Inbox', + sourceRoot: 'photos/2026', + context: OWNER, + }) + ).rejects.toThrow('top-level folder name'); + }); +}); diff --git a/backend/tests/services/user-volumes.test.js b/backend/tests/services/user-volumes.test.js new file mode 100644 index 000000000..fbc0e0c5a --- /dev/null +++ b/backend/tests/services/user-volumes.test.js @@ -0,0 +1,362 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Volumes handed to one account. + * + * This is how an administrator gives somebody a folder and nothing else, so the + * validation is not paperwork — a reserved label collides with a route the + * explorer already owns, and a duplicate label makes two different folders + * answer to the same name in the sidebar, with only one of them reachable. + * + * It sat at 45%, and the half that was missing is every refusal. + */ + +let ctx; + +const setup = async () => { + const envContext = await setupTestEnv({ + tag: 'user-volumes-test-', + modules: ['src/services/db', 'src/services/userVolumesService'], + }); + const service = envContext.requireFresh('src/services/userVolumesService'); + const { getDb } = envContext.requireFresh('src/services/db'); + const db = await getDb(); + const now = new Date().toISOString(); + for (const id of ['alice', 'bob']) { + db.prepare( + `INSERT INTO users (id, email, email_verified, username, display_name, roles, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ).run(id, `${id}@example.com`, 1, id, id, '["user"]', now, now); + } + const dirs = {}; + for (const name of ['media', 'archive', 'other']) { + dirs[name] = path.join(envContext.tmpRoot, name); + await fs.mkdir(dirs[name], { recursive: true }); + } + ctx = { envContext, service, dirs, tmpRoot: envContext.tmpRoot }; + return ctx; +}; + +afterEach(async () => { + if (ctx) { + await ctx.envContext.cleanup(); + ctx = null; + } +}); + +const rejection = async (promise) => { + try { + await promise; + return null; + } catch (error) { + return error; + } +}; + +describe('adding a volume', () => { + it('stores it and hands back the client shape', async () => { + const { service, dirs } = await setup(); + + const volume = await service.addVolumeToUser({ + userId: 'alice', + label: 'Media', + volumePath: dirs.media, + }); + + expect(volume).toMatchObject({ + userId: 'alice', + label: 'Media', + path: dirs.media, + accessMode: 'readwrite', + }); + expect(volume.id).toBeTruthy(); + // The snake_case columns must not leak to a client. + expect(volume).not.toHaveProperty('user_id'); + }); + + it('accepts read-only', async () => { + const { service, dirs } = await setup(); + + const volume = await service.addVolumeToUser({ + userId: 'alice', + label: 'Archive', + volumePath: dirs.archive, + accessMode: 'readonly', + }); + + expect(volume.accessMode).toBe('readonly'); + }); + + it.each([ + ['no user', { userId: '', label: 'Media' }, 400], + ['no label', { userId: 'alice', label: ' ' }, 400], + ['a made-up access mode', { userId: 'alice', label: 'Media', accessMode: 'append' }, 400], + ])('refuses %s', async (_label, overrides, status) => { + const { service, dirs } = await setup(); + + const error = await rejection( + service.addVolumeToUser({ volumePath: dirs.media, ...overrides }) + ); + + expect(error?.status).toBe(status); + }); + + /** + * `personal`, `share` and `volumes` are paths the explorer already routes. + * A volume answering to one of them shadows the real thing. + */ + it.each(['personal', 'share', 'volumes', 'Personal', 'SHARE'])( + 'refuses the reserved label %s', + async (label) => { + const { service, dirs } = await setup(); + + const error = await rejection( + service.addVolumeToUser({ userId: 'alice', label, volumePath: dirs.media }) + ); + + expect(error?.status).toBe(400); + expect(String(error?.message)).toMatch(/reserved/i); + } + ); + + it('refuses a path that is not there', async () => { + const { service, tmpRoot } = await setup(); + + const error = await rejection( + service.addVolumeToUser({ + userId: 'alice', + label: 'Ghost', + volumePath: path.join(tmpRoot, 'nowhere'), + }) + ); + + expect(error?.status).toBe(400); + }); + + it('refuses a file where a directory was expected', async () => { + const { service, tmpRoot } = await setup(); + const file = path.join(tmpRoot, 'notes.txt'); + await fs.writeFile(file, 'x'); + + const error = await rejection( + service.addVolumeToUser({ userId: 'alice', label: 'Notes', volumePath: file }) + ); + + expect(error?.status).toBe(400); + expect(String(error?.message)).toMatch(/directory/i); + }); + + it('refuses the same path twice for one person', async () => { + const { service, dirs } = await setup(); + await service.addVolumeToUser({ userId: 'alice', label: 'Media', volumePath: dirs.media }); + + const error = await rejection( + service.addVolumeToUser({ userId: 'alice', label: 'Films', volumePath: dirs.media }) + ); + + expect(error?.status).toBe(409); + }); + + it('refuses the same label twice for one person', async () => { + const { service, dirs } = await setup(); + await service.addVolumeToUser({ userId: 'alice', label: 'Media', volumePath: dirs.media }); + + const error = await rejection( + service.addVolumeToUser({ userId: 'alice', label: 'Media', volumePath: dirs.archive }) + ); + + expect(error?.status).toBe(409); + }); + + /** Both clashes are per-person. Two accounts naming their own folder Media is fine. */ + it('lets a second person reuse a label, and a path', async () => { + const { service, dirs } = await setup(); + await service.addVolumeToUser({ userId: 'alice', label: 'Media', volumePath: dirs.media }); + + const bob = await service.addVolumeToUser({ + userId: 'bob', + label: 'Media', + volumePath: dirs.media, + }); + + expect(bob.userId).toBe('bob'); + }); + + it('trims the label rather than storing the spaces', async () => { + const { service, dirs } = await setup(); + + const volume = await service.addVolumeToUser({ + userId: 'alice', + label: ' Media ', + volumePath: dirs.media, + }); + + expect(volume.label).toBe('Media'); + }); +}); + +describe('listing and finding', () => { + it('returns only that person’s volumes, ordered by label', async () => { + const { service, dirs } = await setup(); + await service.addVolumeToUser({ userId: 'alice', label: 'Zulu', volumePath: dirs.media }); + await service.addVolumeToUser({ userId: 'alice', label: 'Alpha', volumePath: dirs.archive }); + await service.addVolumeToUser({ userId: 'bob', label: 'Bravo', volumePath: dirs.other }); + + const volumes = await service.getVolumesForUser('alice'); + + expect(volumes.map((v) => v.label)).toEqual(['Alpha', 'Zulu']); + }); + + it('answers with an empty list for somebody who has none', async () => { + const { service } = await setup(); + + expect(await service.getVolumesForUser('bob')).toEqual([]); + }); + + it('answers null for an id that is not there', async () => { + const { service } = await setup(); + + expect(await service.getVolumeById('no-such-volume')).toBeNull(); + }); +}); + +describe('matching a path to a volume', () => { + it('matches on the first segment, which is the label the UI shows', async () => { + const { service, dirs } = await setup(); + await service.addVolumeToUser({ userId: 'alice', label: 'Media', volumePath: dirs.media }); + + const found = await service.getUserVolumeForPath('alice', 'Media/Films/2026'); + + expect(found?.label).toBe('Media'); + }); + + it.each([ + ['leading slashes', '///Media/Films'], + ['a trailing slash', 'Media/'], + ])('normalises %s away first', async (_label, input) => { + const { service, dirs } = await setup(); + await service.addVolumeToUser({ userId: 'alice', label: 'Media', volumePath: dirs.media }); + + expect((await service.getUserVolumeForPath('alice', input))?.label).toBe('Media'); + }); + + it('answers null for the root, which is no volume', async () => { + const { service, dirs } = await setup(); + await service.addVolumeToUser({ userId: 'alice', label: 'Media', volumePath: dirs.media }); + + expect(await service.getUserVolumeForPath('alice', '')).toBeNull(); + expect(await service.getUserVolumeForPath('alice', '/')).toBeNull(); + }); + + /** Somebody else's volume is not yours, whatever it is called. */ + it('does not match a volume belonging to another person', async () => { + const { service, dirs } = await setup(); + await service.addVolumeToUser({ userId: 'bob', label: 'Media', volumePath: dirs.media }); + + expect(await service.getUserVolumeForPath('alice', 'Media/Films')).toBeNull(); + }); + + it('matches the label exactly, not by prefix', async () => { + const { service, dirs } = await setup(); + await service.addVolumeToUser({ userId: 'alice', label: 'Media', volumePath: dirs.media }); + + expect(await service.getUserVolumeForPath('alice', 'MediaArchive/x')).toBeNull(); + }); +}); + +describe('changing and removing', () => { + it('renames a volume', async () => { + const { service, dirs } = await setup(); + const volume = await service.addVolumeToUser({ + userId: 'alice', + label: 'Media', + volumePath: dirs.media, + }); + + const updated = await service.updateUserVolume(volume.id, { label: 'Films' }); + + expect(updated.label).toBe('Films'); + }); + + it('changes the access mode', async () => { + const { service, dirs } = await setup(); + const volume = await service.addVolumeToUser({ + userId: 'alice', + label: 'Media', + volumePath: dirs.media, + }); + + const updated = await service.updateUserVolume(volume.id, { accessMode: 'readonly' }); + + expect(updated.accessMode).toBe('readonly'); + }); + + it('refuses a rename onto a label that person already uses', async () => { + const { service, dirs } = await setup(); + await service.addVolumeToUser({ userId: 'alice', label: 'Media', volumePath: dirs.media }); + const second = await service.addVolumeToUser({ + userId: 'alice', + label: 'Archive', + volumePath: dirs.archive, + }); + + const error = await rejection(service.updateUserVolume(second.id, { label: 'Media' })); + + expect(error?.status).toBe(409); + }); + + it('refuses a rename onto a reserved label', async () => { + const { service, dirs } = await setup(); + const volume = await service.addVolumeToUser({ + userId: 'alice', + label: 'Media', + volumePath: dirs.media, + }); + + const error = await rejection(service.updateUserVolume(volume.id, { label: 'share' })); + + expect(error?.status).toBe(400); + }); + + it.each([ + ['update', (s, id) => s.updateUserVolume(id, { label: 'X' })], + ['remove', (s, id) => s.removeVolumeFromUser(id)], + ])('answers 404 to %s on a volume that is gone', async (_label, act) => { + const { service } = await setup(); + + const error = await rejection(act(service, 'no-such-volume')); + + expect(error?.status).toBe(404); + }); + + it('removes a volume and leaves the others', async () => { + const { service, dirs } = await setup(); + const media = await service.addVolumeToUser({ + userId: 'alice', + label: 'Media', + volumePath: dirs.media, + }); + await service.addVolumeToUser({ userId: 'alice', label: 'Archive', volumePath: dirs.archive }); + + expect(await service.removeVolumeFromUser(media.id)).toBe(true); + expect((await service.getVolumesForUser('alice')).map((v) => v.label)).toEqual(['Archive']); + }); + + /** Removing the assignment must not remove what it points at. */ + it('leaves the folder on disk alone', async () => { + const { service, dirs } = await setup(); + const volume = await service.addVolumeToUser({ + userId: 'alice', + label: 'Media', + volumePath: dirs.media, + }); + + await service.removeVolumeFromUser(volume.id); + + expect((await fs.stat(dirs.media)).isDirectory()).toBe(true); + }); +}); diff --git a/backend/tests/services/userSearchService.test.js b/backend/tests/services/userSearchService.test.js index 587735a7f..9817a7e22 100644 --- a/backend/tests/services/userSearchService.test.js +++ b/backend/tests/services/userSearchService.test.js @@ -121,7 +121,15 @@ describe('User Search Service', () => { INSERT INTO users (id, email, username, display_name, roles, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?) ` - ).run('user-no-display', 'nodisplay@example.com', 'nodisplayuser', null, '["user"]', now, now); + ).run( + 'user-no-display', + 'nodisplay@example.com', + 'nodisplayuser', + null, + '["user"]', + now, + now + ); const result = await searchUsersForMentions('nodisplay', 10); expect(result.Users.length).toBe(1); diff --git a/backend/tests/services/username-login.test.js b/backend/tests/services/username-login.test.js new file mode 100644 index 000000000..b6343a898 --- /dev/null +++ b/backend/tests/services/username-login.test.js @@ -0,0 +1,298 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Signing in with a username instead of an email address. + * + * Asked for in issue #5, and the reason is a fair one: a username is what + * somebody chose, an address is what their mail provider gave them. + * + * Two things had to be settled first. A username is not unique in the schema — + * the constraint was lost in an early migration, and `createLocalUser` derives + * one from the local part of the address, so `alice@example.com` and + * `alice@other.org` both become `alice`. A name that answers for two accounts + * identifies neither, and choosing between them would be choosing whose account + * a stranger signs into. + * + * And the lockout: it used to be keyed on what was typed. One account with two + * names would then have had one budget of failed attempts per name, and anyone + * alternating between them would never have exhausted either. + */ + +let envContext; +let users; + +const build = async (env = {}) => { + envContext = await setupTestEnv({ + tag: 'username-login-', + env: { AUTH_MAX_FAILED: '3', AUTH_LOCK_MINUTES: '15', ...env }, + modules: ['src/services/db', 'src/services/users'], + }); + users = envContext.requireFresh('src/services/users'); +}; + +const makeUser = async (overrides = {}) => + users.createLocalUser({ + email: 'alice@example.com', + username: 'alice', + displayName: 'Alice', + password: 'motdepasse', + roles: ['user'], + ...overrides, + }); + +const signIn = (identifier, password = 'motdepasse') => + users.attemptLocalLogin({ identifier, password }); + +afterEach(async () => { + if (envContext) await envContext.cleanup(); + envContext = null; +}); + +describe('signing in', () => { + beforeEach(async () => { + await build(); + await makeUser(); + }); + + it('works with the username', async () => { + const user = await signIn('alice'); + + expect(user?.email).toBe('alice@example.com'); + }); + + it('still works with the email address', async () => { + const user = await signIn('alice@example.com'); + + expect(user?.username).toBe('alice'); + }); + + /** Nobody remembers whether they capitalised their own name. */ + it('does not mind how the username was capitalised', async () => { + expect(await signIn('ALICE')).toBeTruthy(); + expect(await signIn('Alice')).toBeTruthy(); + }); + + it('does not mind how the address was capitalised either', async () => { + expect(await signIn('Alice@Example.COM')).toBeTruthy(); + }); + + it('ignores the spaces around what was typed', async () => { + expect(await signIn(' alice ')).toBeTruthy(); + }); + + it('refuses the right name with the wrong password', async () => { + expect(await signIn('alice', 'au-hasard')).toBeNull(); + }); + + it('refuses a name that belongs to nobody', async () => { + expect(await signIn('bob')).toBeNull(); + }); + + it('refuses nothing at all', async () => { + expect(await signIn('')).toBeNull(); + expect(await signIn(null)).toBeNull(); + expect(await signIn(' ')).toBeNull(); + }); + + /** The field used to be called `email`, and callers may still say so. */ + it('accepts the older name for the field', async () => { + const user = await users.attemptLocalLogin({ email: 'alice', password: 'motdepasse' }); + + expect(user?.email).toBe('alice@example.com'); + }); +}); + +describe('a username that answers for two accounts', () => { + /** + * The column carries no uniqueness constraint, so an installation upgraded + * from an older version can already hold this. Signing in with it would mean + * picking one of them, which is picking whose account a stranger reaches. + */ + const seedDuplicates = async () => { + const db = await envContext.requireFresh('src/services/db').getDb(); + await makeUser(); + await makeUser({ email: 'alice@other.org', username: null }); + db.prepare('UPDATE users SET username = ? WHERE email = ?').run('alice', 'alice@other.org'); + return db; + }; + + beforeEach(async () => { + await build(); + }); + + it('signs nobody in', async () => { + await seedDuplicates(); + + expect(await signIn('alice')).toBeNull(); + }); + + it('leaves both of them their address', async () => { + await seedDuplicates(); + + expect((await signIn('alice@example.com'))?.email).toBe('alice@example.com'); + expect((await signIn('alice@other.org'))?.email).toBe('alice@other.org'); + }); + + it('is not created by a new account taking a name already in use', async () => { + await makeUser(); + + await expect(makeUser({ email: 'alice@other.org' })).rejects.toThrow(/username/i); + }); + + it('is not created by capitalising it differently either', async () => { + await makeUser(); + + await expect(makeUser({ email: 'alice@other.org', username: 'Alice' })).rejects.toThrow( + /username/i + ); + }); + + it('is not created by renaming an account onto another', async () => { + const alice = await makeUser(); + const bob = await makeUser({ email: 'bob@example.com', username: 'bob' }); + + await expect( + users.updateUserProfile({ userId: bob.id, username: 'ALICE' }) + ).rejects.toMatchObject({ status: 409 }); + expect(alice.username).toBe('alice'); + }); + + it('does not stop an account keeping the name it already has', async () => { + const alice = await makeUser(); + + const updated = await users.updateUserProfile({ + userId: alice.id, + username: 'alice', + displayName: 'Alice A.', + }); + + expect(updated.displayName).toBe('Alice A.'); + }); + + it('does not stop an account clearing its username', async () => { + const alice = await makeUser(); + + const updated = await users.updateUserProfile({ userId: alice.id, username: '' }); + + expect(updated.username).toBeNull(); + }); + + /** An account with no username is not an account named "". */ + it('is not what two accounts without a username are', async () => { + await makeUser({ username: null }); + await makeUser({ email: 'bob@example.com', username: null }); + + expect(await signIn('')).toBeNull(); + }); + + /** + * An older version could store an empty string rather than nothing. It names + * no account, so it must not stop the next account being created without a + * username of its own. + */ + it('is not what an account with an empty username is', async () => { + const alice = await makeUser(); + const db = await envContext.requireFresh('src/services/db').getDb(); + db.prepare('UPDATE users SET username = ? WHERE id = ?').run('', alice.id); + + await expect(makeUser({ email: 'bob@example.com', username: null })).resolves.toBeTruthy(); + expect(await signIn('')).toBeNull(); + }); +}); + +/** + * A timeout of its own, for the reason `tests/routes/auth.test.js` gives: every + * attempt here is a bcrypt comparison in pure JavaScript, four to six a test, + * and that costs CPU. About a second alone; past the five-second default the + * moment the machine is busy — which is how "forgets the attempts once one of + * them works" failed a full run with the frontend suite beside it, and passed + * three runs out of three on its own. + */ +describe('the lockout after failed attempts', { timeout: 30_000 }, () => { + beforeEach(async () => { + await build(); + await makeUser(); + }); + + it('locks the account after enough of them', async () => { + await signIn('alice', 'faux'); + await signIn('alice', 'faux'); + await signIn('alice', 'faux'); + + await expect(signIn('alice')).rejects.toMatchObject({ status: 423 }); + }); + + /** + * One account, two names, one budget. Keyed on what was typed, alternating + * between the address and the username would have given twice the attempts — + * and with the lock never tripping, indefinitely many. + */ + it('counts attempts against the account, not against the name used', async () => { + await signIn('alice', 'faux'); + await signIn('alice@example.com', 'faux'); + await signIn('ALICE', 'faux'); + + await expect(signIn('alice@example.com')).rejects.toMatchObject({ status: 423 }); + }); + + it('locks the account whichever name is tried afterwards', async () => { + await signIn('alice@example.com', 'faux'); + await signIn('alice@example.com', 'faux'); + await signIn('alice@example.com', 'faux'); + + await expect(signIn('alice')).rejects.toMatchObject({ status: 423 }); + }); + + it('forgets the attempts once one of them works', async () => { + await signIn('alice', 'faux'); + await signIn('alice', 'faux'); + + expect(await signIn('alice')).toBeTruthy(); + + await signIn('alice', 'faux'); + await signIn('alice', 'faux'); + expect(await signIn('alice')).toBeTruthy(); + }); + + /** + * A name that belongs to nobody is not counted at all: the lock is per + * account, and counting for an unknown name would let anyone lock a + * colleague out by guessing at their address. + */ + it('counts nothing against a name that belongs to nobody', async () => { + await signIn('mallory', 'faux'); + await signIn('mallory', 'faux'); + await signIn('mallory', 'faux'); + await signIn('mallory', 'faux'); + + expect(await signIn('mallory')).toBeNull(); + }); + + /** + * An account that signs in through an identity provider has no password + * here. Guessing at one is still guessing, and still counted. + */ + it('counts attempts against an account that has no password', async () => { + const db = await envContext.requireFresh('src/services/db').getDb(); + const alice = db.prepare('SELECT id FROM users WHERE email = ?').get('alice@example.com'); + db.prepare('DELETE FROM auth_methods WHERE user_id = ?').run(alice.id); + + await signIn('alice', 'faux'); + await signIn('alice', 'faux'); + await signIn('alice', 'faux'); + + await expect(signIn('alice')).rejects.toMatchObject({ status: 423 }); + }); + + it('does not lock one account out by failing on another', async () => { + await makeUser({ email: 'bob@example.com', username: 'bob' }); + + await signIn('bob', 'faux'); + await signIn('bob', 'faux'); + await signIn('bob', 'faux'); + + expect(await signIn('alice')).toBeTruthy(); + }); +}); diff --git a/backend/tests/services/version-marks.test.js b/backend/tests/services/version-marks.test.js new file mode 100644 index 000000000..e43d391ea --- /dev/null +++ b/backend/tests/services/version-marks.test.js @@ -0,0 +1,134 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The count behind the mark a listing puts on a file. + * + * One query answers a whole folder, so what it must get right is which rows + * it answers for. The route tests cover the folder boundary; these cover the + * three states it has to read through — a history whose file went to the + * trash, a version half-written or half-deleted, and a root registered twice. + */ + +let env; +let versions; +let store; +let trashStore; +let db; + +const load = (relative) => require(modulePath(relative)); + +const volume = (...segments) => path.join(env.volumeDir, ...segments); + +const write = async (relative, content) => { + await fs.mkdir(path.dirname(volume(relative)), { recursive: true }); + await fs.writeFile(volume(relative), content); +}; + +/** Save the way the editor does, which is what makes a version. */ +const save = (relative, content) => + load('src/services/versions/operations').saveFile( + volume(relative), + (temporary) => fs.writeFile(temporary, content), + { source: 'editor' } + ); + +const marks = (folder) => versions.marksForFolder(volume(folder)); + +beforeEach(async () => { + env = await setupTestEnv({ tag: 'version-marks-' }); + versions = load('src/services/versions'); + store = load('src/services/versions/store'); + trashStore = load('src/services/trash/store'); + db = await load('src/services/db').getDb(); + await write('Projects/notes.md', 'one\n'); +}); + +afterEach(async () => { + load('src/services/trash/maintenance').stop(); + vi.restoreAllMocks(); + await env.cleanup(); +}); + +const historyOf = (relativePath) => + store.listFiles(db, {}).find((file) => file.relativePath === relativePath); + +describe('what the count reads', () => { + it('answers the top of a zone, which is a folder like any other', async () => { + // `locateZoneRoot` refuses to name the zone of the root itself — true for + // a file being deleted, which cannot go into its own volume's trash, and + // useless for a listing. The files at the top of a volume have histories + // like any others. + await save('Projects/notes.md', 'two\n'); + + expect((await marks('Projects')).get('notes.md')).toMatchObject({ versions: 1 }); + }); + + it('leaves out the history of a file that has gone to the trash', async () => { + // A file deleted and then a new one created at the same path: the old + // history is still on the books, under the same path, and would put its + // count on a file that has nothing to do with it. + await save('Projects/notes.md', 'two\n'); + const history = historyOf('notes.md'); + store.setFileState(db, history.id, 'trashed'); + + expect((await marks('Projects')).get('notes.md')).toBeUndefined(); + }); + + it('counts a version that is kept, and not one on its way in or out', async () => { + await save('Projects/notes.md', 'two\n'); + await save('Projects/notes.md', 'three\n'); + const history = historyOf('notes.md'); + const [first, second] = store.listVersionsOfFile(db, history.id); + + expect((await marks('Projects')).get('notes.md').versions).toBe(2); + + // Written before the disk is touched, and not yet a version anybody has. + store.setVersionState(db, first.id, 'capturing'); + expect((await marks('Projects')).get('notes.md').versions).toBe(1); + + // On its way out, on a volume that was not available to remove it from. + store.setVersionState(db, second.id, 'purging'); + expect((await marks('Projects')).get('notes.md')).toBeUndefined(); + }); + + it('adds up a root that has been registered twice', async () => { + // One physical tree, two zone rows: the second is what an installation + // that lost its database and rebuilt it looks like, and a file's history + // may sit under either. + await save('Projects/notes.md', 'two\n'); + const first = trashStore.listZones(db).find((zone) => zone.root === volume('Projects')); + const second = trashStore.insertZone(db, { + id: 'second-row-same-root', + root: first.root, + space: first.space, + }); + const history = historyOf('notes.md'); + store.insertFile(db, { + id: 'history-under-the-other-row', + zoneId: second.id, + relativePath: 'notes.md', + }); + store.insertVersion(db, { + id: 'version-under-the-other-row', + fileId: 'history-under-the-other-row', + zoneId: second.id, + state: 'kept', + size: 100, + modifiedAt: new Date().toISOString(), + }); + + expect(historyOf('notes.md').id).toBe(history.id); + expect((await marks('Projects')).get('notes.md')).toMatchObject({ + versions: 2, + bytes: 4 + 100, + }); + }); + + it('answers nothing for a folder in no zone at all', async () => { + expect((await marks('.')).size).toBe(0); + }); +}); diff --git a/backend/tests/services/versions-lifecycle-case.test.js b/backend/tests/services/versions-lifecycle-case.test.js new file mode 100644 index 000000000..1cbff710d --- /dev/null +++ b/backend/tests/services/versions-lifecycle-case.test.js @@ -0,0 +1,76 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { modulePath, setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * The histories under a folder, and under another whose name differs from it + * only by case. + * + * On a Linux volume `Docs` and `docs` are two folders. The histories under one + * of them were found with `LIKE 'Docs/%'`, which SQLite matches without regard + * to case: moving `Docs` reassigned the histories of `docs/…` to files under + * the new name, and deleting it for good purged them. + * + * The rows are written straight into the store rather than made by saving + * files, because this machine's disk may not hold both folders at once, and the + * defect is in the query. + */ + +let env; +let db; +let store; +let lifecycle; +let zoneId; + +const load = (relative) => require(modulePath(relative)); +const volume = (...segments) => path.join(env.volumeDir, ...segments); + +const HISTORIES = ['Docs', 'Docs/a.txt', 'Docs/sub/b.txt', 'docs/c.txt', 'Docs2/d.txt', 'Docs.txt']; + +beforeEach(async () => { + env = await setupTestEnv({ tag: 'versions-case-', env: { UPLOAD_STORAGE_RESERVE: '0' } }); + store = load('src/services/versions/store'); + lifecycle = load('src/services/versions/lifecycle'); + db = await load('src/services/db').getDb(); + await fs.mkdir(volume('Projects'), { recursive: true }); + zoneId = (await load('src/services/trash/zones').ensureZone(volume('Projects'))).id; + HISTORIES.forEach((relativePath, index) => + store.insertFile(db, { id: `vf-${index}`, zoneId, relativePath }) + ); +}); + +afterEach(async () => { + await env.cleanup(); +}); + +const live = () => + db + .prepare( + "SELECT relative_path FROM version_files WHERE zone_id = ? AND state = 'live' ORDER BY relative_path" + ) + .pluck() + .all(zoneId); + +describe('the histories of a folder', () => { + it('follow it when it is renamed, and those of the folder spelt otherwise stay', async () => { + const moved = await lifecycle.onMoved(volume('Projects', 'Docs'), volume('Projects', 'Papers')); + + expect(moved).toBe(3); + expect(live()).toEqual([ + 'Docs.txt', + 'Docs2/d.txt', + 'Papers', + 'Papers/a.txt', + 'Papers/sub/b.txt', + 'docs/c.txt', + ]); + }); + + it('go when it is deleted for good, and those of the folder spelt otherwise stay', async () => { + await lifecycle.onDeleted(volume('Projects', 'Docs')); + + expect(live()).toEqual(['Docs.txt', 'Docs2/d.txt', 'docs/c.txt']); + }); +}); diff --git a/backend/tests/services/versions-lifecycle.test.js b/backend/tests/services/versions-lifecycle.test.js index 8cbb3a03d..a2c786e68 100644 --- a/backend/tests/services/versions-lifecycle.test.js +++ b/backend/tests/services/versions-lifecycle.test.js @@ -94,7 +94,6 @@ const save = (relative, content) => const withHistory = async (relative, ...contents) => { await write(relative, contents[0]); for (const content of contents.slice(1)) { - // eslint-disable-next-line no-await-in-loop await save(relative, content); } }; @@ -121,7 +120,6 @@ const keptContents = async (file) => { const expectConsistent = async () => { for (const zone of trashStore.listZones(db)) { - // eslint-disable-next-line no-await-in-loop expect((await verify.verifyZone(zone)).violations).toEqual([]); } }; @@ -366,11 +364,9 @@ describe('a file that disappears outside the application', () => { it('orphans nothing when too many files vanish at once', async () => { for (let index = 0; index < 6; index += 1) { - // eslint-disable-next-line no-await-in-loop await withHistory(`Projects/file-${index}.txt`, 'before', 'after'); } for (let index = 0; index < 6; index += 1) { - // eslint-disable-next-line no-await-in-loop await fs.rm(volume(`Projects/file-${index}.txt`)); } diff --git a/backend/tests/services/versions-operations.test.js b/backend/tests/services/versions-operations.test.js index c219a69f1..cbc83e0fe 100644 --- a/backend/tests/services/versions-operations.test.js +++ b/backend/tests/services/versions-operations.test.js @@ -254,7 +254,6 @@ describe('saving over a file', () => { await setVersions({ maxPerFile: 2 }); await write('Projects/report.txt', 'v0'); for (const content of ['v1', 'v2', 'v3', 'v4']) { - // eslint-disable-next-line no-await-in-loop await save('Projects/report.txt', content); } @@ -479,7 +478,6 @@ describe('the recovery of a zone', () => { it('touches nothing when too many contents have vanished at once', async () => { await write('Projects/report.txt', 'v0'); for (let index = 1; index <= 6; index += 1) { - // eslint-disable-next-line no-await-in-loop await save('Projects/report.txt', `v${index}`); } await fs.rm(zones.versionsDirectory(zoneOf().root), { recursive: true }); diff --git a/backend/tests/setup/default-directories.js b/backend/tests/setup/default-directories.js new file mode 100644 index 000000000..81b4466cf --- /dev/null +++ b/backend/tests/setup/default-directories.js @@ -0,0 +1,48 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { afterAll } from 'vitest'; + +/** + * Give a test that never chose its directories temporary ones. + * + * Most suites go through setupTestEnv, which sets CONFIG_DIR and CACHE_DIR. + * A few load the configuration on their own, and fall back on /config and + * /cache. Loading it writes there now: the session secret is generated and kept + * in CONFIG_DIR, and operations in flight are recorded under CACHE_DIR. On a + * developer's machine that only logs a warning; run as root — the media job's + * container, a host — it would create a real /config/session-secret. + * + * A directory set by the environment the suite runs in is left alone. One this + * file set for an earlier test file in the same worker is replaced, since that + * file removed it when it finished. + */ + +const OWNED = '__NEXTEXPLORER_TEST_DEFAULT_DIRS__'; + +const owned = new Set((process.env[OWNED] || '').split(path.delimiter).filter(Boolean)); +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'nextexplorer-default-dirs-')); +const assigned = {}; + +for (const [name, sub] of [ + ['CONFIG_DIR', 'config'], + ['CACHE_DIR', 'cache'], +]) { + const current = process.env[name]; + if (current && !owned.has(current)) continue; + owned.delete(current); + assigned[name] = path.join(root, sub); + process.env[name] = assigned[name]; + owned.add(assigned[name]); +} +process.env[OWNED] = [...owned].join(path.delimiter); + +afterAll(() => { + for (const [name, value] of Object.entries(assigned)) { + if (process.env[name] === value) delete process.env[name]; + owned.delete(value); + } + process.env[OWNED] = [...owned].join(path.delimiter); + fs.rmSync(root, { recursive: true, force: true }); +}); diff --git a/backend/tests/setup/loopback-listen.js b/backend/tests/setup/loopback-listen.js new file mode 100644 index 000000000..875f32144 --- /dev/null +++ b/backend/tests/setup/loopback-listen.js @@ -0,0 +1,51 @@ +import net from 'node:net'; + +/** + * Make a test server listen on the address its client actually dials. + * + * Supertest opens a server with `listen(0)` and then connects to + * `127.0.0.1:`. Given no host, Node binds the wildcard address, and the + * kernel is free to hand out a port another process already holds on 127.0.0.1 + * specifically: the two bindings do not conflict, the wildcard being the less + * specific of the two. The connection is then resolved the other way round — + * 127.0.0.1 wins over the wildcard — so the request is answered by that other + * process, and the server the test just opened never sees it. + * + * That is not hypothetical. macOS draws `listen(0)` ports from 49152-65535, and + * desktop applications settle inside that range: a mail client holding + * 127.0.0.1:61814 answered 404 to whichever request drew that port. A full run + * opens a few thousand servers, so about one run in fifteen lost one request — + * a different test each time, failing on a status nothing in the application + * had produced, with no trace in any log because the request never arrived. + * + * Naming the address closes it: the kernel will not allocate a port already + * taken on 127.0.0.1, so the server a test opens is the one its request + * reaches. It has to be bound synchronously, because supertest reads + * `address().port` on the line after `listen()` — passing a host to `listen()` + * defers the bind past that point and hands it null. + * + * Only the bare `listen(0)` form is redirected; a test that names an address or + * asks for a particular port means what it says, and is left alone. + */ +const originalListen = net.Server.prototype.listen; +const IPv4 = 4; +const DEFAULT_BACKLOG = 511; + +net.Server.prototype.listen = function listen(...args) { + const bareEphemeralPort = args[0] === 0 && (args.length === 1 || typeof args[1] === 'function'); + if (!bareEphemeralPort) return originalListen.apply(this, args); + + const onListening = args[1]; + if (onListening) this.once('listening', onListening); + + try { + this._listen2('127.0.0.1', 0, IPv4, DEFAULT_BACKLOG, undefined, 0); + return this; + } catch { + // Node no longer binds this way: fall back to what it does by default. The + // suite is then exposed to the collision again, which is the situation it + // was in before this file existed. + if (onListening) this.removeListener('listening', onListening); + return originalListen.apply(this, args); + } +}; diff --git a/backend/tests/utils/cbor.test.js b/backend/tests/utils/cbor.test.js new file mode 100644 index 000000000..433325177 --- /dev/null +++ b/backend/tests/utils/cbor.test.js @@ -0,0 +1,164 @@ +import { describe, expect, it } from 'vitest'; + +const { decode, decodeFirst, CborError } = require('../../src/utils/cbor'); + +/** + * The CBOR reader, against the examples published with the format. + * + * Every vector below is from RFC 8949 Appendix A. They are the only way to + * know this reads what the rest of the world writes rather than what its own + * writer would have produced — the same reason the TOTP code is checked + * against RFC 6238's published codes. + * + * The refusals matter as much: this parses bytes that arrive before anybody is + * signed in, so a truncated length, a second value hiding behind the first, or + * a form CTAP2 forbids has to be an error and not a shrug. + */ + +const from = (hex) => Buffer.from(hex, 'hex'); + +describe('what RFC 8949 says these bytes mean', () => { + it.each([ + ['00', 0], + ['01', 1], + ['0a', 10], + ['17', 23], + ['1818', 24], + ['1819', 25], + ['1864', 100], + ['1903e8', 1000], + ['1a000f4240', 1000000], + ['1b000000e8d4a51000', 1000000000000], + ['20', -1], + ['29', -10], + ['3863', -100], + ['3903e7', -1000], + ])('reads %s as %s', (hex, expected) => { + expect(decode(from(hex))).toBe(expected); + }); + + it.each([ + ['60', ''], + ['6161', 'a'], + ['6449455446', 'IETF'], + ['62225c', '"\\'], + ['62c3bc', 'ü'], + ])('reads the text %s', (hex, expected) => { + expect(decode(from(hex))).toBe(expected); + }); + + it.each([ + ['40', ''], + ['4401020304', '01020304'], + ])('reads the bytes %s', (hex, expected) => { + const value = decode(from(hex)); + expect(Buffer.isBuffer(value)).toBe(true); + expect(value.toString('hex')).toBe(expected); + }); + + it.each([ + ['f4', false], + ['f5', true], + ['f6', null], + ['f7', undefined], + ])('reads the simple value %s', (hex, expected) => { + expect(decode(from(hex))).toBe(expected); + }); + + it('reads arrays, nested ones included', () => { + expect(decode(from('80'))).toEqual([]); + expect(decode(from('83010203'))).toEqual([1, 2, 3]); + expect(decode(from('8301820203820405'))).toEqual([1, [2, 3], [4, 5]]); + }); + + it('reads maps as maps, keeping integer keys integers', () => { + expect(decode(from('a0'))).toEqual(new Map()); + expect(decode(from('a201020304'))).toEqual( + new Map([ + [1, 2], + [3, 4], + ]) + ); + expect(decode(from('a26161016162820203'))).toEqual( + new Map([ + ['a', 1], + ['b', [2, 3]], + ]) + ); + expect(decode(from('826161a161626163'))).toEqual(['a', new Map([['b', 'c']])]); + }); + + it('keeps a negative label apart from the text of it', () => { + // A COSE key labels its coordinates -1, -2, -3. Decoded into an object + // those become "-1" beside any "-1" somebody sent as text. + // {-1: 1, "-1": 2} — the label and the text of it, side by side. + const map = decode(from('a22001622d3102')); + expect(map.get(-1)).toBe(1); + expect(map.get('-1')).toBe(2); + }); +}); + +describe('what it refuses', () => { + it.each([ + ['an indefinite-length byte string', '5f42010243030405ff'], + ['an indefinite-length array', '9fff'], + ['an indefinite-length map', 'bf61610161629f0203ffff'], + ['a tag', 'c11a514b67b0'], + ['a half-precision float', 'f93c00'], + ['a double', 'fb3ff199999999999a'], + ['an unassigned length form', '1c'], + ])('refuses %s', (_what, hex) => { + expect(() => decode(from(hex))).toThrow(CborError); + }); + + it('refuses a length that runs past the data', () => { + expect(() => decode(from('4401'))).toThrow(/past the end/); + expect(() => decode(from('83010203040506'))).toThrow(CborError); + }); + + it('refuses a second value hiding behind the first', () => { + expect(() => decode(from('0101'))).toThrow(/more than one value/); + }); + + it('refuses the same map key twice', () => { + expect(() => decode(from('a2010201ff'))).toThrow(CborError); + expect(() => decode(from('a201020103'))).toThrow(/same key twice/); + }); + + it('refuses an integer too large to be read exactly', () => { + expect(() => decode(from('1bffffffffffffffff'))).toThrow(/larger than this reads/); + }); + + it('refuses nesting without end', () => { + // Twenty opening arrays, one deeper than it will follow. + const deep = Buffer.from('81'.repeat(20) + '01', 'hex'); + expect(() => decode(deep)).toThrow(/nested too deeply/); + }); + + it('refuses anything that is not bytes', () => { + expect(() => decode('0a')).toThrow(/needs bytes/); + }); +}); + +describe('reading one value out of more data', () => { + it('says where the value ended', () => { + const attestation = Buffer.concat([from('a201020304'), Buffer.from('trailing')]); + const { value, bytesRead } = decodeFirst(attestation); + + expect(value).toEqual( + new Map([ + [1, 2], + [3, 4], + ]) + ); + expect(bytesRead).toBe(5); + expect(attestation.subarray(bytesRead).toString()).toBe('trailing'); + }); + + it('hands back bytes the caller can keep', () => { + const source = from('4401020304'); + const value = decode(source); + source.fill(0); + expect(value.toString('hex')).toBe('01020304'); + }); +}); diff --git a/backend/tests/utils/client-disconnect.test.js b/backend/tests/utils/client-disconnect.test.js new file mode 100644 index 000000000..927ef8075 --- /dev/null +++ b/backend/tests/utils/client-disconnect.test.js @@ -0,0 +1,47 @@ +import { describe, it, expect, vi } from 'vitest'; +import { EventEmitter } from 'node:events'; + +const { whenClientDisconnects } = require('../../src/utils/clientDisconnect'); + +/** + * Every abandoned search went on running for its full budget. `close` fires on + * a finished response too, so telling the two apart is the whole job: a + * promise that settles when a response ends normally would cancel the search + * that is about to answer. + */ +const response = (writableEnded = false) => Object.assign(new EventEmitter(), { writableEnded }); + +describe('noticing that nobody is waiting any more', () => { + it('settles when the connection closes with nothing written', async () => { + const res = response(); + const gone = whenClientDisconnects(res); + let settled = false; + gone.then(() => (settled = true)); + + res.emit('close'); + await Promise.resolve(); + + expect(settled).toBe(true); + }); + + it('stays pending when the response closes because it answered', async () => { + const res = response(); + const gone = whenClientDisconnects(res); + const settled = vi.fn(); + gone.then(settled); + + res.writableEnded = true; + res.emit('close'); + await Promise.resolve(); + + expect(settled).not.toHaveBeenCalled(); + }); + + it('never settles for a response that had already finished', async () => { + const settled = vi.fn(); + whenClientDisconnects(response(true)).then(settled); + await Promise.resolve(); + + expect(settled).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/tests/utils/compressed-response.test.js b/backend/tests/utils/compressed-response.test.js new file mode 100644 index 000000000..8da60561e --- /dev/null +++ b/backend/tests/utils/compressed-response.test.js @@ -0,0 +1,68 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Which coding a response is sent in, read from what the client wrote. + * + * What a browser sends is easy; what matters is what it refuses. A coding + * refused with `q=0` and sent anyway is a page the client cannot read, and it + * says so in the same header that accepts it. + */ + +let envContext; +let chooseEncoding; + +beforeAll(async () => { + envContext = await setupTestEnv({ tag: 'compressed-response-test-' }); + ({ chooseEncoding } = envContext.requireFresh('src/utils/compressedResponse')); +}); + +afterAll(async () => { + await envContext.cleanup(); +}); + +describe('choosing the content coding', () => { + it.each([ + // What browsers send: Edge and Chrome over plain http, then over HTTPS. + ['gzip, deflate', 'gzip'], + ['gzip, deflate, br, zstd', 'br'], + ['br', 'br'], + ['GZIP', 'gzip'], + ['x-gzip', 'gzip'], + ['*', 'br'], + ])('%j → %s', (header, expected) => { + expect(chooseEncoding(header)).toBe(expected); + }); + + it.each([ + ['br;q=0, gzip', 'gzip'], + ['gzip;q=0, deflate', 'identity'], + ['br;q=0, gzip;q=0', 'identity'], + ['gzip; q=0.000', 'identity'], + ['*;q=0', 'identity'], + ['gzip, *;q=0', 'gzip'], + ])('honours a refusal: %j → %s', (header, expected) => { + expect(chooseEncoding(header)).toBe(expected); + }); + + it.each([ + ['br;q=0.5, gzip;q=0.8', 'gzip'], + ['gzip;q=0.2, br;q=0.3', 'br'], + ['identity, gzip;q=0.5', 'identity'], + ['gzip;q=0.001', 'gzip'], + ])('follows the qualities: %j → %s', (header, expected) => { + expect(chooseEncoding(header)).toBe(expected); + }); + + it.each([ + [undefined, 'identity'], + ['', 'identity'], + ['identity', 'identity'], + ['deflate', 'identity'], + // A quality that is not one is not an acceptance. + ['gzip;q=1.5', 'identity'], + ['gzip;q=high', 'identity'], + ])('sends nothing compressed without a clear yes: %j → %s', (header, expected) => { + expect(chooseEncoding(header)).toBe(expected); + }); +}); diff --git a/backend/tests/utils/map-with-concurrency.test.js b/backend/tests/utils/map-with-concurrency.test.js new file mode 100644 index 000000000..625c42eea --- /dev/null +++ b/backend/tests/utils/map-with-concurrency.test.js @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest'; + +const { mapWithConcurrency } = require('../../src/utils/mapWithConcurrency'); + +/** + * `Promise.all(list.map(...))` is fine for a list the code chose and quite + * different for one a request brought with it. This is what keeps the number + * of operations in flight ours rather than the caller's. + */ +describe('mapping with a bound on what is in flight', () => { + const settle = () => new Promise((resolve) => setTimeout(resolve, 1)); + + it('never has more running than it was allowed', async () => { + let running = 0; + let peak = 0; + + await mapWithConcurrency( + Array.from({ length: 200 }, (unused, index) => index), + async (value) => { + running += 1; + peak = Math.max(peak, running); + await settle(); + running -= 1; + return value; + }, + 4 + ); + + expect(peak).toBe(4); + }); + + it('answers in the order it was asked, whatever finishes first', async () => { + const result = await mapWithConcurrency( + [30, 5, 20, 1], + async (delay) => { + await new Promise((resolve) => setTimeout(resolve, delay)); + return delay; + }, + 4 + ); + + expect(result).toEqual([30, 5, 20, 1]); + }); + + it('does not start a worker for a list it does not have', async () => { + let calls = 0; + const result = await mapWithConcurrency([], async () => { + calls += 1; + }); + + expect(result).toEqual([]); + expect(calls).toBe(0); + }); +}); diff --git a/backend/tests/utils/ndjson-throttle.test.js b/backend/tests/utils/ndjson-throttle.test.js new file mode 100644 index 000000000..8bc620e98 --- /dev/null +++ b/backend/tests/utils/ndjson-throttle.test.js @@ -0,0 +1,68 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { throttleProgress } from '../../src/utils/ndjsonStream.js'; + +/** + * A bulk operation reports once per item. Three thousand files meant three + * thousand socket writes and as many reactive updates in the browser, for a + * bar with a hundred distinct positions. Throttling is only safe if the last + * position always arrives: a bar stuck at 97% reads as a hung operation. + */ + +afterEach(() => vi.useRealTimers()); + +describe('Progress throttling', () => { + it('keeps the first event and drops the flood behind it', () => { + vi.useFakeTimers(); + const write = vi.fn(); + const report = throttleProgress(write, 100); + + for (let i = 1; i <= 500; i += 1) report({ type: 'progress', completedItems: i }); + + // One write instead of five hundred. + expect(write).toHaveBeenCalledTimes(1); + expect(write.mock.calls[0][0].completedItems).toBe(1); + }); + + it('lets one through per interval', () => { + vi.useFakeTimers(); + const write = vi.fn(); + const report = throttleProgress(write, 100); + + report({ type: 'progress', completedItems: 1 }); + vi.advanceTimersByTime(150); + report({ type: 'progress', completedItems: 2 }); + vi.advanceTimersByTime(150); + report({ type: 'progress', completedItems: 3 }); + + expect(write).toHaveBeenCalledTimes(3); + }); + + it('flushes the last position when the work ends', () => { + vi.useFakeTimers(); + const write = vi.fn(); + const report = throttleProgress(write, 100); + + report({ type: 'progress', completedItems: 1, percent: 0 }); + for (let i = 2; i <= 3000; i += 1) { + report({ type: 'progress', completedItems: i, percent: Math.round((i / 3000) * 100) }); + } + report.flush(); + + // Whatever was held back, completion is what the user is left looking at. + const last = write.mock.calls.at(-1)[0]; + expect(last.completedItems).toBe(3000); + expect(last.percent).toBe(100); + }); + + it('does not write twice when nothing is pending', () => { + vi.useFakeTimers(); + const write = vi.fn(); + const report = throttleProgress(write, 100); + + report({ type: 'progress', completedItems: 1 }); + report.flush(); + report.flush(); + + expect(write).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/tests/utils/onlyoffice-document-types.test.js b/backend/tests/utils/onlyoffice-document-types.test.js new file mode 100644 index 000000000..fefea3d7d --- /dev/null +++ b/backend/tests/utils/onlyoffice-document-types.test.js @@ -0,0 +1,61 @@ +import { createRequire } from 'module'; +import { describe, it, expect } from 'vitest'; + +const require = createRequire(import.meta.url); +const { getDocumentType } = require('../../src/utils/onlyofficeDocumentTypes'); + +/** + * The Document Server validates documentType against the extension and refuses + * the config when the two disagree — the user sees "the file content does not + * match the file extension", which names the file and never the mapping that + * sent it to the wrong editor. + * + * This is the expression it validates with, copied from a running server's + * web-apps/apps/api/documents/api.js. Its capture groups are, in order, the + * five editors. Checking our mapping against it is the only way to know the + * two agree; a hand-written list of examples would only prove the examples. + */ +const DOCUMENT_SERVER_GROUPS = [ + ['cell', 'xls|xlsx|ods|csv|tsv|gsheet|xlsm|xlt|xltm|xltx|fods|ots|xlsb|sxc|et|ett|numbers'], + ['slide', 'pps|ppsx|ppt|pptx|odp|gslides|pot|potm|potx|ppsm|pptm|fodp|otp|sxi|dps|dpt|key|odg'], + ['pdf', 'pdf|djvu|xps|oxps'], + [ + 'word', + 'doc|docx|odt|gdoc|txt|rtf|mht|htm|html|mhtml|epub|docm|dot|dotm|dotx|fodt|ott|fb2|xml|oform|docxf|sxw|stw|wps|wpt|pages|hwp|hwpx|md|hml', + ], + ['diagram', 'vsdx|vssx|vstx|vsdm|vssm|vstm'], +]; + +describe('ONLYOFFICE document types', () => { + it('agrees with the Document Server on every extension it accepts', () => { + const disagreements = []; + + for (const [expected, extensions] of DOCUMENT_SERVER_GROUPS) { + for (const ext of extensions.split('|')) { + const actual = getDocumentType(ext); + if (actual !== expected) disagreements.push({ ext, expected, actual }); + } + } + + expect(disagreements).toEqual([]); + }); + + it('opens a drawing with the slide editor, not the text one', () => { + // The case that surfaced this: .odg went through the catch-all and was + // announced as a word document, so the Document Server refused it. + expect(getDocumentType('odg')).toBe('slide'); + }); + + it('recognises the editors added after word/cell/slide', () => { + expect(getDocumentType('pdf')).toBe('pdf'); + expect(getDocumentType('vsdx')).toBe('diagram'); + }); + + it('refuses an extension the Document Server has no editor for', () => { + // Null, not a guess. Guessing is what produced an error raised by the + // editor rather than a refusal here, where the setting can be named. + for (const ext of ['zip', 'png', 'mp4', 'iso', '']) { + expect(getDocumentType(ext)).toBeNull(); + } + }); +}); diff --git a/backend/tests/utils/path-containment.test.js b/backend/tests/utils/path-containment.test.js index 6a3c95aa9..31e06395f 100644 --- a/backend/tests/utils/path-containment.test.js +++ b/backend/tests/utils/path-containment.test.js @@ -37,7 +37,9 @@ describe('Volume path containment', () => { // The lexical check passes (the string starts with the volume root), so // only the real-path check can catch this. - await expect(resolveVolumePath('escape/secret.txt')).rejects.toThrow(/outside the configured volume/i); + await expect(resolveVolumePath('escape/secret.txt')).rejects.toThrow( + /outside the configured volume/i + ); await expect(resolveVolumePath('escape')).rejects.toThrow(/outside the configured volume/i); }); @@ -93,7 +95,9 @@ describe('Volume path containment', () => { const { resolveVolumePath } = env.requireFresh('src/utils/pathUtils'); await expect(resolveVolumePath('dead')).rejects.toThrow(/outside the configured volume/i); - await expect(resolveVolumePath('dead/child.txt')).rejects.toThrow(/outside the configured volume/i); + await expect(resolveVolumePath('dead/child.txt')).rejects.toThrow( + /outside the configured volume/i + ); }); it('accepts a broken link whose target stays inside the volume', async () => { @@ -205,9 +209,9 @@ describe('Other spaces containment', () => { await fs.writeFile(path.join(outside, 'secret.txt'), 'not yours'); await fs.symlink(outside, path.join(userRoot, 'escape')); - await expect( - resolveLogicalPath('personal/escape/secret.txt', { user }) - ).rejects.toThrow(/outside the configured user directory/i); + await expect(resolveLogicalPath('personal/escape/secret.txt', { user })).rejects.toThrow( + /outside the configured user directory/i + ); }); /** A refusal must reach the caller, not the process. */ @@ -360,7 +364,9 @@ describe('the containment check on its own', () => { await expect(assertRealPathWithinRoot('/etc/nothing/here', env.volumeDir)).rejects.toThrow( /outside the configured volume/i ); - await expect(assertRealPathWithinRoot(path.join(env.tmpRoot, 'elsewhere', 'file.txt'), env.volumeDir)).rejects.toThrow(/outside the configured volume/i); + await expect( + assertRealPathWithinRoot(path.join(env.tmpRoot, 'elsewhere', 'file.txt'), env.volumeDir) + ).rejects.toThrow(/outside the configured volume/i); }); it('refuses a path outside the root when it does exist', async () => { @@ -369,18 +375,22 @@ describe('the containment check on its own', () => { await fs.mkdir(outside, { recursive: true }); await fs.writeFile(path.join(outside, 'secret.txt'), 'not yours'); - await expect(assertRealPathWithinRoot(path.join(outside, 'secret.txt'), env.volumeDir)).rejects.toThrow( - /outside the configured volume/i - ); + await expect( + assertRealPathWithinRoot(path.join(outside, 'secret.txt'), env.volumeDir) + ).rejects.toThrow(/outside the configured volume/i); }); it('accepts what is inside, existing or not', async () => { const { env, assertRealPathWithinRoot } = await withRoot('containment-direct-inside-'); await fs.mkdir(path.join(env.volumeDir, 'Documents'), { recursive: true }); - await expect(assertRealPathWithinRoot(path.join(env.volumeDir, 'Documents'), env.volumeDir)).resolves.not.toThrow(); + await expect( + assertRealPathWithinRoot(path.join(env.volumeDir, 'Documents'), env.volumeDir) + ).resolves.not.toThrow(); // A file about to be created is not an escape. - await expect(assertRealPathWithinRoot(path.join(env.volumeDir, 'Documents', 'new.txt'), env.volumeDir)).resolves.not.toThrow(); + await expect( + assertRealPathWithinRoot(path.join(env.volumeDir, 'Documents', 'new.txt'), env.volumeDir) + ).resolves.not.toThrow(); await expect(assertRealPathWithinRoot(env.volumeDir, env.volumeDir)).resolves.not.toThrow(); }); @@ -390,6 +400,8 @@ describe('the containment check on its own', () => { const { env, assertRealPathWithinRoot } = await withRoot('containment-direct-absent-'); const absent = path.join(env.tmpRoot, 'not-mounted-yet'); - await expect(assertRealPathWithinRoot(path.join(absent, 'file.txt'), absent)).resolves.not.toThrow(); + await expect( + assertRealPathWithinRoot(path.join(absent, 'file.txt'), absent) + ).resolves.not.toThrow(); }); }); diff --git a/backend/tests/utils/request-context.test.js b/backend/tests/utils/request-context.test.js new file mode 100644 index 000000000..ae30706ba --- /dev/null +++ b/backend/tests/utils/request-context.test.js @@ -0,0 +1,207 @@ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import realFs from 'node:fs'; +import { setupTestEnv } from '../helpers/env-test-utils.js'; + +/** + * Containment resolves a real path per selected item, and a bulk copy of a few + * thousand files repeats the same parent lookups. On network storage each one + * is a round-trip, so they are memoized — but only for the length of one + * request: this cache feeds a security check, and a stale answer there is not + * a stale answer anywhere. + */ + +let currentEnv; + +afterEach(async () => { + vi.restoreAllMocks(); + if (currentEnv) { + await currentEnv.cleanup(); + currentEnv = null; + } +}); + +const load = async (tag) => { + currentEnv = await setupTestEnv({ + tag, + modules: [ + 'src/config/env', + 'src/config/index', + 'src/utils/requestContext', + 'src/utils/pathUtils', + ], + }); + // Load the context first: pathUtils captures this very instance when it is + // required, and reloading it afterwards would hand the test a different + // AsyncLocalStorage than the one the cache actually uses. + const context = currentEnv.requireFresh('src/utils/requestContext'); + return { + context, + pathUtils: currentEnv.requireFresh('src/utils/pathUtils'), + env: currentEnv, + }; +}; + +describe('Per-request realpath cache', () => { + it('resolves the shared parent once instead of once per item', async () => { + const { pathUtils, context, env } = await load('request-cache-hit-'); + await fs.mkdir(path.join(env.volumeDir, 'destination'), { recursive: true }); + + const resolveTwentyTargets = async () => { + for (let i = 0; i < 20; i += 1) { + await pathUtils.resolveVolumePath(`destination/new-${i}.txt`); + } + }; + + // Files a copy is about to create: each one fails to resolve and falls back + // to its parent, which is the same directory twenty times over. + // + // The spy is on `fs/promises`, which is what containment calls now: these + // lookups run on every path a request touches, and on network storage each + // synchronous one blocked the only thread the server has. + const withoutCache = vi.spyOn(fs, 'realpath'); + await resolveTwentyTargets(); + const uncached = withoutCache.mock.calls.length; + withoutCache.mockRestore(); + + const withCache = vi.spyOn(fs, 'realpath'); + await context.runInRequestContext(resolveTwentyTargets); + const cached = withCache.mock.calls.length; + + // 20 misses (never cached: the file may have just been created) plus one + // lookup for the parent they share, instead of one parent lookup each. + // + // The volume root does not appear in either count. It is resolved once for + // the life of the process and held in its own cache, and it is the one + // lookup here that is still synchronous — a single call at startup rather + // than one per path, which is why it was left alone. + expect(uncached).toBe(40); + expect(cached).toBe(21); + }); + + it('does not carry answers over to the next request', async () => { + const { pathUtils, context, env } = await load('request-cache-scope-'); + await fs.mkdir(path.join(env.volumeDir, 'docs'), { recursive: true }); + await fs.writeFile(path.join(env.volumeDir, 'docs', 'a.txt'), 'x'); + + await context.runInRequestContext(() => pathUtils.resolveVolumePath('docs/a.txt')); + + const spy = vi.spyOn(fs, 'realpath'); + await context.runInRequestContext(() => pathUtils.resolveVolumePath('docs/a.txt')); + + // A fresh request asks the filesystem again: between two requests the + // directory may have become a symbolic link somewhere else. + expect(spy.mock.calls.length).toBeGreaterThan(0); + }); + + it('still refuses an escape when the cache is warm', async () => { + const { pathUtils, context, env } = await load('request-cache-escape-'); + const outside = path.join(env.tmpRoot, 'outside'); + await fs.mkdir(outside, { recursive: true }); + await fs.writeFile(path.join(outside, 'secret.txt'), 'not yours'); + await fs.mkdir(path.join(env.volumeDir, 'ok'), { recursive: true }); + await fs.symlink(outside, path.join(env.volumeDir, 'escape')); + + await context.runInRequestContext(async () => { + expect(await pathUtils.resolveVolumePath('ok')).toContain('ok'); + await expect(pathUtils.resolveVolumePath('escape/secret.txt')).rejects.toThrow(/outside/i); + // And again, now that the cache holds an answer for it. + await expect(pathUtils.resolveVolumePath('escape/secret.txt')).rejects.toThrow(/outside/i); + }); + }); + + it('works outside a request context', async () => { + const { pathUtils, env } = await load('request-cache-none-'); + await fs.mkdir(path.join(env.volumeDir, 'plain'), { recursive: true }); + + // Startup code and background jobs run with no request around them. + expect(await pathUtils.resolveVolumePath('plain')).toContain('plain'); + }); +}); + +/** + * The access rules are consulted for every path, so a bulk operation asked for + * the settings thousands of times over — several queries and a JSON parse each + * time, to re-read values that cannot change during one request. + */ +describe('Settings read once per request', () => { + it('answers repeated callers from one read', async () => { + const { context, env } = await load('settings-per-request-'); + const settings = env.requireFresh('src/services/settingsService'); + + const reads = await context.runInRequestContext(async () => { + const first = await settings.getSettings(); + const rest = await Promise.all(Array.from({ length: 49 }, () => settings.getSettings())); + return [first, ...rest]; + }); + + // The same object throughout: one read, shared. Counting queries would + // prove nothing, since prepared statements are cached either way. + reads.forEach((value) => expect(value).toBe(reads[0])); + }); + + it('reads again on the next request', async () => { + const { context, env } = await load('settings-next-request-'); + const settings = env.requireFresh('src/services/settingsService'); + + const first = await context.runInRequestContext(() => settings.getSettings()); + const second = await context.runInRequestContext(() => settings.getSettings()); + + // Two requests, two reads: a change between them has to be visible. + expect(second).not.toBe(first); + expect(second).toEqual(first); + }); +}); + +/** + * The property this conversion exists for, asserted directly. + * + * Containment runs on every path a request touches, and a bulk operation + * resolves one per selected item — up to thirty-two hops each when links are + * chased. Synchronously, on the network mount most deployments point at, every + * one of those was a round trip during which the only thread the server has + * served nothing: not another request, not the liveness probe, not the response + * already half written. Nothing stops a synchronous call being reintroduced by + * someone who has not read that paragraph, except this. + */ +describe('what containment does to the event loop', () => { + it('resolves a path without a single synchronous filesystem call', async () => { + const { pathUtils, context, env } = await load('containment-async-'); + await fs.mkdir(path.join(env.volumeDir, 'docs', 'deep'), { recursive: true }); + await fs.writeFile(path.join(env.volumeDir, 'docs', 'deep', 'a.txt'), 'x'); + + // Resolve once first: the volume root is looked up synchronously exactly + // once for the life of the process and then held, and that one call is + // deliberate — a few at startup rather than one per path. + await pathUtils.resolveVolumePath('docs/deep/a.txt'); + + const realpathSync = vi.spyOn(realFs, 'realpathSync'); + const lstatSync = vi.spyOn(realFs, 'lstatSync'); + + // Inside a request, which is the only place the shortcut runs — it is the + // one that asks for an lstat, so resolving outside a request would leave + // that call unexercised and this test asserting nothing about it. + await context.runInRequestContext(async () => { + await pathUtils.resolveVolumePath('docs/deep/a.txt'); + await pathUtils.resolveVolumePath('docs/deep/not-created-yet.txt'); + }); + + expect(realpathSync).not.toHaveBeenCalled(); + expect(lstatSync).not.toHaveBeenCalled(); + }); + + // A broken link is the only thing that reaches `readlink`: a valid one is + // resolved by realpath and never gets there. + it('follows a broken link asynchronously, and still refuses it', async () => { + const { pathUtils, env } = await load('containment-async-broken-'); + await fs.symlink(path.join(env.tmpRoot, 'nowhere'), path.join(env.volumeDir, 'dead')); + await pathUtils.resolveVolumePath('docs').catch(() => {}); + + const readlinkSync = vi.spyOn(realFs, 'readlinkSync'); + + await expect(pathUtils.resolveVolumePath('dead')).rejects.toThrow(/outside/i); + + expect(readlinkSync).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/tests/utils/totp.test.js b/backend/tests/utils/totp.test.js new file mode 100644 index 000000000..395b6c61a --- /dev/null +++ b/backend/tests/utils/totp.test.js @@ -0,0 +1,125 @@ +import { describe, it, expect } from 'vitest'; + +const { encodeBase32, decodeBase32 } = require('../../src/utils/base32'); +const { generateSecret, totpCode, verifyTotp, otpauthUri } = require('../../src/utils/totp'); + +/** + * The codes an authenticator app shows. + * + * Checked against RFC 6238's own test vectors rather than against itself: the + * whole point of implementing this is that it agrees with every phone on the + * planet, and only the published answers can say so. + */ + +/** The RFC's secret: the ASCII digits 1234567890 twice, as base32. */ +const RFC_SECRET = encodeBase32(Buffer.from('12345678901234567890', 'ascii')); + +describe('base32', () => { + /** RFC 4648's own vectors. */ + it.each([ + ['', ''], + ['f', 'MY'], + ['fo', 'MZXQ'], + ['foo', 'MZXW6'], + ['foob', 'MZXW6YQ'], + ['fooba', 'MZXW6YTB'], + ['foobar', 'MZXW6YTBOI'], + ])('encodes %o', (plain, encoded) => { + expect(encodeBase32(Buffer.from(plain, 'ascii'))).toBe(encoded); + }); + + it('reads back what it wrote', () => { + expect(decodeBase32('MZXW6YTBOI').toString('ascii')).toBe('foobar'); + }); + + /** How a secret arrives when somebody reads it off a screen. */ + it('forgives spaces, lower case and padding', () => { + expect(decodeBase32('mzxw 6ytb oi==').toString('ascii')).toBe('foobar'); + }); + + it('refuses what is not base32 rather than decoding it wrong', () => { + expect(() => decodeBase32('MZXW6YTB01')).toThrow(); + expect(() => decodeBase32('')).toThrow(); + }); +}); + +describe('the codes RFC 6238 publishes', () => { + /** + * The RFC prints eight digits; an authenticator app shows the last six of + * the same number, which is what this produces. + */ + it.each([ + [59, '287082'], + [1111111109, '081804'], + [1111111111, '050471'], + [1234567890, '005924'], + [2000000000, '279037'], + [20000000000, '353130'], + ])('at T=%i is %s', (seconds, expected) => { + expect(totpCode(RFC_SECRET, { at: seconds * 1000 })).toBe(expected); + }); +}); + +describe('checking a code', () => { + const at = 1111111109 * 1000; + + it('accepts the code of the moment, and says which step it was', () => { + const step = verifyTotp(RFC_SECRET, '081804', { at }); + expect(step).toBe(Math.floor(1111111109 / 30)); + }); + + /** A phone whose clock is half a minute out is still that person's phone. */ + it('accepts one step either side', () => { + expect(verifyTotp(RFC_SECRET, totpCode(RFC_SECRET, { at: at - 30000 }), { at })).not.toBeNull(); + expect(verifyTotp(RFC_SECRET, totpCode(RFC_SECRET, { at: at + 30000 }), { at })).not.toBeNull(); + }); + + it('refuses two steps away', () => { + expect(verifyTotp(RFC_SECRET, totpCode(RFC_SECRET, { at: at - 90000 }), { at })).toBeNull(); + expect(verifyTotp(RFC_SECRET, totpCode(RFC_SECRET, { at: at + 90000 }), { at })).toBeNull(); + }); + + /** + * What stops a code being used twice: the account remembers the last step it + * let through, and the same digits offered again belong to that step. + */ + it('refuses a step already used, however right the digits are', () => { + const step = verifyTotp(RFC_SECRET, '081804', { at }); + expect(verifyTotp(RFC_SECRET, '081804', { at, after: step })).toBeNull(); + expect(verifyTotp(RFC_SECRET, '081804', { at, after: step - 1 })).toBe(step); + }); + + it('refuses anything that is not six digits', () => { + for (const code of ['', '12345', '1234567', 'abcdef', '0818o4', null, undefined]) { + expect(verifyTotp(RFC_SECRET, code, { at })).toBeNull(); + } + }); + + /** Apps show the digits in two groups of three, and people type what they see. */ + it('reads a code typed with the spaces the app shows', () => { + expect(verifyTotp(RFC_SECRET, '081 804', { at })).not.toBeNull(); + }); + + it('refuses rather than throwing when the secret itself is unusable', () => { + expect(verifyTotp('not base32!', '081804', { at })).toBeNull(); + }); +}); + +describe('what the phone is handed', () => { + it('draws a secret of the length the RFC uses', () => { + const secret = generateSecret(); + expect(decodeBase32(secret)).toHaveLength(20); + expect(secret).not.toBe(generateSecret()); + }); + + it('names the account and the issuer, as every app expects', () => { + const uri = new URL(otpauthUri({ secret: RFC_SECRET, account: 'someone@example.com' })); + + expect(uri.protocol).toBe('otpauth:'); + expect(decodeURIComponent(uri.pathname)).toBe('/NextExplorer:someone@example.com'); + expect(uri.searchParams.get('secret')).toBe(RFC_SECRET); + expect(uri.searchParams.get('issuer')).toBe('NextExplorer'); + expect(uri.searchParams.get('digits')).toBe('6'); + expect(uri.searchParams.get('period')).toBe('30'); + }); +}); diff --git a/docker/check-heic.cjs b/docker/check-heic.cjs new file mode 100644 index 000000000..74617a5ec --- /dev/null +++ b/docker/check-heic.cjs @@ -0,0 +1,69 @@ +// Decode a HEIC the way the thumbnail service does — ffmpeg to PNG, then sharp +// — and print the colour near each side, as "R G B R G B", left then right. +// +// .github/workflows/build-image.yml runs this inside each built image, so the +// ffmpeg asked is the one that image ships. The lean image builds ffmpeg with +// nearly every encoder removed; PNG is kept because the service needs it, and +// that is why this goes through PNG rather than a raw pixel dump, which the +// lean ffmpeg cannot write at all. +// +// Usage: node check-heic.cjs [video filter] +// The filter defaults to the service's reduction; another one (hflip, a crop) +// is how the check was shown to catch a flipped or partial decode. +const { spawn } = require('node:child_process'); +const path = require('node:path'); +const { createRequire } = require('node:module'); + +// sharp is resolved from the working directory, /app in the image, rather than +// from wherever this file happens to be mounted. +const sharp = createRequire(path.join(process.cwd(), 'index.js'))('sharp'); + +const [source, filter = 'scale=64:-1:flags=lanczos'] = process.argv.slice(2); +if (!source) { + console.error('usage: node check-heic.cjs [video filter]'); + process.exit(2); +} + +const ffmpeg = spawn( + 'ffmpeg', + [ + '-hide_banner', + '-loglevel', + 'error', + '-i', + source, + '-map', + '0:v:0', + '-frames:v', + '1', + '-vf', + filter, + '-vcodec', + 'png', + '-f', + 'image2pipe', + 'pipe:1', + ], + { stdio: ['ignore', 'pipe', 'inherit'] } +); + +const chunks = []; +ffmpeg.stdout.on('data', (chunk) => chunks.push(chunk)); +ffmpeg.on('error', (error) => { + console.error(`ffmpeg could not be started: ${error.message}`); + process.exit(1); +}); +ffmpeg.on('close', async (code) => { + if (code !== 0) { + console.error(`ffmpeg exited with ${code}`); + process.exit(1); + } + const { data, info } = await sharp(Buffer.concat(chunks)) + .raw() + .toBuffer({ resolveWithObject: true }); + const at = (x) => { + const i = (Math.floor(info.height / 2) * info.width + x) * info.channels; + return [data[i], data[i + 1], data[i + 2]]; + }; + console.log([...at(6), ...at(info.width - 6)].join(' ')); +}); diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index d47fd519c..5b343dfd9 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,7 +1,11 @@ #!/bin/bash set -e -# Provide sensible defaults to avoid surprises on first run. +# Provide sensible defaults to avoid surprises on first run. Whether they were +# asked for is kept, so the message below can tell somebody their setting is +# being ignored without saying it to everybody who never wrote one. +PUID_REQUESTED=${PUID:+yes} +PGID_REQUESTED=${PGID:+yes} PUID=${PUID:-1000} PGID=${PGID:-1000} @@ -12,38 +16,72 @@ ensure_dir() { mkdir -p "$1" } -# Ensure the base appuser exists before attempting modifications. -if ! id appuser >/dev/null 2>&1; then - echo "ERROR: Expected user 'appuser' to be present in the image." - exit 1 -fi +# Who this script is running as, which decides everything below. +# +# Started as root — the default, and what Compose's `user: root` also gives — +# the entrypoint renumbers appuser to PUID:PGID, takes ownership of the +# directories that map to host volumes, and drops to it. +# +# Started as anyone else (`docker run --user`, Compose `user: 1000:1000`, a +# Kubernetes securityContext with runAsNonRoot) none of that is possible: +# groupmod, usermod and chown all need root, and with `set -e` the first one +# would take the container with it before the server ever started. It is not +# needed either — the process is already the user it was asked to be. So the +# id juggling is skipped and the application runs as whoever started it. +STARTED_AS_UID=$(id -u) +STARTED_AS_GID=$(id -g) +RUNNING_AS_ROOT=false +[ "$STARTED_AS_UID" = "0" ] && RUNNING_AS_ROOT=true + +if [ "$RUNNING_AS_ROOT" = "true" ]; then + # Ensure the base appuser exists before attempting modifications. + if ! id appuser >/dev/null 2>&1; then + echo "ERROR: Expected user 'appuser' to be present in the image." + exit 1 + fi -CURRENT_UID=$(id -u appuser) -CURRENT_GID=$(id -g appuser) + CURRENT_UID=$(id -u appuser) + CURRENT_GID=$(id -g appuser) -# Update user/group IDs only when they differ from the requested values. -if [ "$CURRENT_UID" != "$PUID" ] || [ "$CURRENT_GID" != "$PGID" ]; then - echo "INFO: Updating appuser UID:GID from ${CURRENT_UID}:${CURRENT_GID} to ${PUID}:${PGID}" - groupmod -o -g "$PGID" appuser - usermod -o -u "$PUID" appuser + # Update user/group IDs only when they differ from the requested values. + if [ "$CURRENT_UID" != "$PUID" ] || [ "$CURRENT_GID" != "$PGID" ]; then + echo "INFO: Updating appuser UID:GID from ${CURRENT_UID}:${CURRENT_GID} to ${PUID}:${PGID}" + groupmod -o -g "$PGID" appuser + usermod -o -u "$PUID" appuser + fi +elif [ -n "$PUID_REQUESTED" ] || [ -n "$PGID_REQUESTED" ]; then + echo "INFO: PUID/PGID (${PUID}:${PGID}) ignored: the container was started as ${STARTED_AS_UID}:${STARTED_AS_GID}, which is already what the process runs as." fi # Guarantee every host-facing directory exists before touching it. -ensure_dir "/app" -ensure_dir "$CONFIG_DIR" -ensure_dir "$CACHE_DIR" -ensure_dir "${CACHE_DIR}/thumbnails" -ensure_dir "${CONFIG_DIR}/extensions" -ensure_dir "${CONFIG_DIR}/extensions/icons" -ensure_dir "${CONFIG_DIR}/extensions/brand" - -# Fix ownership on key directories that map to host volumes. -for path in "$CONFIG_DIR" "$CACHE_DIR"; do - if [ -e "$path" ]; then - chown -R appuser:appuser "$path" +# +# Advisory when we are not root: a mount whose permissions do not allow it is +# the deployment's business, and the application says so far better than a +# shell abort with no message does. +for path in \ + "/app" \ + "$CONFIG_DIR" \ + "$CACHE_DIR" \ + "${CACHE_DIR}/thumbnails"; do + if [ "$RUNNING_AS_ROOT" = "true" ]; then + ensure_dir "$path" + elif ! ensure_dir "$path" 2>/dev/null; then + echo "WARN: could not create ${path} as ${STARTED_AS_UID}:${STARTED_AS_GID}; the mount must already provide it" fi done +# Fix ownership on key directories that map to host volumes. Only root can, and +# only root needs to: started as a fixed user, the deployment has already +# decided who owns these — a Kubernetes fsGroup does under a cluster what +# PUID/PGID do under Docker. +if [ "$RUNNING_AS_ROOT" = "true" ]; then + for path in "$CONFIG_DIR" "$CACHE_DIR"; do + if [ -e "$path" ]; then + chown -R appuser:appuser "$path" + fi + done +fi + is_true() { case "${1:-}" in 1|true|TRUE|yes|YES|on|ON) return 0 ;; @@ -52,13 +90,20 @@ is_true() { } DEMO_MODE="${DEMO_MODE:-false}" +# Demo mode also pre-fills the sign-in form, which a demo may want without +# paying for the sample archive on every boot: it is 80 MB, and where storage is +# not persistent that download happens at every restart. Defaults to on, so +# existing demos are unaffected. +DEMO_SAMPLES="${DEMO_SAMPLES:-true}" SAMPLE_URL="${SAMPLE_URL:-https://github.com/vikramsoni2/nextExplorer/releases/download/v2.0.0/samples.zip}" SAMPLES_DIR="${SAMPLES_DIR:-/mnt/Samples}" -if is_true "$DEMO_MODE"; then +if is_true "$DEMO_MODE" && is_true "$DEMO_SAMPLES"; then echo "INFO: DEMO_MODE enabled; seeding demo samples into ${SAMPLES_DIR} (read-only)" - mkdir -p "$SAMPLES_DIR" + if ! mkdir -p "$SAMPLES_DIR" 2>/dev/null; then + echo "WARN: could not create ${SAMPLES_DIR}; continuing without seeded samples" + fi if ! SAMPLE_URL="$SAMPLE_URL" SAMPLES_DIR="$SAMPLES_DIR" node /app/src/scripts/downloadSamples.js; then echo "WARN: DEMO_MODE sample download failed; continuing without seeded samples" @@ -66,5 +111,10 @@ if is_true "$DEMO_MODE"; then fi -echo "INFO: Launching process as appuser (${PUID}:${PGID})" -exec gosu appuser "$@" +if [ "$RUNNING_AS_ROOT" = "true" ]; then + echo "INFO: Launching process as appuser (${PUID}:${PGID})" + exec gosu appuser "$@" +fi + +echo "INFO: Launching process as ${STARTED_AS_UID}:${STARTED_AS_GID}, the user the container was started as" +exec "$@" diff --git a/docker/healthcheck.js b/docker/healthcheck.js index f8e85fff0..fc0e0b013 100644 --- a/docker/healthcheck.js +++ b/docker/healthcheck.js @@ -1,24 +1,46 @@ -/* eslint-env node */ const http = require('http'); -const options = { - host: 'localhost', - port: process.env.PORT || 3000, - timeout: 2000, - path: '/healthz', -}; +/** + * What Docker runs to decide whether this container is healthy. + * + * Two things it has to do that the first version did not. `timeout` on + * `http.request` arms the socket but aborts nothing on its own — without a + * listener the script simply waits, and a server that accepts the connection + * and never answers turns into Docker's own ten-second timeout with no output + * at all. And a non-200 deserves to say what it was: `/healthz` sits behind the + * authentication middleware, so a redirect to an identity provider is a + * plausible failure and looks nothing like a crash. + */ +const TIMEOUT_MS = 2000; -const request = http.request(options, (res) => { - console.log(`STATUS: ${res.statusCode}`); - if (res.statusCode === 200) { - process.exit(0); - } else { +const request = http.request( + { + host: '127.0.0.1', + port: process.env.PORT || 3000, + path: '/healthz', + timeout: TIMEOUT_MS, + }, + (response) => { + if (response.statusCode === 200) { + response.resume(); + process.exit(0); + } + + const location = response.headers.location ? ` -> ${response.headers.location}` : ''; + console.log(`UNHEALTHY: /healthz answered ${response.statusCode}${location}`); + response.resume(); process.exit(1); } +); + +request.on('timeout', () => { + console.log(`UNHEALTHY: /healthz did not answer within ${TIMEOUT_MS} ms`); + request.destroy(); + process.exit(1); }); -request.on('error', (err) => { - console.log(`ERROR: ${err.message}`); +request.on('error', (error) => { + console.log(`UNHEALTHY: ${error.message}`); process.exit(1); }); diff --git a/docker/verify-7zip-password.js b/docker/verify-7zip-password.js new file mode 100644 index 000000000..184b9cf5e --- /dev/null +++ b/docker/verify-7zip-password.js @@ -0,0 +1,37 @@ +const pty = require('@homebridge/node-pty-prebuilt-multiarch'); + +const [archivePath, outputPath, password] = process.argv.slice(2); +if (!archivePath || !outputPath || password === undefined) { + process.exitCode = 2; + throw new Error('Expected archive path, output path, and password.'); +} + +const child = pty.spawn('7z', ['x', '-y', `-o${outputPath}`, archivePath], { + name: 'xterm-256color', + cols: 120, + rows: 40, + env: { ...process.env, TERM: 'xterm-256color' }, +}); + +let output = ''; +let passwordWritten = false; +const timeout = setTimeout(() => { + child.kill('SIGKILL'); + throw new Error('Timed out waiting for 7-Zip password prompt.'); +}, 10_000); + +child.onData((chunk) => { + output = `${output}${chunk}`.slice(-2000); + if (!passwordWritten && /enter password/i.test(output)) { + passwordWritten = true; + child.write(`${password}\r`); + } +}); + +child.onExit(({ exitCode }) => { + clearTimeout(timeout); + if (exitCode !== 0 || !passwordWritten) { + process.stderr.write(output); + process.exit(exitCode || 1); + } +}); diff --git a/docker/verify-ffmpeg.sh b/docker/verify-ffmpeg.sh new file mode 100755 index 000000000..5df55cf00 --- /dev/null +++ b/docker/verify-ffmpeg.sh @@ -0,0 +1,195 @@ +#!/bin/sh +# Prove a freshly built ffmpeg can decode everything this application offers. +# +# The build strips ffmpeg down, and the way that goes wrong is silent: a +# container or codec quietly stops being decodable and thumbnails for that +# format simply never appear. Nothing errors, nothing logs, a folder of .wmv +# files just looks empty of previews. +# +# So the build proves it instead of assuming it. Alpine's own ffmpeg — a build +# dependency, never shipped — synthesises one short clip per format, and the +# binary we just built has to get a frame out of each. Any failure fails the +# image build, which is the only place this can be caught before a user is. +# +# Usage: verify-ffmpeg.sh +set -eu + +OURS="$1" +OURS_PROBE="$2" +SYSTEM_FFMPEG="${SYSTEM_FFMPEG:-ffmpeg}" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +failures=0 +checked=0 + +note() { printf ' %s\n' "$1"; } + +# One second of colour bars, encoded as asked. Small on purpose: this proves a +# decoder runs, not that it is fast. +# +# 25 fps rather than something arbitrary: MPEG-1 and MPEG-2 accept only the +# handful of rates their specifications name and refuse to encode at anything +# else. A fixture that will not encode is reported as a skip, so getting this +# wrong drops a format out of the check instead of failing it — which is how +# `.mpeg` went unverified on the first run. +make_video() { + file="$1" + shift + "$SYSTEM_FFMPEG" -hide_banner -loglevel error -y \ + -f lavfi -i "testsrc=size=64x64:rate=25:duration=1" \ + "$@" "$file" 2>"$WORK/encode.log" +} + +make_audio() { + file="$1" + shift + "$SYSTEM_FFMPEG" -hide_banner -loglevel error -y \ + -f lavfi -i "sine=frequency=440:duration=1" \ + "$@" "$file" 2>"$WORK/encode.log" +} + +# The thumbnail path, exactly as thumbnailService runs it: seek, one frame, +# scale, out through a pipe. +decode_one_frame() { + file="$1" + out="$WORK/frame.jpg" + rm -f "$out" + "$OURS" -hide_banner -loglevel error -y -i "$file" \ + -map 0:v:0 -frames:v 1 -vf "scale=32:-1" -vcodec mjpeg -f image2 "$out" \ + 2>"$WORK/decode.log" || return 1 + [ -s "$out" ] || return 1 +} + +check_video() { + label="$1" + file="$WORK/$2" + shift 2 + + checked=$((checked + 1)) + if ! make_video "$file" "$@"; then + note "SKIP $label — this Alpine ffmpeg cannot produce the fixture" + checked=$((checked - 1)) + return 0 + fi + if decode_one_frame "$file"; then + note "ok $label" + else + note "FAIL $label" + sed 's/^/ /' "$WORK/decode.log" || true + failures=$((failures + 1)) + fi +} + +# Audio is read for metadata and album art rather than decoded to a picture, so +# the bar is that ffprobe names the stream. +check_audio() { + label="$1" + file="$WORK/$2" + shift 2 + + checked=$((checked + 1)) + if ! make_audio "$file" "$@"; then + note "SKIP $label — this Alpine ffmpeg cannot produce the fixture" + checked=$((checked - 1)) + return 0 + fi + if "$OURS_PROBE" -hide_banner -loglevel error \ + -show_entries stream=codec_name -of default=nw=1:nk=1 "$file" \ + 2>"$WORK/probe.log" | grep -q .; then + note "ok $label" + else + note "FAIL $label" + sed 's/^/ /' "$WORK/probe.log" || true + failures=$((failures + 1)) + fi +} + +echo "ffmpeg build check — $($OURS -version 2>/dev/null | head -n1)" + +echo "video containers and codecs the explorer offers previews for:" +check_video "mp4 / h264" clip.mp4 -c:v libx264 -pix_fmt yuv420p +check_video "mp4 / hevc" hevc.mp4 -c:v libx265 -pix_fmt yuv420p -tag:v hvc1 +check_video "mov / h264" clip.mov -c:v libx264 -pix_fmt yuv420p +check_video "m4v / h264" clip.m4v -c:v libx264 -pix_fmt yuv420p +check_video "mkv / vp9" clip.mkv -c:v libvpx-vp9 +check_video "webm / vp8" clip.webm -c:v libvpx +check_video "mp4 / av1" av1.mp4 -c:v libaom-av1 -cpu-used 8 +check_video "avi / mpeg4" clip.avi -c:v mpeg4 +check_video "wmv / wmv2" clip.wmv -c:v wmv2 +check_video "flv / flv1" clip.flv -c:v flv +check_video "mpg / mpeg2" clip.mpg -c:v mpeg2video +check_video "mpeg / mpeg1" clip.mpeg -c:v mpeg1video +check_video "mov / prores" prores.mov -c:v prores_ks -pix_fmt yuv422p10le +check_video "mjpeg in avi" mjpeg.avi -c:v mjpeg -pix_fmt yuvj420p + +echo "audio, read for metadata and album art:" +check_audio "mp3" tone.mp3 -c:a libmp3lame +check_audio "flac" tone.flac -c:a flac +check_audio "wav" tone.wav -c:a pcm_s16le +check_audio "m4a" tone.m4a -c:a aac +check_audio "ogg" tone.ogg -c:a libvorbis +check_audio "opus" tone.opus -c:a libopus +check_audio "wma" tone.wma -c:a wmav2 + +# A HEIC is HEVC inside an ISOBMFF container with an `heic` brand, but a +# working hevc/mp4 above does not stand for it: reading one takes the HEIF +# demuxer, which ffmpeg only has from 7.1, and an older build decodes the mp4 +# and cannot open the HEIC. It is checked on the finished image instead, for +# both variants, in .github/workflows/build-image.yml. +# A subtitle track has to come out as WebVTT, because that is the only subtitle +# format a browser will display. This exercises the two halves separately: a +# file that is nothing but subtitles proves the demuxer and the WebVTT muxer, +# and a track pulled out of a Matroska proves the mapping the route actually +# uses. Both are new enough to the build that a missing muxer would otherwise +# only show up as a caption menu that is silently empty. +check_subtitle() { + label="$1" + source="$2" + shift 2 + + checked=$((checked + 1)) + out="$WORK/out.vtt" + rm -f "$out" + if ! "$OURS" -hide_banner -loglevel error -y -i "$source" "$@" -f webvtt "$out" \ + 2>"$WORK/vtt.log"; then + note "FAIL $label" + sed 's/^/ /' "$WORK/vtt.log" || true + failures=$((failures + 1)) + return 0 + fi + # Present is not enough: an empty document is what a muxer that ran but wrote + # nothing leaves behind, and a caption track like that shows no words. + if grep -q "WEBVTT" "$out" && grep -q "a subtitle line" "$out"; then + note "ok $label" + else + note "FAIL $label — output was not usable WebVTT" + sed 's/^/ /' "$out" || true + failures=$((failures + 1)) + fi +} + +echo "subtitles, converted to the one format a browser displays:" +printf '1\n00:00:00,500 --> 00:00:02,000\na subtitle line\n' > "$WORK/subs.srt" +check_subtitle "srt to webvtt" "$WORK/subs.srt" + +if "$SYSTEM_FFMPEG" -hide_banner -loglevel error -y \ + -f lavfi -i "testsrc=size=64x64:rate=25:duration=2" -i "$WORK/subs.srt" \ + -map 0:v -map 1:s -c:v libx264 -pix_fmt yuv420p -c:s srt \ + "$WORK/subbed.mkv" 2>"$WORK/encode.log"; then + check_subtitle "mkv subtitle track to webvtt" "$WORK/subbed.mkv" -map 0:s:0 +else + note "SKIP mkv subtitle track — this Alpine ffmpeg cannot produce the fixture" +fi + +echo "still images:" +check_video "png" still.png -frames:v 1 -c:v png +check_video "jpeg" still.jpg -frames:v 1 -c:v mjpeg -pix_fmt yuvj420p + +echo +if [ "$failures" -gt 0 ]; then + echo "ffmpeg build check FAILED: $failures of $checked formats could not be decoded" >&2 + exit 1 +fi +echo "ffmpeg build check passed: $checked formats decoded" diff --git a/docs/admin/user-volumes.md b/docs/admin/user-volumes.md index 64f4d70b4..6238221b3 100644 --- a/docs/admin/user-volumes.md +++ b/docs/admin/user-volumes.md @@ -29,12 +29,14 @@ Each assignment creates a top-level entry in the user’s sidebar using the assi Make sure the directories you want to expose exist inside the container and are readable/writable by the container user as appropriate. Typical pattern: + - `VOLUME_ROOT=/mnt` - Mount team folders as subdirectories under `/mnt` (e.g., `/mnt/Projects`, `/mnt/Media`, `/mnt/Finance`) ### 2) Create or pick a user profile Go to **Settings → Admin → Users**, then: + - Select an existing user profile, or - Create a new local user profile (so you can pre-assign volumes before their first login) diff --git a/docs/integrations/onlyoffice.md b/docs/integrations/onlyoffice.md index 6178674ab..af8374017 100644 --- a/docs/integrations/onlyoffice.md +++ b/docs/integrations/onlyoffice.md @@ -4,28 +4,60 @@ Use ONLYOFFICE Document Server to edit office files (DOCX, XLSX, PPTX, ODT, ODS, ## Environment variables -| Variable | Required? | Description | -| ---------------------------- | ----------------- | ----------------------------------------------------------------------------------------- | -| `ONLYOFFICE_URL` | Yes | Public URL of your Document Server (e.g., `https://office.example.com`). | -| `PUBLIC_URL` | Yes | nextExplorer’s public URL so ONLYOFFICE knows where to download files and post callbacks. | -| `ONLYOFFICE_SECRET` | Yes | JWT secret shared between nextExplorer and ONLYOFFICE for signing requests/responses. | -| `ONLYOFFICE_LANG` | No (default `en`) | Language code for the editor UI. | -| `ONLYOFFICE_FORCE_SAVE` | No | When true, users must use the editor’s Save button rather than relying on autosave. | -| `ONLYOFFICE_FILE_EXTENSIONS` | No | Comma-separated list of extensions you want to surface beyond the defaults. | - -Without `ONLYOFFICE_SECRET`, a secret derived from the session secret is used — -never the session secret itself, which signs every session cookie — and the log -says so at start. Set `ONLYOFFICE_SECRET` on both sides. +| Variable | Required? | Description | +| ---------------------------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ONLYOFFICE_URL` | Yes | Public URL of your Document Server (e.g., `https://office.example.com`). | +| `PUBLIC_URL` | Yes | nextExplorer’s public URL so ONLYOFFICE knows where to download files and post callbacks. | +| `ONLYOFFICE_SECRET` | Yes | JWT secret shared between nextExplorer and ONLYOFFICE for signing requests/responses. | +| `ONLYOFFICE_LANG` | No (default `en`) | Language code for the editor UI. | +| `ONLYOFFICE_FORCE_SAVE` | No | When true, the editor Save button immediately writes the current version through the callback. | +| `ONLYOFFICE_AUTO_SAVE_INTERVAL_MS` | No (default `30000`) | Minimum delay between background force-saves of a changed document. Set `0` to save only on close. Values are clamped between `0` and `300000`. | +| `ONLYOFFICE_FORCE_SAVE_TIMEOUT_MS` | No (default `10000`) | Retry window in milliseconds for a force-save when Document Server is still receiving the final changes. Minimum `7000`; closing remains immediate. | +| `ONLYOFFICE_FILE_EXTENSIONS` | No | Comma-separated list of extensions you want to surface beyond the defaults. | ## How it works +During editing, ONLYOFFICE synchronizes changes with its Document Server first. +That internal synchronization does not rewrite the source file mounted in +NextExplorer on every keystroke. The source file is replaced only when +Document Server calls the storage callback: normally after the last editor +closes, or after a force-save. NextExplorer schedules a bounded force-save +after ONLYOFFICE confirms it has received changes, then retries briefly if the +final changes are still arriving. This keeps a usable current version on the +mounted storage while avoiding a full document conversion for every keystroke. +The replacement remains atomic: the existing document is kept if the updated +version cannot be downloaded completely. + 1. Opening a compatible file triggers a call to `/api/onlyoffice/config`, which returns editor configuration and a signed `config.token` when `ONLYOFFICE_SECRET` is set. 2. ONLYOFFICE fetches the file through `/api/onlyoffice/file?path=...` with an `Authorization: Bearer ` header. -3. After editing, ONLYOFFICE posts to `/api/onlyoffice/callback?path=...`, again authorized with the token; nextExplorer saves the changes automatically. +3. When ONLYOFFICE has delivered changes to Document Server, nextExplorer asks it to force-save at most once per `ONLYOFFICE_AUTO_SAVE_INTERVAL_MS`. When the preview closes, nextExplorer waits only for its own API to accept a final request, never for the longer document conversion and callback. The normal delayed close callback remains a fallback. + +## Editing activity and co-editing + +NextExplorer shows a pencil indicator beside a document when it is open in ONLYOFFICE. The state is deliberately advisory: it is not a filesystem lock. Copying, moving, renaming, or deleting an active document remains possible, but the explorer displays a warning before the operation continues. + +The indicator uses the local editor session and ONLYOFFICE callback status updates. It expires automatically when a browser or Document Server disappears, so stale state can never block work. + +Co-editing is native to ONLYOFFICE. Two people simply open the same file through NextExplorer with write permission; ONLYOFFICE recognizes the shared document key and opens its normal collaborative session. No separate co-edit link, shared session, or additional NextExplorer setting is required. + +The document key is what decides this, and it stays stable for as long as anyone has the document open — two different keys would be two independent sessions on one file, invisible to each other, where whoever saved last would overwrite the other. The key changes once the document is released, so a later reader is served the saved file rather than a cached copy. A document renamed from the editor keeps its session, and anyone opening it under the new name joins the one already running. + +## What the editor can do + +Beyond editing, the toolbar reaches back into NextExplorer: + +- **Close**, drawn by the Document Server itself, which force-saves on the way out rather than leaving the last changes to a delayed callback. +- **Rename** the open document, and **Save as** under a new name — both keep the running session, so co-editors are not dropped. A name already taken, even by a file that arrives while the copy downloads, gets “(1)”. +- **Share** the document without leaving it, through the usual share dialog. +- **Mentions**: typing `@` in a comment offers the users who can already reach the document. +- **Compare** against another document, and **insert** an image, spreadsheet or presentation picked from your own storage. Each is handed to the Document Server as a short-lived read-only URL for that one file. +- The editor follows the app's light or dark theme, and reloads the document when it has moved on. + +**New file** offers blank Word, Excel and PowerPoint documents from a drawer beside it; the new document opens straight in the editor. ## Security notes -- Tokens are signed with HS256 using `ONLYOFFICE_SECRET`. Keep this secret in sync with the Document Server’s `services.CoAuthoring.secret` (`local.json`). +- Tokens are signed with HS256 using `ONLYOFFICE_SECRET`. Keep this secret in sync with the Document Server’s `services.CoAuthoring.secret` (`local.json`). It can be supplied as `ONLYOFFICE_SECRET_FILE` instead, which keeps it out of `docker inspect` — see [Secrets](/configuration/environment#secrets). - To inspect the secret, run inside the Document Server container: ```bash jq -r '.services.CoAuthoring.secret.session.string' /etc/onlyoffice/documentserver/local.json diff --git a/docs/public/openapi.json b/docs/public/openapi.json index e6aa903a2..6fc261af5 100644 --- a/docs/public/openapi.json +++ b/docs/public/openapi.json @@ -3072,34 +3072,88 @@ }, "responses": { "200": { - "description": "What landed where, once it has all landed.", + "description": "Progress as it goes, and what landed where.", "content": { - "application/json": { + "application/x-ndjson": { "schema": { - "type": "object", - "properties": { - "success": { - "type": "boolean" + "description": "One line; the stream is a sequence of these.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "start" + } + }, + "required": ["type"], + "additionalProperties": true }, - "destination": { - "type": "string" + { + "type": "object", + "properties": { + "type": { + "const": "progress" + } + }, + "required": ["type"], + "additionalProperties": true }, - "items": { - "type": "array", - "items": { - "type": "object", - "properties": { - "from": { - "type": "string" + { + "allOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "done" + } }, - "to": { - "type": "string" - } + "required": ["type"] + }, + { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "destination": { + "type": "string" + }, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + } + } + } + }, + "additionalProperties": true } - } + ] + }, + { + "type": "object", + "properties": { + "type": { + "const": "error" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": ["type", "message"], + "additionalProperties": true } - }, - "additionalProperties": true + ] } } } @@ -3171,34 +3225,88 @@ }, "responses": { "200": { - "description": "What landed where, once it has all landed.", + "description": "Progress as it goes, and what landed where.", "content": { - "application/json": { + "application/x-ndjson": { "schema": { - "type": "object", - "properties": { - "success": { - "type": "boolean" + "description": "One line; the stream is a sequence of these.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "start" + } + }, + "required": ["type"], + "additionalProperties": true }, - "destination": { - "type": "string" + { + "type": "object", + "properties": { + "type": { + "const": "progress" + } + }, + "required": ["type"], + "additionalProperties": true }, - "items": { - "type": "array", - "items": { - "type": "object", - "properties": { - "from": { - "type": "string" + { + "allOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "done" + } }, - "to": { - "type": "string" - } + "required": ["type"] + }, + { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "destination": { + "type": "string" + }, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + } + } + } + }, + "additionalProperties": true } - } + ] + }, + { + "type": "object", + "properties": { + "type": { + "const": "error" + }, + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": ["type", "message"], + "additionalProperties": true } - }, - "additionalProperties": true + ] } } } @@ -3221,6 +3329,47 @@ } } }, + "/api/files/recent-destinations": { + "get": { + "operationId": "listRecentDestinations", + "summary": "Folders this account recently copied or moved into", + "description": "Only those it can still reach.", + "tags": ["Files"], + "security": [ + { + "session": [] + }, + { + "apiToken": [] + } + ], + "x-access": "account", + "responses": { + "200": { + "description": "Done.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["items"] + } + } + } + }, + "401": { + "$ref": "#/components/responses/E401" + } + } + } + }, "/api/files/delete-impact": { "post": { "operationId": "describeDeletion", @@ -11120,6 +11269,66 @@ } } }, + "/api/onlyoffice/storage-file": { + "post": { + "operationId": "pickFileForOnlyoffice", + "summary": "A file the editor inserts or compares, signed for it", + "tags": ["ONLYOFFICE"], + "security": [ + { + "session": [] + } + ], + "x-access": "session", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "The document." + }, + "c": { + "type": "string", + "description": "The editor’s command." + } + }, + "required": ["path"] + } + } + } + }, + "responses": { + "200": { + "description": "Done.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {}, + "additionalProperties": true + } + } + } + }, + "400": { + "$ref": "#/components/responses/E400" + }, + "401": { + "$ref": "#/components/responses/E401" + }, + "403": { + "$ref": "#/components/responses/E403" + }, + "404": { + "$ref": "#/components/responses/E404" + } + } + } + }, "/api/onlyoffice/users": { "get": { "operationId": "listOnlyofficeUsers", @@ -11723,66 +11932,6 @@ } } } - }, - "/api/onlyoffice/storage-file": { - "post": { - "operationId": "pickFileForOnlyoffice", - "summary": "A file the editor inserts or compares, signed for it", - "tags": ["ONLYOFFICE"], - "security": [ - { - "session": [] - } - ], - "x-access": "session", - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "The document." - }, - "c": { - "type": "string", - "description": "The editor’s command." - } - }, - "required": ["path"] - } - } - } - }, - "responses": { - "200": { - "description": "Done.", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": {}, - "additionalProperties": true - } - } - } - }, - "400": { - "$ref": "#/components/responses/E400" - }, - "401": { - "$ref": "#/components/responses/E401" - }, - "403": { - "$ref": "#/components/responses/E403" - }, - "404": { - "$ref": "#/components/responses/E404" - } - } - } } }, "components": { diff --git a/docs/quick-launch/overview.md b/docs/quick-launch/overview.md index 84ff40d48..5589a7bee 100644 --- a/docs/quick-launch/overview.md +++ b/docs/quick-launch/overview.md @@ -5,16 +5,16 @@ nextExplorer is packaged as a single Docker image that hosts both the API/backen ## What you need - **Docker Engine 24+ & Docker Compose v2.** The official image targets modern platforms; use the Compose workflow shown below for reproducibility. -- **Host folders to expose as volumes.** Every `/host/path:/mnt/Label` mount becomes a top-level volume in the UI. Keep the folder readable by the container user (use `PUID`/`PGID` to match the host if needed). -- **Persistent config storage.** Mount a directory to `/config` so SQLite, `app-config.json`, extensions, and the generated session secret survive upgrades. Back this directory up before major changes. -- **Optional cache storage.** Thumbnails, search indexes, and temporary files go into `/cache`; it can be cleared safely when troubleshooting. +- **Host folders to expose as volumes.** Every `/host/path:/mnt/Label` mount becomes a top-level volume in the UI. A volume is the mount itself, so the application never renames, moves or deletes one, and never creates one: that is done where the server is configured. Keep the folder readable by the container user (use `PUID`/`PGID` to match the host if needed). A volume mounted `:ro`, or one the container user may not write in, shows a lock on the home page and in the sidebar, and offers no writes to anyone. +- **Persistent config storage.** Mount a directory to `/config` so `app.db` — accounts, shares, settings — your logo and the session secret survive upgrades. Back this directory up before major changes. Without `SESSION_SECRET`, a secret is generated at the first start and kept in `/config/session-secret`, so sessions survive restarts as long as `/config` does. +- **Cache storage.** Thumbnails, RAW previews, sessions and `index.db` — the search index and folder sizes — go into `/cache`. Nothing in it needs a backup, but mount it persistently: clearing it signs everyone out and rebuilds the indexes with a pass over the volumes. ## Sample Docker Compose (production focused) ```yaml services: nextexplorer: - image: nxzai/explorer:latest + image: ghcr.io/cerede2000/explorer:latest container_name: nextexplorer restart: unless-stopped ports: @@ -39,8 +39,8 @@ services: ## Volume strategy - **Each `/mnt/Label` mount becomes a sidebar volume.** Give folders human-friendly labels to avoid confusion, e.g., `/mnt/Projects`, `/mnt/Media`. -- **`/config`:** Stores the SQLite database, `app-config.json`, and any installed extensions/themes (see `backend/src/config/env.js` for how `CONFIG_DIR` can be overridden). Back this folder up before upgrades. -- **`/cache`:** Holds thumbnails, ripgrep indexes, and other ephemeral state; deleting it is safe but will trigger regrowth. +- **`/config`:** Stores `app.db`, `logos/` and `session-secret` (see `backend/src/config/env.js` for how `CONFIG_DIR` can be overridden). Back this folder up before upgrades. +- **`/cache`:** Holds thumbnails, RAW previews, `sessions.db`, `index.db` and uploads in progress. Deleting it loses no data, but signs everyone out and rebuilds the indexes. - **Permission tip:** The entrypoint chown’s `/config` and `/cache` to the container user (default `1000:1000`). Override with `PUID`/`PGID` for custom ownership. ## First run checklist @@ -63,7 +63,7 @@ docker compose pull docker compose up -d ``` -Persistent state lives under your `/config` mount (`app.db`, `app-config.json`, extensions) while `/cache` can be rebuilt. After pulling an image, verify the entrypoint remaps any legacy `/cache` configs to `/config` and restart the service. +Persistent state lives under your `/config` mount (`app.db`, `logos/`, `session-secret`) while `/cache` can be rebuilt. An installation that started on 1.1.7 or earlier kept `app.db` in `/cache`: nothing moves it any more, so copy it to `/config` by hand before upgrading it. The server warns at start when it finds such a file there. ## What’s next diff --git a/docs/reference/cors.md b/docs/reference/cors.md index 68d712f21..8c6e57ed2 100644 --- a/docs/reference/cors.md +++ b/docs/reference/cors.md @@ -10,7 +10,7 @@ If your browser shows errors like: ## The quick fix (most common) -1. Decide the *one* URL you will use to access nextExplorer (scheme + host + optional port), for example: +1. Decide the _one_ URL you will use to access nextExplorer (scheme + host + optional port), for example: - `https://files.example.com` 2. Set it on the server: - `PUBLIC_URL=https://files.example.com` @@ -22,7 +22,7 @@ If you access nextExplorer from multiple domains (or you have a separate fronten ### `CORS_ORIGINS` / `CORS_ORIGIN` / `ALLOWED_ORIGINS` -- **What it is:** A comma-separated list of allowed *origins* (no paths). +- **What it is:** A comma-separated list of allowed _origins_ (no paths). - **Examples:** - `CORS_ORIGINS=https://files.example.com` - `CORS_ORIGINS=https://files.example.com,https://admin.example.com` @@ -81,4 +81,3 @@ CORS_ORIGINS=http://localhost:5173 - Confirm your reverse proxy forwards `X-Forwarded-Proto`, `X-Forwarded-Host`, and `X-Forwarded-For`. - Make sure you’re visiting the exact `PUBLIC_URL` (including `https` vs `http` and ports). - Double-check there’s no browser cache/service-worker holding onto an old origin. - diff --git a/docs/reference/troubleshooting.md b/docs/reference/troubleshooting.md index 04b36623e..15ad1449d 100644 --- a/docs/reference/troubleshooting.md +++ b/docs/reference/troubleshooting.md @@ -5,14 +5,17 @@ Keep this page handy when deployment, authentication, or UI behaviors need quick ## Authentication & sessions - **OIDC redirect errors:** Make sure your identity provider uses `${PUBLIC_URL}/callback` (or `OIDC_CALLBACK_URL`) as the redirect URI. -- **Session resets after restart:** Set `SESSION_SECRET` so the app doesn't regenerate a new secret each start. -- **Users logged out after browser restart:** Sessions persist by default for 30 days. If users are being logged out, check that `SESSION_SECRET` is set and stable. Adjust `SESSION_MAX_AGE_DAYS` to change the session duration. +- **Session resets after restart:** Check that `/config` is persistent and writable by the server’s user, since the session secret generated when `SESSION_SECRET` is unset is kept there; the log says `Could not store the session secret` otherwise. Or set `SESSION_SECRET`. +- **Users logged out after browser restart:** Sessions persist by default for 30 days. If users are being logged out, check that `SESSION_SECRET`, or `/config/session-secret` when it is unset, stays the same. Adjust `SESSION_MAX_AGE_DAYS` to change the session duration. - **Users not admin:** Confirm that the user's `groups`, `roles`, or `entitlements` include a value listed in `OIDC_ADMIN_GROUPS` (case-insensitive). - **Cookies marked insecure behind HTTPS:** Ensure `PUBLIC_URL` uses `https` and your proxy forwards `X-Forwarded-Proto` and `Host`. ## Access & permissions -- **Path marked read-only or hidden:** Check Settings → Access Control for matching rules; `hidden` and `ro` rules block writes even if user has permission. +- **Path marked read-only or hidden:** Check Settings → Access Control for matching rules. A `hidden` rule applies to everyone, administrators included; an `ro` rule restricts every account except administrators, so test one with an ordinary account. +- **A rule seems to do nothing:** Its path must be the one NextExplorer shows, volume first (`torrents`, not `mnt/torrents`). The rule editor flags a path that names no folder and offers the folder probably meant. +- **A lock beside a volume:** Nothing can be written in it, and New, Upload and Delete are not offered there — to administrators either. Hover the lock for the reason: the volume is mounted read-only (`:ro` in the Compose file), the server's user may not write in it (match `PUID`/`PGID` to the owner on the host), or a rule or the volume's assignment keeps your account to reading. +- **A volume cannot be renamed or deleted:** By design. A volume is a mount, usually somebody's data shared with other programs, so the application refuses to rename, move, copy or delete one — through the interface and through the API alike. Change the mount in your Compose file instead. - **Missing volume entries:** Confirm your `docker-compose` mounts include `/mnt/Label` entries and the container has read access. - **Path not found after remounting:** Restart the container whenever you change volume mounts in your Compose file so the app rescans volumes. @@ -20,7 +23,7 @@ Keep this page handy when deployment, authentication, or UI behaviors need quick - **Slow or missing search results:** Install or enable ripgrep. The official image bundles `rg`; custom builds need either the tool or fallback search (which may skip large files controlled by `SEARCH_MAX_FILESIZE`). - **Thumbnails not generating:** Verify FFmpeg/ffprobe are available (paths override via `FFMPEG_PATH`/`FFPROBE_PATH`) and that `/cache` is writable. -- **Cache rebuild:** Clearing `/cache` removes thumbnails/indexes but keeps user data. The app regenerates thumbnails when you revisit folders. +- **Cache rebuild:** Clearing `/cache` keeps every account, share and setting, but signs everyone out, and the search index and folder sizes (`index.db`) are rebuilt by a pass over the volumes. Thumbnails come back as folders are visited. ## Reverse proxy issues @@ -31,9 +34,10 @@ Keep this page handy when deployment, authentication, or UI behaviors need quick ## Updates & persistence -- **Settings lost after update:** Mount `/config` persistently; it contains `app.db`, `app-config.json`, and extensions. Back this folder up before upgrading. -- **`/cache` filling disk:** `/cache` holds thumbnails and indexes; delete it if you need to reclaim space (the app rebuilds contents as needed). +- **Settings lost after update:** Mount `/config` persistently; it contains `app.db`, `logos/` and `session-secret`. Back this folder up before upgrading. An installation that started on 1.1.7 or earlier kept `app.db` in `/cache`, and nothing moves it to `/config` any more; the server warns at start when it finds that file. +- **`/cache` filling disk:** `/cache` holds thumbnails, sessions and `index.db`. Deleting it reclaims the space at the cost above. `THUMBNAIL_CACHE_MAX_FILES` bounds the thumbnails and `RAW_PREVIEW_CACHE_MAX_FILES` the RAW previews; `SEARCH_INDEX_EXCLUDE` keeps folders nobody searches out of the index. ## ONLYOFFICE token errors - "Document security token is not correctly configured" typically means the Document Server and nextExplorer share mismatched `ONLYOFFICE_SECRET`. Double-check the secret stored in `/etc/onlyoffice/documentserver/local.json` and update both sides to match. +- If `ONLYOFFICE_SECRET` is not set at all, nextExplorer signs with a secret derived from the session secret instead of reusing it, and logs a warning at startup. That derived secret stays the same as long as the session secret does: `SESSION_SECRET`, or `/config/session-secret` when it is unset. Deployments that relied on the old fallback — Document Server configured with the value of `SESSION_SECRET`, no `ONLYOFFICE_SECRET` — must now set `ONLYOFFICE_SECRET` explicitly on both sides. diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 000000000..fd0ed2cd3 --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,180 @@ +import js from '@eslint/js'; +import globals from 'globals'; +import prettier from 'eslint-config-prettier'; +import pluginVue from 'eslint-plugin-vue'; + +/** + * One configuration for the whole repository. + * + * It replaces three `.eslintrc.cjs` files — root, backend, frontend — that + * ESLint 9 stopped reading. Flat configuration has no `env` and no + * `/* eslint-env *\/` comment: what a file may refer to is declared here, by + * the paths it applies to, which is why the sections below are grouped by + * where the code runs rather than by which package it belongs to. + */ + +/** + * Deleting someone's files goes through the trash. Removing from disk directly + * is allowed only in the files below, which remove what the application itself + * created (temporary uploads, caches, extraction staging) or implement the + * permanent deletion the trash hands back to. A new file that deletes content + * must go through services/trash, or be added here with the reason it does not. + */ +const mayDeleteFromDisk = [ + 'backend/src/services/trash/**', + 'backend/src/services/versions/**', + 'backend/src/services/fileTransferService.js', + // Its own staging copy of the index database, under the cache directory. + 'backend/src/services/indexDb.js', + // What an operation recorded itself as writing, after a stop interrupted it. + 'backend/src/services/inFlightFiles.js', + // A rename that refuses to overwrite: the old name of a file it has just + // linked, or an empty placeholder of its own. + 'backend/src/utils/placeWithoutOverwrite.js', + // What an undone operation wrote, told apart by inode from what others added. + 'backend/src/utils/ownedTree.js', + // Its own decompressed copies of archives, under the cache directory: made + // again from the archive whenever they are missing, so nothing is lost by + // taking one away. + 'backend/src/services/archiveCacheService.js', + 'backend/src/services/archiveService.js', + 'backend/src/services/rawPreviewService.js', + 'backend/src/services/thumbnailService.js', + 'backend/src/services/tusUploadService.js', + 'backend/src/services/uploadRemnants.js', + 'backend/src/services/uploadService.js', + 'backend/src/routes/onlyoffice.js', + // A logo it wrote into /config/logos, once another has replaced it, or when + // the settings could not be switched to it. + 'backend/src/services/brandingLogo.js', + // The hidden folder it extracts into, which it made itself and which never + // holds anything but what came out of the archive. + 'backend/src/routes/archive.js', + 'backend/src/routes/zip.js', + 'backend/src/scripts/**', +]; + +const deletionGoesThroughTheTrash = + 'Deleting from disk goes through services/trash (see eslint.config.mjs).'; + +const FS_OBJECTS = ['fs', 'fsp', 'fss', 'fsSync', 'fsPromises', 'promises']; +const FS_DELETIONS = ['rm', 'rmSync', 'unlink', 'unlinkSync', 'rmdir', 'rmdirSync']; + +export default [ + { + ignores: [ + '**/coverage/', + '**/dist/', + '**/dist-ssr/', + '**/storybook-static/', + '**/.vitepress/cache/', + '**/.vitepress/dist/', + // The frontend build, copied where the image serves it (and where the + // browser tests put it): minified output, not source. + 'backend/src/public/', + ], + }, + + js.configs.recommended, + + // Everything, unless a section below says otherwise: modern syntax, and the + // globals every JavaScript runtime has. + { + languageOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + globals: { ...globals.es2022 }, + }, + rules: { + // An error nobody reads is a decision, and this is how the codebase + // writes it down: `catch (_)`. ESLint 10 began reporting every caught + // name that goes unused, which is worth having — for the ones that were + // given a real name and then forgotten. + 'no-unused-vars': ['error', { caughtErrorsIgnorePattern: '^_' }], + }, + }, + + // The backend: CommonJS, running under Node. + { + files: ['backend/**/*.js'], + languageOptions: { + sourceType: 'commonjs', + globals: { ...globals.node }, + }, + }, + + // Its suites and its vitest configuration are ES modules, while src is not. + { + files: ['backend/tests/**/*.js', 'backend/vitest.config.js'], + languageOptions: { sourceType: 'module' }, + }, + + { + files: ['backend/src/**/*.js'], + ignores: mayDeleteFromDisk, + rules: { + 'no-restricted-properties': [ + 'error', + ...FS_OBJECTS.flatMap((object) => + FS_DELETIONS.map((property) => ({ + object, + property, + message: deletionGoesThroughTheTrash, + })) + ), + ], + 'no-restricted-syntax': [ + 'error', + { + selector: + "CallExpression[callee.name='spawn'][arguments.0.value='rm'], CallExpression[callee.property.name='spawn'][arguments.0.value='rm']", + message: deletionGoesThroughTheTrash, + }, + { + selector: + "VariableDeclarator[init.callee.name='require'][init.arguments.0.value=/^(node:)?fs(\\u002Fpromises)?$/] > ObjectPattern > Property[key.name=/^(rm|rmSync|unlink|unlinkSync|rmdir|rmdirSync)$/]", + message: deletionGoesThroughTheTrash, + }, + ], + }, + }, + + // The frontend: ES modules in a browser, with Vue's own rules. + ...pluginVue.configs['flat/essential'].map((config) => ({ + ...config, + files: config.files ?? ['frontend/**/*.{js,vue}'], + })), + { + files: ['frontend/**/*.{js,vue}'], + languageOptions: { globals: { ...globals.browser } }, + }, + // The documentation site's own components, which run in the reader's browser. + { + files: ['docs/.vitepress/theme/**/*.{js,vue}'], + languageOptions: { globals: { ...globals.browser } }, + }, + + // What builds and tests the frontend runs under Node, not in a browser — and + // the end-to-end suite drives a browser from Node, so it refers to both. + { + files: ['frontend/*.config.js', 'frontend/vitest.setup.js', 'frontend/e2e/**/*.{js,mjs}'], + languageOptions: { globals: { ...globals.node } }, + }, + + // Helper scripts, the ones shipped with the image and the ones that are not. + { + files: ['docker/**/*.js', 'scripts/**/*.js', '**/*.cjs'], + languageOptions: { + sourceType: 'commonjs', + globals: { ...globals.node }, + }, + }, + { + files: ['scripts/**/*.mjs', 'docs/**/*.mjs'], + languageOptions: { globals: { ...globals.node } }, + }, + + // Last, so that everything Prettier decides is switched off here rather than + // argued about twice. + prettier, +]; diff --git a/frontend/.eslintrc.cjs b/frontend/.eslintrc.cjs deleted file mode 100644 index 76300276a..000000000 --- a/frontend/.eslintrc.cjs +++ /dev/null @@ -1,15 +0,0 @@ -/* eslint-env node */ -module.exports = { - root: true, - extends: ['plugin:vue/vue3-essential', 'eslint:recommended', 'prettier'], - parserOptions: { - ecmaVersion: 'latest', - }, - // `ecmaVersion: latest` sets the syntax and not what exists at run time, so - // `globalThis` — standard since ES2020, and what a store reaches for when it has to - // touch a timer or a listener the browser owns — read as an undefined name. - env: { - browser: true, - es2022: true, - }, -}; diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 467b4a7df..ad0fe492e 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,4 +1,4 @@ -FROM public.ecr.aws/docker/library/node:20-bookworm +FROM public.ecr.aws/docker/library/node:24.21-bookworm RUN apt-get update && apt-get install -y --no-install-recommends \ openssh-client \ && rm -rf /var/lib/apt/lists/* diff --git a/frontend/package.json b/frontend/package.json index 15bde9e02..c3cc4cf07 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,91 +1,68 @@ { "name": "explorer", - "version": "3.1.0", + "engines": { + "node": ">=24 <25" + }, + "version": "3.11.0", "private": true, "type": "module", "scripts": { "dev": "vite", "build": "vite build", "preview": "vite preview", - "storybook": "storybook dev -p 6006", - "build-storybook": "storybook build", "test:unit": "vitest", - "pretest:e2e": "playwright install chromium", + "pretest:e2e": "playwright install chromium firefox", "test:e2e": "playwright test --config playwright.config.js", - "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs --fix --ignore-path .gitignore", + "lint": "eslint . --fix", "format": "prettier --config ../prettier.config.cjs --write .", - "format:check": "prettier --config ../prettier.config.cjs --check ." + "format:check": "prettier --config ../prettier.config.cjs --check .", + "test:coverage": "vitest run --coverage --coverage.reporter=text-summary --coverage.reporter=json-summary" }, "dependencies": { - "@codemirror/lang-javascript": "^6.2.2", "@codemirror/language-data": "^6.3.2", - "@codemirror/theme-one-dark": "^6.1.2", "@coleqiu/vue-drag-select": "^2.0.6-beta.1", "@floating-ui/vue": "^1.1.9", "@fsegurai/codemirror-theme-bundle": "^6.3.0", - "@fsegurai/codemirror-theme-github-dark": "^6.2.2", "@headlessui/vue": "^1.7.22", "@heroicons/vue": "^2.1.3", "@onlyoffice/document-editor-vue": "^1.6.1", "@uppy/core": "^5.2.0", "@uppy/drop-target": "^4.1.0", - "@uppy/status-bar": "^5.1.0", "@uppy/tus": "^5.1.1", "@uppy/xhr-upload": "^5.1.1", - "@vueuse/components": "^10.9.0", "@vueuse/core": "^10.9.0", "@xterm/addon-fit": "^0.10.0", "@xterm/xterm": "^5.5.0", - "axios": "^1.6.8", - "browser-fs-access": "^0.35.0", "codemirror": "^6.0.1", "dayjs": "^1.11.11", - "dompurify": "^3.0.8", - "dropzone": "^6.0.0-beta.2", + "dompurify": "^3.4.12", "flatpickr": "^4.6.13", "marked": "^12.0.2", - "nanoid": "^5.0.7", "pinia": "^2.1.7", "qrcode-generator": "^2.0.4", - "tippy.js": "^6.3.7", - "vue": "^3.4.21", - "vue-codemirror": "^6.1.1", - "vue-i18n": "^9.14.0", + "vue": "^3.5.0", + "vue-i18n": "^11.4.10", "vue-router": "^4.3.0", "vuedraggable": "^4.1.0" }, "devDependencies": { "@playwright/test": "^1.62.1", - "@storybook/addon-essentials": "^8.4.7", - "@storybook/addon-interactions": "^8.4.7", - "@storybook/addon-links": "^8.4.7", - "@storybook/blocks": "^8.4.7", - "@storybook/test": "^8.4.7", - "@storybook/vue3": "^8.4.7", - "@storybook/vue3-vite": "^8.4.7", "@tailwindcss/typography": "^0.5.19", - "@tailwindcss/vite": "^4.1.18", - "@vicons/antd": "^0.12.0", - "@vicons/carbon": "^0.12.0", - "@vicons/fluent": "^0.12.0", - "@vicons/ionicons4": "^0.12.0", - "@vicons/ionicons5": "^0.12.0", - "@vicons/material": "^0.12.0", - "@vicons/tabler": "^0.12.0", - "@vicons/utils": "^0.1.4", - "@vitejs/plugin-vue": "^6.0.4", + "@tailwindcss/vite": "^4.3.3", + "@vicons/fluent": "^0.13.0", + "@vicons/ionicons5": "^0.13.0", + "@vicons/material": "^0.13.0", + "@vicons/tabler": "^0.13.0", + "@vitejs/plugin-vue": "^6.0.9", + "@vitest/coverage-v8": "^4.1.11", "@vue/test-utils": "^2.4.5", - "eslint": "^8.57.0", - "eslint-config-prettier": "^9.1.2", - "eslint-plugin-vue": "^9.23.0", "jsdom": "^24.0.0", "postcss": "^8.4.38", "prettier": "^3.6.2", "sass": "^1.77.2", - "storybook": "^8.4.7", "tailwindcss": "^4.1.18", - "vite": "^5.2.8", - "vite-plugin-vue-devtools": "^7.0.25", - "vitest": "^4.0.18" + "vite": "^7.3.6", + "vite-plugin-vue-devtools": "^8.2.1", + "vitest": "^4.1.11" } } diff --git a/frontend/src/App.spec.js b/frontend/src/App.spec.js index 8be9c3c68..42b112764 100644 --- a/frontend/src/App.spec.js +++ b/frontend/src/App.spec.js @@ -9,6 +9,13 @@ const dismissConfigWarning = vi.fn(() => { } }); +// The language somebody's account is set to is watched from here now, which +// reaches a store. This test is about the configuration gate above it, and a +// store it never exercises would only be a second thing to keep in step. +vi.mock('@/composables/useAccountLanguage', () => ({ + useAccountLanguage: () => {}, +})); + vi.mock('@/composables/useConfigErrorGate', () => ({ useConfigErrorGate: () => ({ configError, diff --git a/frontend/src/App.vue b/frontend/src/App.vue index b332a762b..19e383cac 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -1,16 +1,14 @@ @@ -72,6 +320,10 @@ const destination = computed(() => operation.value?.destination ?? ''); animation: clipboardSlide 1.35s ease-in-out infinite; } +.clipboard-bar--determinate { + transition: width 0.2s ease; +} + @media (prefers-reduced-motion: reduce) { .clipboard-bar--animated { animation: none; diff --git a/frontend/src/components/ConfigWarningNotice.vue b/frontend/src/components/ConfigWarningNotice.vue index 9e02e2e2e..8ff67f180 100644 --- a/frontend/src/components/ConfigWarningNotice.vue +++ b/frontend/src/components/ConfigWarningNotice.vue @@ -47,7 +47,12 @@ onKeyStroke('Escape', dismiss); diff --git a/frontend/src/components/DestinationPickerDialog.spec.js b/frontend/src/components/DestinationPickerDialog.spec.js new file mode 100644 index 000000000..e6961a1f4 --- /dev/null +++ b/frontend/src/components/DestinationPickerDialog.spec.js @@ -0,0 +1,355 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mount, flushPromises } from '@vue/test-utils'; +import { createI18n } from 'vue-i18n'; +import { createPinia, setActivePinia } from 'pinia'; + +const browse = vi.fn(); +const fetchRecentDestinations = vi.fn(); +const fetchFavorites = vi.fn(); + +vi.mock('@/api', () => ({ + browse: (...args) => browse(...args), + fetchRecentDestinations: (...args) => fetchRecentDestinations(...args), + fetchFavorites: (...args) => fetchFavorites(...args), + addFavorite: vi.fn(), + updateFavorite: vi.fn(), + reorderFavorites: vi.fn(), + removeFavorite: vi.fn(), + normalizePath: (value) => String(value || '').replace(/^\/+|\/+$/g, ''), +})); + +import DestinationPickerDialog from './DestinationPickerDialog.vue'; +import { useDestinationPicker } from '@/composables/useDestinationPicker'; +import { useFavoritesStore } from '@/stores/favorites'; + +/** + * Choosing where something goes. + * + * This dialog exists because dragging is switched off on touch devices, so it + * has to stand on its own: offer the folders someone actually uses, and refuse + * — before the transfer, not after — the destinations the server would reject + * anyway. A picker that lets you choose a folder and then fails is worse than + * one that never offered it. + */ + +const i18n = createI18n({ + legacy: false, + locale: 'en', + messages: { + en: { + common: { cancel: 'Cancel', close: 'Close', loadingEllipsis: 'Loading…' }, + storagePicker: { root: 'Storage', breadcrumb: 'Folder path' }, + destinationPicker: { + moveTitle: 'Move to', + copyTitle: 'Copy to', + restoreTitle: 'Restore to', + moveHere: 'Move here', + copyHere: 'Copy here', + restoreHere: 'Restore here', + noFolders: 'No folders here', + rootRejected: 'Pick a volume or folder first', + itselfRejected: 'A folder cannot be moved into itself', + descendantRejected: 'A folder cannot be moved into one of its own folders', + alreadyThereRejected: 'Already here', + versionCopyTitle: 'Restore a copy to', + versionCopyHere: 'Restore a copy here', + replaceTitle: 'Choose the file to replace', + replaceHere: 'Replace this file', + fileRequired: 'Pick a file', + sameFileRejected: 'Pick a file other than this one', + nothingHere: 'Nothing here', + }, + }, + }, +}); + +const folder = (name, path = '') => ({ name, path, kind: 'directory' }); + +const listing = (items, path = '') => ({ items, path }); + +// ModalDialog teleports to the body, so the dialog is inspected there rather +// than inside the wrapper. +let wrapper = null; +const mountDialog = () => { + wrapper = mount(DestinationPickerDialog, { + global: { plugins: [i18n] }, + attachTo: document.body, + }); + return wrapper; +}; + +const buttons = () => Array.from(document.querySelectorAll('button')); +const bodyText = () => document.body.textContent || ''; + +/** The button that commits the choice. */ +const confirmButton = () => + buttons().find((button) => /^(Move|Copy|Restore) here$/.test(button.textContent.trim())); + +describe('DestinationPickerDialog', () => { + let picker; + + afterEach(() => { + wrapper?.unmount(); + wrapper = null; + }); + + beforeEach(() => { + setActivePinia(createPinia()); + vi.clearAllMocks(); + document.body.innerHTML = ''; + fetchRecentDestinations.mockResolvedValue([]); + fetchFavorites.mockResolvedValue([]); + browse.mockResolvedValue(listing([])); + + picker = useDestinationPicker(); + picker.isOpen.value = false; + picker.items.value = []; + picker.mode.value = 'move'; + picker.initialPath.value = ''; + }); + + it('hands back the folder that was open when confirmed', async () => { + browse.mockResolvedValue(listing([folder('Archive', 'Docs')], 'Docs')); + + mountDialog(); + const chosen = picker.pick({ mode: 'move', items: [{ name: 'a.txt', path: 'Inbox' }] }); + await flushPromises(); + + confirmButton().click(); + await flushPromises(); + + await expect(chosen).resolves.toBe('Docs'); + }); + + it('reports nothing chosen when the dialog is dismissed', async () => { + // Callers must be able to tell "cancelled" from "chose the root", which the + // server would refuse anyway. + mountDialog(); + const chosen = picker.pick({ items: [{ name: 'a.txt', path: 'Docs' }] }); + await flushPromises(); + + buttons() + .find((button) => button.textContent.trim() === 'Cancel') + .click(); + await flushPromises(); + + await expect(chosen).resolves.toBeNull(); + }); + + it('refuses the root, which has no volume to write to', async () => { + browse.mockResolvedValue(listing([folder('Docs')], '')); + + mountDialog(); + picker.pick({ items: [{ name: 'a.txt', path: 'Docs' }] }); + await flushPromises(); + + expect(bodyText()).toContain('Pick a volume or folder first'); + expect(confirmButton().disabled).toBe(true); + }); + + it('refuses a folder being moved into itself', async () => { + browse.mockResolvedValue(listing([], 'Docs/Reports')); + + mountDialog(); + picker.pick({ items: [{ name: 'Reports', path: 'Docs', kind: 'directory' }] }); + await flushPromises(); + + expect(bodyText()).toContain('A folder cannot be moved into itself'); + expect(confirmButton().disabled).toBe(true); + }); + + it('refuses a folder being moved inside one of its own folders', async () => { + // The deeper case, which is the one people actually hit by browsing into it. + browse.mockResolvedValue(listing([], 'Docs/Reports/2026')); + + mountDialog(); + picker.pick({ items: [{ name: 'Reports', path: 'Docs', kind: 'directory' }] }); + await flushPromises(); + + expect(bodyText()).toContain('A folder cannot be moved into one of its own folders'); + expect(confirmButton().disabled).toBe(true); + }); + + it('refuses moving something back where it already is', async () => { + browse.mockResolvedValue(listing([], 'Docs')); + + mountDialog(); + picker.pick({ mode: 'move', items: [{ name: 'a.txt', path: 'Docs' }] }); + await flushPromises(); + + expect(bodyText()).toContain('Already here'); + }); + + it('allows copying into the folder something is already in', async () => { + // Copying beside the original is a real thing to want — it produces the + // usual duplicate — so the move-only check must not apply here. + browse.mockResolvedValue(listing([], 'Docs')); + + mountDialog(); + picker.pick({ mode: 'copy', items: [{ name: 'a.txt', path: 'Docs' }] }); + await flushPromises(); + + expect(bodyText()).not.toContain('Already here'); + expect(confirmButton().disabled).toBe(false); + }); + + it('offers recent destinations and favorites, without repeating one', async () => { + fetchRecentDestinations.mockResolvedValue(['Docs/Reports', 'Media']); + fetchFavorites.mockResolvedValue([ + { id: '1', path: 'Media' }, + { id: '2', path: 'Backups' }, + ]); + useFavoritesStore(); + + mountDialog(); + picker.pick({ items: [{ name: 'a.txt', path: 'Inbox' }] }); + await flushPromises(); + + const shortcuts = buttons().map((button) => button.textContent); + expect(shortcuts.filter((text) => text.includes('Media'))).toHaveLength(1); + expect(shortcuts.some((text) => text.includes('Docs/Reports'))).toBe(true); + expect(shortcuts.some((text) => text.includes('Backups'))).toBe(true); + }); + + it('leaves out a shortcut that would be an invalid destination', async () => { + // A favorite pointing at the folder being moved is still a favorite; it is + // just not somewhere this transfer can go. + fetchRecentDestinations.mockResolvedValue(['Docs/Reports']); + + mountDialog(); + picker.pick({ items: [{ name: 'Reports', path: 'Docs', kind: 'directory' }] }); + await flushPromises(); + + expect(bodyText()).not.toContain('Docs/Reports'); + }); + + it('still browses when recent destinations cannot be loaded', async () => { + fetchRecentDestinations.mockRejectedValue(new Error('nope')); + browse.mockResolvedValue(listing([folder('Archive', 'Docs')], 'Docs')); + + mountDialog(); + picker.pick({ items: [{ name: 'a.txt', path: 'Inbox' }] }); + await flushPromises(); + + expect(bodyText()).toContain('Archive'); + expect(confirmButton().disabled).toBe(false); + }); + + /** + * Out of the trash, nothing is taken from a folder: the only destination to + * refuse is the root, and the dialog says what will happen in its own words. + */ + it('asks where to restore, in those words, and hands back the folder chosen', async () => { + browse.mockResolvedValue(listing([folder('Archive', 'Docs')], 'Docs')); + + mountDialog(); + const chosen = picker.pick({ mode: 'restore' }); + await flushPromises(); + + expect(bodyText()).toContain('Restore to'); + expect(confirmButton().textContent.trim()).toBe('Restore here'); + confirmButton().click(); + await flushPromises(); + + await expect(chosen).resolves.toBe('Docs'); + }); + + /** + * An earlier version taken out as a copy goes into a folder, and may go beside + * the file it came from: that is the usual place for a copy. + */ + it('asks where to put a copy of a version, beside its file included', async () => { + browse.mockResolvedValue(listing([], 'Docs')); + + mountDialog(); + const chosen = picker.pick({ + mode: 'version-copy', + items: [{ name: 'notes.md', path: 'Docs', kind: 'file' }], + from: 'Docs', + }); + await flushPromises(); + + const confirmCopy = buttons().find((b) => b.textContent.trim() === 'Restore a copy here'); + expect(bodyText()).toContain('Restore a copy to'); + expect(bodyText()).not.toContain('Already here'); + confirmCopy.click(); + await flushPromises(); + + await expect(chosen).resolves.toBe('Docs'); + }); + + describe('choosing a file rather than a folder', () => { + const files = () => + Array.from(document.querySelectorAll('[data-test="destination-picker-file"]')); + const replaceButton = () => buttons().find((b) => b.textContent.trim() === 'Replace this file'); + const source = [{ name: 'notes.md', path: 'Docs', kind: 'file' }]; + + it('lists the files beside the folders, and asks for one before anything is confirmed', async () => { + browse.mockResolvedValue( + listing([folder('Old', 'Docs'), { name: 'draft.md', path: 'Docs', kind: 'md' }], 'Docs') + ); + + mountDialog(); + picker.pick({ mode: 'file', items: source, from: 'Docs' }); + await flushPromises(); + + expect(bodyText()).toContain('Choose the file to replace'); + expect(bodyText()).toContain('Old'); + expect(files().map((button) => button.textContent.trim())).toEqual(['draft.md']); + expect(bodyText()).toContain('Pick a file'); + expect(replaceButton().disabled).toBe(true); + }); + + it('hands back the path of the file chosen', async () => { + browse.mockResolvedValue(listing([{ name: 'draft.md', path: 'Docs', kind: 'md' }], 'Docs')); + + mountDialog(); + const chosen = picker.pick({ mode: 'file', items: source, from: 'Docs' }); + await flushPromises(); + + files()[0].click(); + await flushPromises(); + expect(replaceButton().disabled).toBe(false); + replaceButton().click(); + await flushPromises(); + + await expect(chosen).resolves.toBe('Docs/draft.md'); + }); + + it('refuses the file the version belongs to, which a restore already covers', async () => { + browse.mockResolvedValue(listing([{ name: 'notes.md', path: 'Docs', kind: 'md' }], 'Docs')); + + mountDialog(); + picker.pick({ mode: 'file', items: source, from: 'Docs' }); + await flushPromises(); + + files()[0].click(); + await flushPromises(); + + expect(bodyText()).toContain('Pick a file other than this one'); + expect(replaceButton().disabled).toBe(true); + }); + + it('forgets the file chosen once another folder is opened', async () => { + browse + .mockResolvedValueOnce( + listing([folder('Old', 'Docs'), { name: 'draft.md', path: 'Docs', kind: 'md' }], 'Docs') + ) + .mockResolvedValueOnce(listing([], 'Docs/Old')); + + mountDialog(); + picker.pick({ mode: 'file', items: source, from: 'Docs' }); + await flushPromises(); + files()[0].click(); + await flushPromises(); + + buttons() + .find((b) => b.textContent.trim() === 'Old') + .click(); + await flushPromises(); + + expect(bodyText()).toContain('Nothing here'); + expect(replaceButton().disabled).toBe(true); + }); + }); +}); diff --git a/frontend/src/components/DestinationPickerDialog.vue b/frontend/src/components/DestinationPickerDialog.vue new file mode 100644 index 000000000..1d610288a --- /dev/null +++ b/frontend/src/components/DestinationPickerDialog.vue @@ -0,0 +1,317 @@ + + + diff --git a/frontend/src/components/ExplorerContextMenu.spec.js b/frontend/src/components/ExplorerContextMenu.spec.js new file mode 100644 index 000000000..984c04371 --- /dev/null +++ b/frontend/src/components/ExplorerContextMenu.spec.js @@ -0,0 +1,1317 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mount, flushPromises } from '@vue/test-utils'; +import { defineComponent, h, inject, ref } from 'vue'; +import { createPinia, setActivePinia } from 'pinia'; + +/** + * The right-click menu. + * + * 409 statements at 0.7%, and it is where a permission becomes a thing somebody + * can click. Everything the explorer refuses is refused twice — once by the + * guard that runs the action, once by this menu deciding whether to offer it — + * and the second one is what people actually see. An entry that stays live on a + * read-only share is not a cosmetic problem: it is a person told they may do + * something, finding out afterwards that they may not. + * + * The menu is three different menus depending on what was clicked — the + * background, a file, a folder — and the differences are the interesting part. + * Paste belongs to a folder and the background but not to a file. Rename needs + * a target. Open-in-terminal is for a file only, and only where the terminal is + * switched on at all. + */ + +let actions; +let fileStore; +let features; +let favorites; + +const infoOpen = vi.fn(); +const infoClose = vi.fn(); +const versionsOpen = vi.fn(); +const openEditorForFavorite = vi.fn(); +const getDeleteImpact = vi.fn(async () => ({ shareCount: 0, shares: [] })); +const terminalOpen = vi.fn(); +const routerPush = vi.fn(); + +vi.mock('@floating-ui/vue', () => ({ + useFloating: () => ({ x: ref(0), y: ref(0), strategy: ref('fixed'), update: vi.fn() }), + offset: vi.fn(), + flip: vi.fn(), + shift: vi.fn(), + autoUpdate: vi.fn(), + size: vi.fn(), +})); +vi.mock('vue-router', () => ({ useRouter: () => ({ push: routerPush }) })); +/** + * `t` gives back the key, and the values interpolated into it when there are + * any — so a message naming a file, or counting several, can be told apart from + * the same message about something else. + */ +const translate = (key, params) => + params && typeof params === 'object' ? `${key} ${JSON.stringify(params)}` : key; +vi.mock('vue-i18n', () => ({ useI18n: () => ({ t: translate }) })); +/** + * A stable object that always reads the current test's actions. + * + * `useDeleteConfirm` is a module-level singleton: it calls `useFileActions()` + * once, on the first mount, and keeps whatever it got for the rest of the file. + * Handing it the live object directly would freeze every later test's selection + * to the first one's. + */ +const actionsProxy = new Proxy( + {}, + { + get: (_target, property) => actions[property], + has: (_target, property) => property in actions, + ownKeys: () => Reflect.ownKeys(actions), + getOwnPropertyDescriptor: (_target, property) => + Object.getOwnPropertyDescriptor(actions, property), + } +); +vi.mock('@/composables/fileActions', () => ({ useFileActions: () => actionsProxy })); +vi.mock('@/stores/fileStore', () => ({ useFileStore: () => fileStore })); +vi.mock('@/stores/infoPanel', () => ({ + useInfoPanelStore: () => ({ open: infoOpen, close: infoClose }), +})); +vi.mock('@/stores/versionsPanel', () => ({ + useVersionsPanelStore: () => ({ open: versionsOpen }), +})); +vi.mock('@/stores/favorites', () => ({ useFavoritesStore: () => favorites })); +vi.mock('@/stores/features', () => ({ useFeaturesStore: () => features })); +vi.mock('@/stores/terminal', () => ({ useTerminalStore: () => ({ open: terminalOpen }) })); +vi.mock('@/composables/itemSelection', () => ({ + useSelection: () => ({ clearSelection: vi.fn() }), +})); +vi.mock('@/composables/useFavoriteEditor', () => ({ + useFavoriteEditor: () => ({ openEditorForFavorite: openEditorForFavorite }), +})); +vi.mock('@/api', () => ({ + normalizePath: (value) => String(value || '').replace(/^\/+|\/+$/g, ''), + getDeleteImpact: (...args) => getDeleteImpact(...args), +})); +// The menu pulls in the share dialog, which is a screen of its own with a date +// picker in it. Stubbed: nothing here is about creating a share. +vi.mock('@/components/ShareDialog.vue', () => ({ + default: defineComponent({ name: 'ShareDialogStub', render: () => null }), +})); + +import ExplorerContextMenu from './ExplorerContextMenu.vue'; +import { explorerContextMenuSymbol as realSymbol } from '@/composables/contextMenu'; + +const FILE = { name: 'report.docx', path: 'Docs', kind: 'docx' }; +const FOLDER = { name: '2026', path: 'Docs', kind: 'directory' }; + +const makeActions = (overrides = {}) => ({ + selectedItems: ref([FILE]), + primaryItem: ref(FILE), + isSingleItemSelected: ref(true), + hasSelection: ref(true), + canRename: ref(true), + canCut: ref(true), + canCopy: ref(true), + canPaste: ref(true), + canDelete: ref(true), + canExtractArchive: ref(false), + canCompressToZip: ref(true), + canDownloadCurrentFolder: ref(false), + isArchiveSelected: ref(false), + isCutActive: ref(false), + isCopyActive: ref(false), + locationCanWrite: ref(true), + locationCanCreateFolder: ref(true), + locationCanCreateFile: ref(true), + locationCanDelete: ref(true), + locationCanUpload: ref(true), + locationCanDownload: ref(true), + resolveItemPath: (item) => (item?.path ? `${item.path}/${item.name}` : item?.name || ''), + isEditableElement: () => false, + runCut: vi.fn(), + runCopy: vi.fn(), + runRename: vi.fn(), + runMoveTo: vi.fn(), + runCopyTo: vi.fn(), + runPasteToDestination: vi.fn(), + runPasteIntoCurrent: vi.fn(), + runExtractArchive: vi.fn(), + runExtractArchiveIntoCurrentFolder: vi.fn(), + runCompressToZip: vi.fn(), + runDownload: vi.fn(), + runDownloadCurrentFolder: vi.fn(), + deleteNow: vi.fn(), + ...overrides, +}); + +let mounted = null; + +/** Mounts the menu with a child that captures the API it provides. */ +const mountMenu = async () => { + let api = null; + const Child = defineComponent({ + setup() { + api = inject(realSymbol); + return () => h('div', 'child'); + }, + }); + const wrapper = mount(ExplorerContextMenu, { + slots: { default: () => h(Child) }, + attachTo: document.body, + // The template uses the global `$t` as well as the `t` from useI18n, and a + // missing one throws during render rather than showing an untranslated + // string — which looks exactly like the menu refusing to open. + global: { mocks: { $t: translate } }, + }); + mounted = wrapper; + await flushPromises(); + return { wrapper, api }; +}; + +const rightClick = () => ({ + clientX: 100, + clientY: 100, + preventDefault: vi.fn(), + stopPropagation: vi.fn(), +}); + +/** + * The menu is teleported to the body, so it is read from the document rather + * than from the wrapper. `t` is mocked to return the key it was given, so each + * entry is identified by its translation key — stable, and it does not require + * adding attributes to the component just to be testable. + */ +const menuPanel = () => + [...document.body.querySelectorAll('div')].find((el) => el.className.includes('min-w-[220px]')); + +const isOpen = () => Boolean(menuPanel()); + +const entries = () => + [...(menuPanel()?.querySelectorAll('button') ?? [])].map((button) => ({ + label: button.querySelector('p')?.textContent?.trim() ?? '', + disabled: button.disabled, + })); + +const labels = () => entries().map((entry) => entry.label); + +const isDisabled = (label) => entries().find((entry) => entry.label === label)?.disabled; + +/** + * Unmounted between tests, not just cleared: the menu registers keydown and + * pointerdown listeners on `window`, and a component left mounted keeps + * answering them. That is how an Escape test that passes alone fails in a run. + */ +afterEach(() => { + // The delete confirmation is a singleton too, so a pending deletion outlives + // the component that asked for it. + mounted?.vm?.closeDeleteConfirm?.(); + mounted?.unmount(); + mounted = null; + document.body.innerHTML = ''; +}); + +beforeEach(() => { + document.body.innerHTML = ''; + setActivePinia(createPinia()); + [infoOpen, terminalOpen, routerPush, openEditorForFavorite].forEach((m) => m.mockReset()); + getDeleteImpact.mockReset(); + getDeleteImpact.mockResolvedValue({ shareCount: 0, shares: [] }); + actions = makeActions(); + fileStore = { + selectedItems: [FILE], + get selectedItemKeys() { + return new Set(fileStore.selectedItems.map((i) => `${i.path}::${i.name}`)); + }, + getCurrentPathItems: [FILE, FOLDER], + currentPath: 'Docs', + getCurrentPath: 'Docs', + currentPathData: { canWrite: true, canDelete: true }, + createFolder: vi.fn(), + createFile: vi.fn(), + createOfficeDocument: vi.fn(), + }; + features = { + terminalEnabled: true, + terminalExtensions: ['sh', 'py'], + archiveExtensions: ['zip'], + onlyofficeEnabled: false, + }; + favorites = { + isFavorite: vi.fn(() => false), + addFavorite: vi.fn().mockResolvedValue({ id: 'f1' }), + removeFavorite: vi.fn().mockResolvedValue(), + }; +}); + +describe('opening it', () => { + it('stays shut until something asks for it', async () => { + await mountMenu(); + + expect(isOpen()).toBe(false); + }); + + it('opens on a right-click on a file', async () => { + const { api } = await mountMenu(); + + api.openItemMenu(rightClick(), FILE); + await flushPromises(); + + expect(isOpen()).toBe(true); + }); + + it('opens on a right-click on the background', async () => { + const { api } = await mountMenu(); + + api.openBackgroundMenu(rightClick()); + await flushPromises(); + + expect(isOpen()).toBe(true); + }); + + /** A right-click must not also trigger the browser's own menu. */ + it('takes the event away from the browser', async () => { + const { api } = await mountMenu(); + const event = rightClick(); + + api.openItemMenu(event, FILE); + + expect(event.preventDefault).toHaveBeenCalled(); + }); + + it('ignores a call with no event, and one with no item', async () => { + const { api } = await mountMenu(); + + api.openItemMenu(null, FILE); + api.openItemMenu(rightClick(), null); + await flushPromises(); + + expect(isOpen()).toBe(false); + }); + + it('closes again', async () => { + const { api } = await mountMenu(); + api.openItemMenu(rightClick(), FILE); + await flushPromises(); + + api.closeMenu(); + await flushPromises(); + + expect(isOpen()).toBe(false); + }); + + /** Escape closes it, because a menu that traps the keyboard is a bug. */ + it('closes on Escape', async () => { + const { api } = await mountMenu(); + api.openItemMenu(rightClick(), FILE); + await flushPromises(); + + window.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape' })); + await flushPromises(); + + expect(isOpen()).toBe(false); + }); +}); + +describe('what a right-click selects', () => { + /** + * Right-clicking a row that is not in the selection selects it. Without this + * the menu acts on whatever was selected before, which is the wrong file. + */ + it('selects the row it was opened on', async () => { + const { api } = await mountMenu(); + fileStore.selectedItems = [FOLDER]; + + api.openItemMenu(rightClick(), FILE); + + expect(fileStore.selectedItems.map((i) => i.name)).toEqual(['report.docx']); + }); + + /** Right-clicking inside a multi-selection keeps it, so a bulk action works. */ + it('leaves a selection alone when the row is already in it', async () => { + const { api } = await mountMenu(); + fileStore.selectedItems = [FILE, FOLDER]; + + api.openItemMenu(rightClick(), FILE); + + expect(fileStore.selectedItems).toHaveLength(2); + }); +}); + +describe('what each of the three menus offers', () => { + const openOn = async (kind) => { + mounted?.unmount(); + document.body.innerHTML = ''; + const { api } = await mountMenu(); + if (kind === 'background') api.openBackgroundMenu(rightClick()); + else api.openItemMenu(rightClick(), kind === 'directory' ? FOLDER : FILE); + await flushPromises(); + }; + + it('offers creation on the background, where there is nothing selected to act on', async () => { + await openOn('background'); + + expect(labels()).toEqual(expect.arrayContaining(['actions.newFolder', 'actions.newFile'])); + }); + + it('offers cut, copy and rename on a file', async () => { + await openOn('file'); + + expect(labels()).toEqual( + expect.arrayContaining(['actions.cut', 'actions.copy', 'actions.rename']) + ); + }); + + /** + * Paste goes into a folder, and into the folder being looked at. Pasting + * "into" a file is not a thing, and offering it is a click that can only fail. + */ + it('offers paste on a folder but not on a file', async () => { + await openOn('directory'); + expect(labels()).toContain('actions.paste'); + + await openOn('file'); + expect(labels()).not.toContain('actions.paste'); + }); + + /** + * A second condition beyond the kind: the terminal is offered for a file it + * could actually run, which is what `TERMINAL_EXTENSIONS` names. A .docx has + * no terminal entry, and that is right. + */ + it('offers the terminal for a script', async () => { + const script = { name: 'deploy.sh', path: 'Docs', kind: 'sh' }; + actions = makeActions({ primaryItem: ref(script), selectedItems: ref([script]) }); + const { api } = await mountMenu(); + api.openItemMenu(rightClick(), script); + await flushPromises(); + + expect(labels()).toContain('context.openWithTerminal'); + }); + + it('does not offer it for a document, which the terminal could not run', async () => { + await openOn('file'); + + expect(labels()).not.toContain('context.openWithTerminal'); + }); + + it('offers no terminal on a folder', async () => { + await openOn('directory'); + + expect(labels()).not.toContain('context.openWithTerminal'); + }); + + it('offers no terminal at all where the deployment has it switched off', async () => { + features.terminalEnabled = false; + const script = { name: 'deploy.sh', path: 'Docs', kind: 'sh' }; + actions = makeActions({ primaryItem: ref(script), selectedItems: ref([script]) }); + + await openOn('file'); + + expect(labels()).not.toContain('context.openWithTerminal'); + }); + + it('offers favourites on a folder', async () => { + await openOn('directory'); + + expect(labels()).toContain('context.addToFavorites'); + }); + + it('says remove rather than add for a folder already favourited', async () => { + favorites.isFavorite = vi.fn(() => true); + + await openOn('directory'); + + expect(labels()).toContain('context.removeFromFavorites'); + expect(labels()).not.toContain('context.addToFavorites'); + }); + + it('offers extraction only for an archive', async () => { + await openOn('file'); + expect(labels()).not.toContain('actions.extractArchive'); + + actions = makeActions({ canExtractArchive: ref(true), isArchiveSelected: ref(true) }); + await openOn('file'); + expect(labels()).toContain('actions.extractArchive'); + }); +}); + +describe("a file's versions", () => { + const openOnFile = async () => { + const { api } = await mountMenu(); + api.openItemMenu(rightClick(), FILE); + await flushPromises(); + }; + + const click = async (label) => { + const button = [...(menuPanel()?.querySelectorAll('button') ?? [])].find( + (candidate) => candidate.querySelector('p')?.textContent?.trim() === label + ); + button.click(); + await flushPromises(); + }; + + beforeEach(() => { + features.versionsEnabled = true; + infoClose.mockReset(); + versionsOpen.mockReset(); + }); + + it('are offered on a single file', async () => { + await openOnFile(); + + expect(labels()).toContain('versions.menu'); + }); + + it('open the history of that file, in place of the details', async () => { + await openOnFile(); + + await click('versions.menu'); + + expect(infoClose).toHaveBeenCalled(); + expect(versionsOpen).toHaveBeenCalledWith(FILE); + }); + + it('are not offered where versions are switched off', async () => { + features.versionsEnabled = false; + + await openOnFile(); + + expect(labels()).not.toContain('versions.menu'); + }); + + it('are not offered on a folder, which has no history of its own', async () => { + actions = makeActions({ primaryItem: ref(FOLDER), selectedItems: ref([FOLDER]) }); + const { api } = await mountMenu(); + api.openItemMenu(rightClick(), FOLDER); + await flushPromises(); + + expect(labels()).not.toContain('versions.menu'); + }); + + it('are not offered for several files at once', async () => { + const other = { name: 'budget.xlsx', path: 'Docs', kind: 'xlsx' }; + actions = makeActions({ isSingleItemSelected: ref(false), selectedItems: ref([FILE, other]) }); + fileStore.selectedItems = [FILE, other]; + + await openOnFile(); + + // The menu did open, on the file clicked, with the rest of what it offers. + expect(labels()).toContain('context.getInfo'); + expect(labels()).not.toContain('versions.menu'); + }); + + it('are not offered through a share whose owner keeps them hidden', async () => { + fileStore.currentPathData = { canWrite: true, canDelete: true, canSeeVersions: false }; + + await openOnFile(); + + expect(labels()).not.toContain('versions.menu'); + }); +}); + +describe('what it will not offer where the location forbids it', () => { + const openOnFile = async (overrides) => { + actions = makeActions(overrides); + const { api } = await mountMenu(); + api.openItemMenu(rightClick(), FILE); + await flushPromises(); + }; + + const openOnBackground = async (overrides) => { + actions = makeActions(overrides); + const { api } = await mountMenu(); + api.openBackgroundMenu(rightClick()); + await flushPromises(); + }; + + /** + * The rename entry is built inside `if (locationCanWrite)`, so a read-only + * location has no rename at all rather than a greyed-out one. + */ + it('drops rename entirely on a read-only location', async () => { + await openOnFile({ locationCanWrite: ref(false) }); + + expect(labels()).not.toContain('actions.rename'); + }); + + it('greys out rename where the target itself cannot be renamed', async () => { + await openOnFile({ canRename: ref(false) }); + + expect(isDisabled('actions.rename')).toBe(true); + }); + + it('greys out cut and copy when there is nothing to cut or copy', async () => { + await openOnFile({ canCut: ref(false), canCopy: ref(false) }); + + expect(isDisabled('actions.cut')).toBe(true); + expect(isDisabled('actions.copy')).toBe(true); + }); + + it('offers no folder creation where folders may not be created', async () => { + await openOnBackground({ locationCanCreateFolder: ref(false) }); + + expect(labels()).not.toContain('actions.newFolder'); + }); + + it('offers no file creation where files may not be created', async () => { + await openOnBackground({ locationCanCreateFile: ref(false) }); + + expect(labels()).not.toContain('actions.newFile'); + }); + + it('greys out compressing when the selection cannot be zipped', async () => { + await openOnFile({ canCompressToZip: ref(false) }); + + expect(isDisabled('actions.compressToZip')).toBe(true); + }); +}); + +describe('running an entry', () => { + const clickEntry = async (label) => { + const button = [...menuPanel().querySelectorAll('button')].find( + (candidate) => candidate.querySelector('p')?.textContent?.trim() === label + ); + button.click(); + await flushPromises(); + }; + + it('cuts', async () => { + const { api } = await mountMenu(); + api.openItemMenu(rightClick(), FILE); + await flushPromises(); + + await clickEntry('actions.cut'); + + expect(actions.runCut).toHaveBeenCalled(); + }); + + it('opens the info panel on what was clicked', async () => { + const { api } = await mountMenu(); + api.openItemMenu(rightClick(), FILE); + await flushPromises(); + + await clickEntry('context.getInfo'); + + expect(infoOpen).toHaveBeenCalled(); + }); + + /** A menu that stays open over the thing it just acted on is in the way. */ + it('closes itself afterwards', async () => { + const { api } = await mountMenu(); + api.openItemMenu(rightClick(), FILE); + await flushPromises(); + + await clickEntry('actions.copy'); + + expect(isOpen()).toBe(false); + }); +}); + +/** Opens the menu on something and hands back the component instance. */ +const openOn = async (item, kind = 'item') => { + const { wrapper, api } = await mountMenu(); + if (kind === 'background') api.openBackgroundMenu(rightClick()); + else api.openItemMenu(rightClick(), item); + await flushPromises(); + return { view: wrapper.vm, api, wrapper }; +}; + +const clickLabel = async (label) => { + const button = [...menuPanel().querySelectorAll('button')].find( + (candidate) => candidate.querySelector('p')?.textContent?.trim() === label + ); + button.click(); + await flushPromises(); +}; + +describe('what the delete confirmation says', () => { + it('names the one thing being deleted', async () => { + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteDialogTitle).toContain('report.docx'); + expect(view.deleteDialogMessage).toContain('report.docx'); + }); + + it('counts them when there are several', async () => { + actions.selectedItems = ref([FILE, FOLDER]); + fileStore.selectedItems = [FILE, FOLDER]; + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteDialogTitle).toContain('"count":2'); + expect(view.deleteDialogMessage).toContain('"count":2'); + }); + + it('falls back to saying nothing in particular when nothing is pending', async () => { + const { view } = await openOn(FILE); + + expect(view.deleteDialogTitle).toBe('context.deleteTitle.generic'); + expect(view.deleteDialogMessage).toBe('context.deleteMessage.generic'); + }); + + /** + * Deleting a file that a share points at breaks the share, and the person + * deleting it is the only one who can weigh that. + */ + it('warns that shares point at what is about to go', async () => { + getDeleteImpact.mockResolvedValue({ shareCount: 3, shares: [] }); + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteShareImpactMessage).toContain('"count":3'); + }); + + /** Into the trash, a share link stops working but is kept for a restore. */ + it('says the share links can come back when it goes to the trash', async () => { + getDeleteImpact.mockResolvedValue({ + shareCount: 2, + shares: [], + trash: { + enabled: true, + retentionDays: 30, + items: [{ path: 'x', disposition: 'trash', reason: null, shareCount: 2 }], + }, + }); + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteShareImpactMessage).toContain('context.deleteLinkedSharesTrash'); + expect(view.deleteShareImpactMessage).toContain('"count":2'); + expect(view.deleteShareImpactMessage).not.toContain('Permanent'); + }); + + it('says the share links go for good with what cannot go to the trash', async () => { + getDeleteImpact.mockResolvedValue({ + shareCount: 1, + shares: [], + trash: { + enabled: true, + retentionDays: 30, + items: [{ path: 'x', disposition: 'permanent', reason: null, shareCount: 1 }], + }, + }); + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteShareImpactMessage).toContain('context.deleteLinkedSharesPermanent'); + expect(view.deleteShareImpactMessage).not.toContain('LinkedSharesTrash'); + }); + + it('says nothing about shares when none point at it', async () => { + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteShareImpactMessage).toBe(''); + }); + + it('reports a check it could not make', async () => { + getDeleteImpact.mockRejectedValue(new Error('Share service unreachable')); + vi.spyOn(console, 'error').mockImplementation(() => {}); + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteImpactError).toBe('Share service unreachable'); + }); + + /** With the trash on, the dialog says the item goes there, and for how long. */ + it('says one item goes to the trash, and for how long', async () => { + getDeleteImpact.mockResolvedValue({ + shareCount: 0, + shares: [], + trash: { + enabled: true, + retentionDays: 30, + items: [{ path: 'Docs/report.docx', disposition: 'trash', reason: null }], + }, + }); + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteDialogMessage).toBe( + 'context.deleteMessage.trashSingle {"name":"report.docx","count":30}' + ); + expect(view.goesToTrash).toBe(true); + expect(view.deletePermanentNotice).toBe(''); + }); + + it('says several items go to the trash', async () => { + actions.selectedItems = ref([FILE, FOLDER]); + fileStore.selectedItems = [FILE, FOLDER]; + getDeleteImpact.mockResolvedValue({ + shareCount: 0, + shares: [], + trash: { + enabled: true, + retentionDays: 7, + items: [ + { path: 'Docs/report.docx', disposition: 'trash', reason: null }, + { path: 'Docs/2026', disposition: 'trash', reason: null }, + ], + }, + }); + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteDialogMessage).toBe( + 'context.deleteMessage.trashMultiple {"items":2,"count":7}' + ); + }); + + /** Never gone for good without being told: which ones, and why. */ + it('warns which items will be gone for good although the trash is on', async () => { + actions.selectedItems = ref([FILE, FOLDER]); + fileStore.selectedItems = [FILE, FOLDER]; + getDeleteImpact.mockResolvedValue({ + shareCount: 0, + shares: [], + trash: { + enabled: true, + retentionDays: 30, + items: [ + { path: 'Docs/report.docx', disposition: 'trash', reason: null }, + { path: 'Docs/2026', disposition: 'permanent', reason: 'other-device' }, + ], + }, + }); + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deletePermanentNotice).toBe( + 'context.deleteSomePermanent {"count":1} context.trashReasons.otherDevice' + ); + // Not the trash wording: something here is permanent. + expect(view.deleteDialogMessage).toContain('context.deleteMessage.multiple'); + expect(view.goesToTrash).toBe(true); + }); + + /** Before the server answers, "irreversible" would be wrong for nearly every item. */ + it('does not call a deletion irreversible while the trash is on and the answer is pending', async () => { + features.trashEnabled = true; + features.trashRetentionDays = 30; + getDeleteImpact.mockReturnValue(new Promise(() => {})); + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteDialogMessage).toBe( + 'context.deleteMessage.trashSingle {"name":"report.docx","count":30}' + ); + expect(view.goesToTrash).toBe(true); + }); + + it('keeps the permanent wording while pending when the trash is off', async () => { + features.trashEnabled = false; + getDeleteImpact.mockReturnValue(new Promise(() => {})); + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteDialogMessage).toContain('context.deleteMessage.single'); + expect(view.goesToTrash).toBe(false); + }); + + it('keeps the permanent wording when the trash is off', async () => { + getDeleteImpact.mockResolvedValue({ + shareCount: 0, + shares: [], + trash: { + enabled: false, + retentionDays: 30, + items: [{ path: 'Docs/report.docx', disposition: 'permanent', reason: 'disabled' }], + }, + }); + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteDialogMessage).toContain('context.deleteMessage.single'); + expect(view.goesToTrash).toBe(false); + expect(view.deletePermanentNotice).toBe(''); + }); + + it('explains how much too large an item turned away by the trash was', async () => { + const { view } = await openOn(FILE); + + expect( + view.keptItemReason({ + reason: 'too-large', + size: 50 * 1024 ** 2, + budgetBytes: 20 * 1024 ** 2, + }) + ).toBe('context.keptReasons.tooLarge {"size":"50 MB","budget":"20 MB"}'); + expect(view.keptItemReason({ reason: 'zone-root' })).toBe('context.trashReasons.zoneRoot'); + }); + + /** Somebody may have unsaved work in it, open in another window right now. */ + it('warns that a document is open in the editor', async () => { + const open = { ...FILE, onlyofficeActivity: { active: true } }; + actions.selectedItems = ref([open]); + fileStore.selectedItems = [open]; + const { view } = await openOn(open); + + await clickLabel('common.delete'); + + expect(view.deleteOnlyOfficeActivityMessage).toBe( + 'context.deleteOnlyofficeOpen {"names":"report.docx"}' + ); + }); + + it('names the first two and counts the rest', async () => { + const open = ['a.docx', 'b.docx', 'c.docx', 'd.docx'].map((name) => ({ + name, + path: 'Docs', + kind: 'docx', + onlyofficeActivity: { active: true }, + })); + actions.selectedItems = ref(open); + fileStore.selectedItems = open; + const { view } = await openOn(open[0]); + + await clickLabel('common.delete'); + + const message = view.deleteOnlyOfficeActivityMessage; + expect(message.startsWith('context.deleteOnlyofficeOpen')).toBe(true); + expect(message).toContain('a.docx, b.docx onlyoffice.andOthers'); + expect(message).toContain('\\"count\\":2'); + expect(message).not.toContain('c.docx'); + }); + + it('says nothing when none of them is open', async () => { + const { view } = await openOn(FILE); + + await clickLabel('common.delete'); + + expect(view.deleteOnlyOfficeActivityMessage).toBe(''); + }); +}); + +describe('what the inline quick actions offer', () => { + const available = async (item, ids, overrides = {}) => { + actions = makeActions(overrides); + const { api } = await mountMenu(); + return ids.filter((id) => api.quickActionAvailable(item, id)); + }; + + const ALL = [ + 'info', + 'copyName', + 'copyPath', + 'copy', + 'download', + 'cut', + 'rename', + 'share', + 'compress', + 'favorite', + 'delete', + ]; + + /** A volume is not a file: it cannot be cut, renamed, shared or deleted. */ + it('offers a volume only what makes sense on a volume', async () => { + const offered = await available({ name: 'media', kind: 'volume' }, ALL); + + expect(offered).toEqual(['info', 'copyName']); + }); + + it('offers a folder everything but nothing extra', async () => { + const offered = await available(FOLDER, ALL); + + expect(offered).toEqual(ALL); + }); + + it('does not offer to favourite a file', async () => { + const offered = await available(FILE, ALL); + + expect(offered).not.toContain('favorite'); + }); + + it('offers nothing at all for nothing at all', async () => { + const offered = await available(null, ALL); + + expect(offered).toEqual([]); + }); + + it('does not invent an action nobody asked for', async () => { + const offered = await available(FILE, ['format-drive']); + + expect(offered).toEqual([]); + }); + + it('withholds deleting where the location forbids it', async () => { + const offered = await available(FILE, ALL, { locationCanDelete: ref(false) }); + + expect(offered).not.toContain('delete'); + }); + + /** Cutting is a move: it needs the right to write there and to remove from here. */ + it('withholds cutting unless both halves of a move are allowed', async () => { + expect(await available(FILE, ['cut'], { locationCanDelete: ref(false) })).toEqual([]); + expect(await available(FILE, ['cut'], { locationCanWrite: ref(false) })).toEqual([]); + }); + + it('withholds renaming and compressing on a read-only location', async () => { + const offered = await available(FILE, ALL, { locationCanWrite: ref(false) }); + + expect(offered).not.toContain('rename'); + expect(offered).not.toContain('compress'); + }); + + it('still offers reading it, on a read-only location', async () => { + const offered = await available(FILE, ALL, { locationCanWrite: ref(false) }); + + expect(offered).toEqual(expect.arrayContaining(['info', 'copyPath', 'copy', 'download'])); + }); +}); + +describe('running a quick action', () => { + const clipboard = { writeText: vi.fn(async () => {}) }; + + beforeEach(() => { + clipboard.writeText.mockClear(); + Object.defineProperty(navigator, 'clipboard', { configurable: true, value: clipboard }); + }); + + /** The run functions act on the selection, so the item has to be in it first. */ + it('acts on the item it was given, not on whatever was selected', async () => { + fileStore.selectedItems = [FOLDER]; + const { api } = await mountMenu(); + + await api.runQuickAction(FILE, 'copy'); + + expect(fileStore.selectedItems).toEqual([FILE]); + expect(actions.runCopy).toHaveBeenCalled(); + }); + + it('leaves a selection alone when the item is already in it', async () => { + fileStore.selectedItems = [FILE, FOLDER]; + const { api } = await mountMenu(); + + await api.runQuickAction(FILE, 'copy'); + + expect(fileStore.selectedItems).toEqual([FILE, FOLDER]); + }); + + it('copies the name on its own', async () => { + const { api } = await mountMenu(); + + await api.runQuickAction(FILE, 'copyName'); + + expect(clipboard.writeText).toHaveBeenCalledWith('report.docx'); + }); + + it('copies the whole path', async () => { + const { api } = await mountMenu(); + + await api.runQuickAction(FILE, 'copyPath'); + + expect(clipboard.writeText).toHaveBeenCalledWith('Docs/report.docx'); + }); + + /** An insecure context has no clipboard; that is not a reason to throw. */ + it('says nothing when the browser will not give up its clipboard', async () => { + clipboard.writeText.mockRejectedValueOnce(new Error('denied')); + const { api } = await mountMenu(); + + await expect(api.runQuickAction(FILE, 'copyName')).resolves.toBeUndefined(); + }); + + it.each([ + ['info', () => expect(infoOpen).toHaveBeenCalled()], + ['download', () => expect(actions.runDownload).toHaveBeenCalled()], + ['cut', () => expect(actions.runCut).toHaveBeenCalled()], + ['rename', () => expect(actions.runRename).toHaveBeenCalled()], + ['compress', () => expect(actions.runCompressToZip).toHaveBeenCalled()], + ])('runs %s', async (id, assert) => { + const { api } = await mountMenu(); + + await api.runQuickAction(FILE, id); + + assert(); + }); + + it('asks before deleting rather than deleting', async () => { + const { api, wrapper } = await mountMenu(); + + await api.runQuickAction(FILE, 'delete'); + + expect(actions.deleteNow).not.toHaveBeenCalled(); + expect(wrapper.vm.isDeleteConfirmOpen).toBe(true); + }); + + it('does nothing for an action that does not exist', async () => { + const { api } = await mountMenu(); + + await api.runQuickAction(FILE, 'format-drive'); + + expect(actions.runCopy).not.toHaveBeenCalled(); + }); + + it('does nothing at all without an item', async () => { + const { api } = await mountMenu(); + + await api.runQuickAction(null, 'copy'); + + expect(actions.runCopy).not.toHaveBeenCalled(); + }); +}); + +describe('marking a folder as a favourite', () => { + it('adds it, and opens the editor so it can be named', async () => { + favorites.addFavorite.mockResolvedValue({ id: 'f1', path: 'Docs/2026' }); + await openOn(FOLDER); + + await clickLabel('context.addToFavorites'); + + expect(favorites.addFavorite).toHaveBeenCalledWith({ path: 'Docs/2026' }); + expect(openEditorForFavorite).toHaveBeenCalledWith({ id: 'f1', path: 'Docs/2026' }); + }); + + it('removes one that is already there', async () => { + favorites.isFavorite.mockReturnValue(true); + await openOn(FOLDER); + + await clickLabel('context.removeFromFavorites'); + + expect(favorites.removeFavorite).toHaveBeenCalledWith('Docs/2026'); + }); + + it('opens no editor when the server did not create one', async () => { + favorites.addFavorite.mockResolvedValue(null); + await openOn(FOLDER); + + await clickLabel('context.addToFavorites'); + + expect(openEditorForFavorite).not.toHaveBeenCalled(); + }); + + it('marks the folder being looked at, from the background menu', async () => { + await openOn(null, 'background'); + + await clickLabel('context.addToFavorites'); + + expect(favorites.addFavorite).toHaveBeenCalledWith({ path: 'Docs' }); + }); + + /** A second click while the first is in flight would add it twice. */ + it('ignores a second click while the first is still going', async () => { + let release; + favorites.addFavorite.mockReturnValue( + new Promise((resolve) => { + release = resolve; + }) + ); + const { view } = await openOn(FOLDER); + + const first = view.runToggleFavoriteForDirectory(); + await view.runToggleFavoriteForDirectory(); + release({ id: 'f1' }); + await first; + + expect(favorites.addFavorite).toHaveBeenCalledTimes(1); + }); + + it('marks nothing from a file', async () => { + const { view } = await openOn(FILE); + + await view.runToggleFavoriteForDirectory(); + + expect(favorites.addFavorite).not.toHaveBeenCalled(); + }); +}); + +describe('opening a file in the terminal', () => { + const SCRIPT = { name: 'backup.sh', path: 'Docs/bin', kind: 'sh' }; + + it('opens it in the folder the file lives in', async () => { + actions = makeActions({ primaryItem: ref(SCRIPT), selectedItems: ref([SCRIPT]) }); + await openOn(SCRIPT); + + await clickLabel('context.openWithTerminal'); + + expect(terminalOpen).toHaveBeenCalledWith('Docs/bin', './backup.sh'); + }); + + /** A name with a space in it must not become two arguments. */ + it('quotes a name the shell would otherwise split', async () => { + const spaced = { name: 'my backup.sh', path: 'Docs', kind: 'sh' }; + actions = makeActions({ primaryItem: ref(spaced), selectedItems: ref([spaced]) }); + await openOn(spaced); + + await clickLabel('context.openWithTerminal'); + + expect(terminalOpen).toHaveBeenCalledWith('Docs', './my\\ backup.sh'); + }); + + it('falls back to the folder on screen for a file with no path of its own', async () => { + const loose = { name: 'run.sh', kind: 'sh' }; + actions = makeActions({ primaryItem: ref(loose), selectedItems: ref([loose]) }); + await openOn(loose); + + await clickLabel('context.openWithTerminal'); + + expect(terminalOpen).toHaveBeenCalledWith('Docs', './run.sh'); + }); +}); + +describe('opening a file in the editor', () => { + it('goes to the editor on that file', async () => { + await openOn(FILE); + + await clickLabel('context.openWithEditor'); + + expect(routerPush).toHaveBeenCalledWith({ path: '/editor/Docs/report.docx' }); + }); + + /** + * A name is not a URL. A `#` in it would cut the path short, and a `?` would + * turn the rest of the name into a query — the editor would open the wrong + * file, or none. + */ + it('encodes a name a URL would otherwise swallow', async () => { + const awkward = { name: 'notes #1 & co?.md', path: 'Docs', kind: 'md' }; + actions = makeActions({ primaryItem: ref(awkward), selectedItems: ref([awkward]) }); + await openOn(awkward); + + await clickLabel('context.openWithEditor'); + + expect(routerPush).toHaveBeenCalledWith({ + path: '/editor/Docs/notes%20%231%20%26%20co%3F.md', + }); + }); + + it('keeps the folders apart while encoding them', async () => { + const nested = { name: 'a.md', path: 'My Docs/2026 #2', kind: 'md' }; + actions = makeActions({ primaryItem: ref(nested), selectedItems: ref([nested]) }); + await openOn(nested); + + await clickLabel('context.openWithEditor'); + + expect(routerPush).toHaveBeenCalledWith({ path: '/editor/My%20Docs/2026%20%232/a.md' }); + }); +}); + +describe('an archive that wants a password', () => { + const ZIP = { name: 'photos.zip', path: 'Docs', kind: 'zip' }; + + const openZip = async (extractResult) => { + actions = makeActions({ + primaryItem: ref(ZIP), + selectedItems: ref([ZIP]), + isArchiveSelected: ref(true), + canExtractArchive: ref(true), + runExtractArchive: vi.fn(async () => extractResult), + }); + fileStore.extractZipArchive = vi.fn(async () => ({})); + return openOn(ZIP); + }; + + it('asks for one when the archive turns out to be locked', async () => { + const { view } = await openZip({ + requiresPassword: true, + path: 'Docs/photos.zip', + destination: 'Docs', + }); + + await clickLabel('actions.extractArchive'); + + expect(view.archivePasswordRequest).toEqual({ + path: 'Docs/photos.zip', + destination: 'Docs', + invalidPassword: undefined, + }); + }); + + it('asks for nothing when the archive opens on its own', async () => { + const { view } = await openZip({ requiresPassword: false }); + + await clickLabel('actions.extractArchive'); + + expect(view.archivePasswordRequest).toBeNull(); + }); + + it('extracts with the password it was given', async () => { + const { view } = await openZip({ + requiresPassword: true, + path: 'Docs/photos.zip', + destination: 'Docs', + }); + await clickLabel('actions.extractArchive'); + + await view.submitArchivePassword('hunter2'); + + expect(fileStore.extractZipArchive).toHaveBeenCalledWith('Docs/photos.zip', { + destination: 'Docs', + password: 'hunter2', + }); + expect(view.archivePasswordRequest).toBeNull(); + }); + + /** A wrong password is a reason to ask again, not to give up silently. */ + it('asks again, saying so, when the password was wrong', async () => { + const { view } = await openZip({ + requiresPassword: true, + path: 'Docs/photos.zip', + destination: 'Docs', + }); + await clickLabel('actions.extractArchive'); + fileStore.extractZipArchive.mockResolvedValue({ + requiresPassword: true, + invalidPassword: true, + }); + + await view.submitArchivePassword('wrong'); + + expect(view.archivePasswordRequest).toMatchObject({ invalidPassword: true }); + }); + + it('cannot be dismissed while it is still trying', async () => { + const { view } = await openZip({ + requiresPassword: true, + path: 'Docs/photos.zip', + destination: 'Docs', + }); + await clickLabel('actions.extractArchive'); + let release; + fileStore.extractZipArchive.mockReturnValue( + new Promise((resolve) => { + release = resolve; + }) + ); + + const pending = view.submitArchivePassword('hunter2'); + view.closeArchivePasswordDialog(); + + expect(view.archivePasswordRequest).not.toBeNull(); + release({}); + await pending; + }); + + it('can be dismissed once it is not', async () => { + const { view } = await openZip({ + requiresPassword: true, + path: 'Docs/photos.zip', + destination: 'Docs', + }); + await clickLabel('actions.extractArchive'); + + view.closeArchivePasswordDialog(); + + expect(view.archivePasswordRequest).toBeNull(); + }); + + it('does nothing when asked for a password it never wanted', async () => { + const { view } = await openZip({ requiresPassword: false }); + + await view.submitArchivePassword('hunter2'); + + expect(fileStore.extractZipArchive).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/src/components/ExplorerContextMenu.vue b/frontend/src/components/ExplorerContextMenu.vue index 090f1b0f8..e11142ab3 100644 --- a/frontend/src/components/ExplorerContextMenu.vue +++ b/frontend/src/components/ExplorerContextMenu.vue @@ -1,9 +1,11 @@ @@ -651,7 +548,7 @@ provide(explorerContextMenuSymbol, { v-if="isOpen" ref="floatingRef" :style="floatingStyles" - class="min-w-[220px] rounded-xl border border-white/10 bg-white/70 p-1.5 text-sm text-zinc-800 shadow-2xl backdrop-blur-xl dark:border-white/10 dark:bg-neutral-800/70 dark:text-zinc-200" + class="min-w-[220px] rounded-xl border border-zinc-200 bg-white p-1.5 text-sm text-zinc-800 shadow-2xl dark:border-white/10 dark:bg-neutral-800 dark:text-zinc-200" @contextmenu.prevent @click.stop > @@ -685,7 +582,7 @@ provide(explorerContextMenuSymbol, { - +

{{ deleteDialogMessage }} @@ -694,7 +591,13 @@ provide(explorerContextMenuSymbol, { {{ $t('context.checkingDeleteImpact') }}

+ {{ deleteOnlyOfficeActivityMessage }} +

+

{{ deleteShareImpactMessage }} @@ -702,26 +605,92 @@ provide(explorerContextMenuSymbol, {

{{ $t('context.deleteImpactUnavailable') }}

+

+ {{ deletePermanentNotice }} +

+

+ {{ deleteVersionsNotice }} +

+
+ + +

{{ keptDialogMessage }}

+
    +
  • + {{ item.name }} + — {{ keptItemReason(item) }} +
    • +
+
+ + +
+ + + + diff --git a/frontend/src/components/FavMenu.spec.js b/frontend/src/components/FavMenu.spec.js new file mode 100644 index 000000000..39226a4fa --- /dev/null +++ b/frontend/src/components/FavMenu.spec.js @@ -0,0 +1,289 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mount, flushPromises } from '@vue/test-utils'; +import { defineComponent, h, reactive, ref } from 'vue'; + +/** + * The favourites in the sidebar. + * + * Every button in it acts on one favourite, and the favourites look alike: a + * remove that reaches the neighbour of the one clicked deletes the wrong + * shortcut, a drop that lands on the parent of the folder shown moves files + * somewhere nobody chose. Remove and edit stay hidden until edit mode is on, so + * a stray click while navigating cannot remove anything; edit mode has to end + * when the pointer goes elsewhere, and a reorder the server refuses must not + * leave the list showing an order that was never saved. + */ + +let favoritesStore; +let dragDrop; +const route = reactive({ params: {} }); +const openBreadcrumb = vi.fn(); +const openEditorForFavorite = vi.fn(); + +vi.mock('@/stores/favorites', () => ({ useFavoritesStore: () => favoritesStore })); +vi.mock('vue-router', () => ({ useRoute: () => route })); +vi.mock('@/composables/navigation', () => ({ useNavigation: () => ({ openBreadcrumb }) })); +vi.mock('@/composables/useFavoriteEditor', () => ({ + useFavoriteEditor: () => ({ openEditorForFavorite }), +})); +vi.mock('@/composables/useFileDragDrop', () => ({ useFileDragDrop: () => dragDrop })); +vi.mock('@/api', () => ({ + normalizePath: (value) => String(value || '').replace(/^\/+|\/+$/g, ''), +})); +vi.mock('vue-i18n', async (importOriginal) => ({ + ...(await importOriginal()), + useI18n: () => ({ t: (key) => key }), +})); +/** + * The drag library, reduced to what the menu relies on: it renders one item per + * favourite, knows whether dragging is allowed, and reports a new order. + */ +vi.mock('vuedraggable', () => ({ + default: defineComponent({ + name: 'DraggableStub', + props: { modelValue: Array, disabled: Boolean }, + emits: ['update:modelValue', 'end'], + setup(props, { slots }) { + return () => + h( + 'div', + props.modelValue.map((element) => h('div', { key: element.id }, slots.item({ element }))) + ); + }, + }), +})); + +import FavMenu from './FavMenu.vue'; + +const PROJECTS = { id: 'fav-projects', path: 'Docs/2026', label: '', icon: 'outline:FolderIcon' }; +const PHOTOS = { id: 'fav-photos', path: 'Media/Photos', label: 'Holiday pictures' }; +const ARCHIVE = { id: 'fav-archive', path: 'Backup/Archive', label: '', available: false }; + +let wrapper = null; + +const mountMenu = async (list) => { + favoritesStore.favorites = list; + wrapper = mount(FavMenu, { attachTo: document.body }); + await flushPromises(); + return wrapper; +}; + +const editModeButton = () => wrapper.find('h4 button'); +const favourite = (label) => + wrapper.findAll('button').find((button) => button.text().trim() === label); +const names = () => + wrapper + .findAll('button') + .map((button) => button.find('span.truncate')) + .filter((span) => span.exists()) + .map((span) => span.text()); +const editButtons = () => wrapper.findAll('button[aria-label="common.edit"]'); +const removeButtons = () => wrapper.findAll('button[aria-label="common.remove"]'); +const draggable = () => wrapper.findComponent({ name: 'DraggableStub' }); + +const enterEditMode = async () => { + await editModeButton().trigger('click'); + await flushPromises(); +}; + +beforeEach(() => { + // A `ref` inside a reactive object is what `storeToRefs` finds in a real store. + favoritesStore = reactive({ + favorites: ref([]), + ensureLoaded: vi.fn(async () => {}), + loadFavorites: vi.fn(async () => {}), + removeFavorite: vi.fn(async () => {}), + reorderFavorites: vi.fn(async () => {}), + }); + dragDrop = { + handleDragOver: vi.fn(), + handleDragLeave: vi.fn(), + handleDrop: vi.fn(), + isDragTarget: vi.fn(() => false), + isCopyDragTarget: vi.fn(() => false), + }; + route.params = {}; + [openBreadcrumb, openEditorForFavorite].forEach((fn) => fn.mockClear()); +}); + +afterEach(() => { + wrapper?.unmount(); + wrapper = null; + document.body.innerHTML = ''; + vi.restoreAllMocks(); +}); + +describe('with no favourites', () => { + it('loads them when it appears', async () => { + await mountMenu([]); + + expect(favoritesStore.ensureLoaded).toHaveBeenCalledTimes(1); + }); + + it('says there are none, and offers nothing to edit', async () => { + await mountMenu([]); + + expect(wrapper.text()).toContain('favorites.emptyTitle'); + expect(names()).toEqual([]); + expect(editModeButton().attributes('disabled')).toBeDefined(); + expect(editButtons()).toHaveLength(0); + expect(removeButtons()).toHaveLength(0); + }); +}); + +describe('the list', () => { + it('names each favourite by its label, or by the last part of its path', async () => { + await mountMenu([PROJECTS, PHOTOS]); + + expect(names()).toEqual(['2026', 'Holiday pictures']); + expect(wrapper.text()).not.toContain('favorites.emptyTitle'); + }); + + it('opens the folder of the favourite clicked', async () => { + await mountMenu([PROJECTS, PHOTOS]); + + await favourite('Holiday pictures').trigger('click'); + + expect(openBreadcrumb).toHaveBeenCalledTimes(1); + expect(openBreadcrumb).toHaveBeenCalledWith('Media/Photos'); + }); + + it('goes nowhere for a favourite that has no path', async () => { + await mountMenu([{ id: 'broken', path: '', label: 'Broken' }]); + + await favourite('Broken').trigger('click'); + + expect(openBreadcrumb).not.toHaveBeenCalled(); + }); + + it('warns about a favourite whose volume is not mounted, and only that one', async () => { + await mountMenu([PROJECTS, ARCHIVE]); + + expect(favourite('Archive').attributes('title')).toBe('favorites.volumeUnavailable'); + expect(favourite('2026').attributes('title')).toBeUndefined(); + }); +}); + +describe('edit mode', () => { + it('offers no edit or remove until it is switched on', async () => { + await mountMenu([PROJECTS, PHOTOS]); + expect(editButtons()).toHaveLength(0); + expect(removeButtons()).toHaveLength(0); + + await enterEditMode(); + + expect(editButtons()).toHaveLength(2); + expect(removeButtons()).toHaveLength(2); + }); + + it('removes the favourite whose remove button was clicked', async () => { + await mountMenu([PROJECTS, PHOTOS]); + await enterEditMode(); + + await removeButtons()[1].trigger('click'); + await flushPromises(); + + expect(favoritesStore.removeFavorite).toHaveBeenCalledTimes(1); + expect(favoritesStore.removeFavorite).toHaveBeenCalledWith('Media/Photos'); + expect(openBreadcrumb).not.toHaveBeenCalled(); + }); + + it('opens the editor on the favourite whose edit button was clicked', async () => { + await mountMenu([PROJECTS, PHOTOS]); + await enterEditMode(); + + await editButtons()[0].trigger('click'); + + expect(openEditorForFavorite).toHaveBeenCalledTimes(1); + expect(openEditorForFavorite).toHaveBeenCalledWith(PROJECTS); + }); + + it('survives a removal the server refuses', async () => { + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + favoritesStore.removeFavorite.mockRejectedValue(new Error('Forbidden')); + await mountMenu([PROJECTS]); + await enterEditMode(); + + await removeButtons()[0].trigger('click'); + await flushPromises(); + + expect(consoleError).toHaveBeenCalled(); + expect(removeButtons()).toHaveLength(1); + }); + + it('ends when the pointer goes down elsewhere on the page', async () => { + await mountMenu([PROJECTS, PHOTOS]); + await enterEditMode(); + + document.body.dispatchEvent(new Event('pointerdown', { bubbles: true })); + await flushPromises(); + + expect(removeButtons()).toHaveLength(0); + }); + + it('carries on when the pointer goes down inside the list', async () => { + await mountMenu([PROJECTS, PHOTOS]); + await enterEditMode(); + + favourite('2026').element.dispatchEvent(new Event('pointerdown', { bubbles: true })); + await flushPromises(); + + expect(removeButtons()).toHaveLength(2); + }); +}); + +describe('reordering', () => { + it('is only possible in edit mode', async () => { + await mountMenu([PROJECTS, PHOTOS]); + expect(draggable().props('disabled')).toBe(true); + + await enterEditMode(); + + expect(draggable().props('disabled')).toBe(false); + }); + + it('is not possible with a single favourite', async () => { + await mountMenu([PROJECTS]); + await enterEditMode(); + + expect(draggable().props('disabled')).toBe(true); + }); + + it('saves the order the favourites were dropped in', async () => { + await mountMenu([PROJECTS, PHOTOS]); + await enterEditMode(); + + draggable().vm.$emit('update:modelValue', [PHOTOS, PROJECTS]); + draggable().vm.$emit('end'); + await flushPromises(); + + expect(favoritesStore.reorderFavorites).toHaveBeenCalledWith(['fav-photos', 'fav-projects']); + expect(favoritesStore.loadFavorites).not.toHaveBeenCalled(); + }); + + it('reloads the saved order when the new one is refused', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}); + favoritesStore.reorderFavorites.mockRejectedValue(new Error('Conflict')); + await mountMenu([PROJECTS, PHOTOS]); + await enterEditMode(); + + draggable().vm.$emit('update:modelValue', [PHOTOS, PROJECTS]); + draggable().vm.$emit('end'); + await flushPromises(); + + expect(favoritesStore.loadFavorites).toHaveBeenCalledTimes(1); + }); +}); + +describe('dropping files onto a favourite', () => { + it('targets the favourite folder itself, not its parent', async () => { + await mountMenu([PROJECTS, PHOTOS]); + + await favourite('Holiday pictures').trigger('dragover'); + await favourite('Holiday pictures').trigger('drop'); + + const target = { name: 'Photos', path: 'Media', destinationPath: 'Media/Photos' }; + expect(dragDrop.handleDragOver).toHaveBeenCalledWith(expect.any(Event), target); + expect(dragDrop.handleDrop).toHaveBeenCalledWith(expect.any(Event), target); + }); +}); diff --git a/frontend/src/components/FavMenu.vue b/frontend/src/components/FavMenu.vue index 841bb3555..3794b8e4c 100644 --- a/frontend/src/components/FavMenu.vue +++ b/frontend/src/components/FavMenu.vue @@ -2,7 +2,7 @@ import { computed, onBeforeUnmount, onMounted, ref } from 'vue'; import { useRoute } from 'vue-router'; import * as OutlineIcons from '@heroicons/vue/24/outline'; -import * as SolidIcons from '@heroicons/vue/24/solid'; +import { ExclamationTriangleIcon } from '@heroicons/vue/24/outline'; import { storeToRefs } from 'pinia'; import draggable from 'vuedraggable'; import { useFavoritesStore } from '@/stores/favorites'; @@ -10,6 +10,8 @@ import { useNavigation } from '@/composables/navigation'; import { normalizePath } from '@/api'; import { useI18n } from 'vue-i18n'; import { useFavoriteEditor } from '@/composables/useFavoriteEditor'; +import { resolveFavoriteIcon } from '@/utils/favoriteIcons'; +import { useFileDragDrop } from '@/composables/useFileDragDrop'; const { ChevronDownIcon, @@ -27,35 +29,10 @@ const { favorites } = storeToRefs(favoritesStore); const route = useRoute(); const { openBreadcrumb } = useNavigation(); const { openEditorForFavorite } = useFavoriteEditor(); +const { handleDragOver, handleDragLeave, handleDrop, isDragTarget, isCopyDragTarget } = + useFileDragDrop(); -const ICON_VARIANTS = { - outline: OutlineIcons, - solid: SolidIcons, -}; - -const resolveIconComponent = (iconName) => { - if (typeof iconName !== 'string') { - return StarIconOutline; - } - - const trimmed = iconName.trim(); - if (!trimmed) { - return StarIconOutline; - } - - if (trimmed.includes(':')) { - const [variantRaw, iconRaw] = trimmed.split(':', 2); - const variantKey = variantRaw.toLowerCase(); - const iconKey = iconRaw.trim(); - const registry = ICON_VARIANTS[variantKey]; - if (registry && registry[iconKey]) { - return registry[iconKey]; - } - } - - return OutlineIcons[trimmed] || SolidIcons[trimmed] || StarIconOutline; -}; - +const resolveIconComponent = resolveFavoriteIcon; const getFavoriteLabel = (favorite = {}) => { const path = favorite.path || ''; const autoLabel = path.split('/').pop() || path; @@ -87,6 +64,19 @@ const handleOpenFavorite = (favorite) => { openBreadcrumb(favorite.path); }; +const favoriteDropTarget = (favorite = {}) => { + const destinationPath = normalizePath(favorite.path || ''); + const segments = destinationPath.split('/').filter(Boolean); + return { + name: segments.at(-1) || '', + path: segments.slice(0, -1).join('/'), + destinationPath, + }; +}; + +const isFavoriteDragTarget = (favorite) => isDragTarget(favoriteDropTarget(favorite)); +const isFavoriteCopyTarget = (favorite) => isCopyDragTarget(favoriteDropTarget(favorite)); + const toggleEditMode = () => { if (!favorites.value.length) return; isEditMode.value = !isEditMode.value; @@ -156,6 +146,7 @@ onBeforeUnmount(() => { {{ t('common.edit') }}