From a969b1396d8f67affe1240f5799b5b8928a59a69 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 02:34:54 +0000 Subject: [PATCH] feat(storage): show disk usage and free space, warn before it runs out Settings gains a Storage panel, and the header a drive icon that appears only when free space is low. Everyone sees their own library: totals by type, previews, and their ten largest files as links, which is the quickest way to find something to delete. Admins (ADMIN_USERS) also see the server: - one bar per disk, split into GAM media, GAM database, everything else, the filesystem's reserve and free space; inside Docker these are the host's numbers, through the data/ bind mount - stray files: files under data/media that no asset refers to, older than a day, with a button that removes them through storage.delete Below DISK_RESERVE_GB (default 5) uploads, URL imports, generations and sub-video extractions are refused with 507 disk_full. They are checked when queued and again when the job starts. A generation is checked before it is sent to fal, so nothing is billed for an output that cannot be saved, and a request already submitted still saves. An upload stores the files that fit and rejects the rest. DISK_WARN_FREE_GB (default 20) sets the header warning. Both are fixed sizes because single large files are what fill the disk. The container's temp directory is shown when it is a separate disk, but does not count toward the level: nothing persistent lives there, and on a dev machine it is often a small tmpfs that would otherwise block every upload. Tests run against a fake disk (conftest's autouse `disk`), so the suite no longer depends on how full the machine running it is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017Ev6jNsXYxKEPPN6EpqpTE --- .env.example | 12 +- README.md | 3 + backend/app/config.py | 14 + backend/app/enrichment/extract_subvideo.py | 6 +- backend/app/enrichment/import_url.py | 7 +- backend/app/generation/run.py | 7 +- backend/app/main.py | 2 + backend/app/routers/assets.py | 19 +- backend/app/routers/clips.py | 4 +- backend/app/routers/generate.py | 4 +- backend/app/routers/imports.py | 2 + backend/app/routers/storage.py | 70 +++ backend/app/schemas_storage.py | 81 +++ backend/app/services/storage_usage.py | 218 +++++++++ backend/app/storage/capacity.py | 170 +++++++ backend/app/storage/local.py | 30 ++ backend/tests/conftest.py | 39 ++ backend/tests/test_clips_api.py | 16 + backend/tests/test_generation_job.py | 34 ++ backend/tests/test_storage.py | 27 + backend/tests/test_storage_usage.py | 463 ++++++++++++++++++ backend/tests/test_url_import.py | 15 + docker-compose.yml | 2 + docs/deployment.md | 42 ++ frontend/src/api/storage.ts | 106 ++++ frontend/src/components/AppShell.test.tsx | 7 + frontend/src/components/AppShell.tsx | 2 + frontend/src/components/StoragePanel.test.tsx | 248 ++++++++++ frontend/src/components/StoragePanel.tsx | 451 +++++++++++++++++ .../src/components/StorageWarning.test.tsx | 54 ++ frontend/src/components/StorageWarning.tsx | 67 +++ frontend/src/views/SettingsView.test.tsx | 13 + frontend/src/views/SettingsView.tsx | 2 + 33 files changed, 2230 insertions(+), 7 deletions(-) create mode 100644 backend/app/routers/storage.py create mode 100644 backend/app/schemas_storage.py create mode 100644 backend/app/services/storage_usage.py create mode 100644 backend/app/storage/capacity.py create mode 100644 backend/tests/test_storage_usage.py create mode 100644 frontend/src/api/storage.ts create mode 100644 frontend/src/components/StoragePanel.test.tsx create mode 100644 frontend/src/components/StoragePanel.tsx create mode 100644 frontend/src/components/StorageWarning.test.tsx create mode 100644 frontend/src/components/StorageWarning.tsx diff --git a/.env.example b/.env.example index 2579a4f..173a077 100644 --- a/.env.example +++ b/.env.example @@ -53,6 +53,15 @@ UPLOAD_MAX_SIZE=4g # large video; short enough that a leaked URL stops working. Default 1 hour. MEDIA_URL_TTL_SECONDS=3600 +# ─── Disk space ────────────────────────────────────────────────────────────── +# +# Free space, in GB, on the disk holding media (and, if separate, the database and the +# container's temp space, where uploads and imports land first). Below the first, the +# header shows a warning; below the second, uploads, imports, generations and clip +# extractions are refused until space is freed. Settings → Storage shows the numbers. +DISK_WARN_FREE_GB=20 +DISK_RESERVE_GB=5 + # ─── Background jobs ───────────────────────────────────────────────────────── JOB_HEARTBEAT_SECONDS=30 @@ -85,7 +94,8 @@ URL_IMPORT_COOKIES_FILE= # ─── Administrators ────────────────────────────────────────────────────────── -# Who may edit the AI generation model catalogue: Gecko Notes user ids or usernames, +# Who may edit the AI generation model catalogue, and see the server's disk space and +# clear stray files under Settings → Storage: Gecko Notes user ids or usernames, # comma-separated, matched exactly (case included). Prefer ids: Notes lets anyone rename # themselves, so a username somebody gives up can be taken by somebody else. Notes' # token carries no admin flag, so this is the only way anyone becomes an admin here. diff --git a/README.md b/README.md index cf58cf2..ac27e63 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,9 @@ user row. The changes that makes possible are specified in images, with the prompt and base assets recorded so a result stays reproducible. - **Cost visibility** — an estimate before anything paid runs, and the real cost tracked per asset and per library afterwards. +- **Disk space** — what the library takes up and what the disk has left, a header + warning before it runs out, and new writes refused before a full disk can damage + anything. See [`docs/deployment.md`](docs/deployment.md#disk-space). ## Tech stack diff --git a/backend/app/config.py b/backend/app/config.py index 0821f9e..d374653 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -60,6 +60,20 @@ class Settings(BaseSettings): # video, short enough that a leaked URL stops working. media_url_ttl_seconds: int = 3600 + # ─── disk space ────────────────────────────────────────────────────────── + # Absolute sizes, not percentages of the disk. What fills it is single files whose + # size has nothing to do with the disk's: a ninety-minute 1080p import is 1-2 GB on + # a 100 GB VPS or a 10 TB array alike, and one generation may download up to + # `generation_max_download_mb`. Ten percent of an 8 TB disk would cry wolf at + # 800 GB free. + # + # Below `disk_warn_free_gb` the header shows a warning. Below `disk_reserve_gb` + # uploads, imports, generations and extractions are refused, which keeps enough + # room for the writes already in flight and for SQLite, which does not recover + # gracefully from a disk that fills under it. + disk_warn_free_gb: float = 20 + disk_reserve_gb: float = 5 + # ─── http ──────────────────────────────────────────────────────────────── cors_origin: str = "" diff --git a/backend/app/enrichment/extract_subvideo.py b/backend/app/enrichment/extract_subvideo.py index 4a715de..9ab3068 100644 --- a/backend/app/enrichment/extract_subvideo.py +++ b/backend/app/enrichment/extract_subvideo.py @@ -29,7 +29,7 @@ from app.ingest.probe import probe from app.models.asset import Asset from app.services import assets as asset_service -from app.storage import build_storage, new_key, thumb_key_for +from app.storage import build_storage, capacity, new_key, thumb_key_for logger = logging.getLogger(__name__) @@ -72,6 +72,10 @@ def run( except (KeyError, TypeError, ValueError) as exc: raise SubvideoExtractionError("Missing or invalid in/out points") from exc + # Again, although the router already did: the job may have waited in the queue. + if not capacity.has_room(): + raise SubvideoExtractionError(capacity.DISK_FULL_MESSAGE) + storage = build_storage() progress("Extracting", 10, "") diff --git a/backend/app/enrichment/import_url.py b/backend/app/enrichment/import_url.py index 525f965..9449510 100644 --- a/backend/app/enrichment/import_url.py +++ b/backend/app/enrichment/import_url.py @@ -45,7 +45,7 @@ from app.services import suggestions as suggestion_service from app.services import tags as tag_service from app.settings_store import load_deepgram_key -from app.storage import StorageError, build_storage, thumb_key_for +from app.storage import StorageError, build_storage, capacity, thumb_key_for logger = logging.getLogger(__name__) @@ -112,6 +112,11 @@ def run(session: Session, job: EnrichmentJob, progress: Progress) -> ImportResul except HTTPException as exc: raise UrlImportError(readable_error(exc)) from exc + # Again, although the router already did: a job can sit queued behind others while + # the disk fills, and every playlist child arrives here without passing the router. + if not capacity.has_room(): + raise UrlImportError(capacity.DISK_FULL_MESSAGE) + with tempfile.TemporaryDirectory(prefix="gam-import-") as workdir: work = Path(workdir) diff --git a/backend/app/generation/run.py b/backend/app/generation/run.py index 80ad9b2..d44a1f8 100644 --- a/backend/app/generation/run.py +++ b/backend/app/generation/run.py @@ -55,7 +55,7 @@ from app.providers.base import ProviderError from app.services import assets as asset_service from app.settings_store import load_fal_key -from app.storage import StorageError, build_storage +from app.storage import StorageError, build_storage, capacity from app.usage import events as usage_events logger = logging.getLogger(__name__) @@ -166,6 +166,11 @@ def run(session: Session, job: EnrichmentJob, progress: Progress) -> GenerationR if state.result is None: if state.queued is None: + # Before submitting, and only then. fal bills when it runs the request, so this + # is the last moment refusing costs nothing; a request already submitted has + # been paid for, and is let try to save its output rather than thrown away. + if not capacity.has_room(): + raise GenerationError(capacity.DISK_FULL_MESSAGE) body = _request_body(session, job, request, progress) progress(STAGE_SUBMITTING, 10, "") state.queued = fal.submit(request.endpoint_id, body, api_key=api_key) diff --git a/backend/app/main.py b/backend/app/main.py index ff6e948..5ce0071 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -34,6 +34,7 @@ from app.routers import providers as providers_router from app.routers import search as search_router from app.routers import settings as settings_router +from app.routers import storage as storage_router from app.routers import tags as tags_router from app.routers import usage as usage_router from app.routers import transcripts as transcripts_router @@ -186,6 +187,7 @@ def me( app.include_router(providers_router.router, prefix="/api/providers", tags=["providers"]) app.include_router(tags_router.router, prefix="/api/tags", tags=["tags"]) app.include_router(usage_router.router, prefix="/api/usage", tags=["usage"]) +app.include_router(storage_router.router, prefix="/api/storage", tags=["storage"]) # Not under /api: these URLs go straight into and