diff --git a/.env.example b/.env.example index 2579a4f..173a077 100644 --- a/.env.example +++ b/.env.example @@ -53,6 +53,15 @@ UPLOAD_MAX_SIZE=4g # large video; short enough that a leaked URL stops working. Default 1 hour. MEDIA_URL_TTL_SECONDS=3600 +# ─── Disk space ────────────────────────────────────────────────────────────── +# +# Free space, in GB, on the disk holding media (and, if separate, the database and the +# container's temp space, where uploads and imports land first). Below the first, the +# header shows a warning; below the second, uploads, imports, generations and clip +# extractions are refused until space is freed. Settings → Storage shows the numbers. +DISK_WARN_FREE_GB=20 +DISK_RESERVE_GB=5 + # ─── Background jobs ───────────────────────────────────────────────────────── JOB_HEARTBEAT_SECONDS=30 @@ -85,7 +94,8 @@ URL_IMPORT_COOKIES_FILE= # ─── Administrators ────────────────────────────────────────────────────────── -# Who may edit the AI generation model catalogue: Gecko Notes user ids or usernames, +# Who may edit the AI generation model catalogue, and see the server's disk space and +# clear stray files under Settings → Storage: Gecko Notes user ids or usernames, # comma-separated, matched exactly (case included). Prefer ids: Notes lets anyone rename # themselves, so a username somebody gives up can be taken by somebody else. Notes' # token carries no admin flag, so this is the only way anyone becomes an admin here. diff --git a/README.md b/README.md index cf58cf2..ac27e63 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,9 @@ user row. The changes that makes possible are specified in images, with the prompt and base assets recorded so a result stays reproducible. - **Cost visibility** — an estimate before anything paid runs, and the real cost tracked per asset and per library afterwards. +- **Disk space** — what the library takes up and what the disk has left, a header + warning before it runs out, and new writes refused before a full disk can damage + anything. See [`docs/deployment.md`](docs/deployment.md#disk-space). ## Tech stack diff --git a/backend/app/config.py b/backend/app/config.py index 0821f9e..d374653 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -60,6 +60,20 @@ class Settings(BaseSettings): # video, short enough that a leaked URL stops working. media_url_ttl_seconds: int = 3600 + # ─── disk space ────────────────────────────────────────────────────────── + # Absolute sizes, not percentages of the disk. What fills it is single files whose + # size has nothing to do with the disk's: a ninety-minute 1080p import is 1-2 GB on + # a 100 GB VPS or a 10 TB array alike, and one generation may download up to + # `generation_max_download_mb`. Ten percent of an 8 TB disk would cry wolf at + # 800 GB free. + # + # Below `disk_warn_free_gb` the header shows a warning. Below `disk_reserve_gb` + # uploads, imports, generations and extractions are refused, which keeps enough + # room for the writes already in flight and for SQLite, which does not recover + # gracefully from a disk that fills under it. + disk_warn_free_gb: float = 20 + disk_reserve_gb: float = 5 + # ─── http ──────────────────────────────────────────────────────────────── cors_origin: str = "" diff --git a/backend/app/enrichment/extract_subvideo.py b/backend/app/enrichment/extract_subvideo.py index 4a715de..9ab3068 100644 --- a/backend/app/enrichment/extract_subvideo.py +++ b/backend/app/enrichment/extract_subvideo.py @@ -29,7 +29,7 @@ from app.ingest.probe import probe from app.models.asset import Asset from app.services import assets as asset_service -from app.storage import build_storage, new_key, thumb_key_for +from app.storage import build_storage, capacity, new_key, thumb_key_for logger = logging.getLogger(__name__) @@ -72,6 +72,10 @@ def run( except (KeyError, TypeError, ValueError) as exc: raise SubvideoExtractionError("Missing or invalid in/out points") from exc + # Again, although the router already did: the job may have waited in the queue. + if not capacity.has_room(): + raise SubvideoExtractionError(capacity.DISK_FULL_MESSAGE) + storage = build_storage() progress("Extracting", 10, "") diff --git a/backend/app/enrichment/import_url.py b/backend/app/enrichment/import_url.py index 525f965..9449510 100644 --- a/backend/app/enrichment/import_url.py +++ b/backend/app/enrichment/import_url.py @@ -45,7 +45,7 @@ from app.services import suggestions as suggestion_service from app.services import tags as tag_service from app.settings_store import load_deepgram_key -from app.storage import StorageError, build_storage, thumb_key_for +from app.storage import StorageError, build_storage, capacity, thumb_key_for logger = logging.getLogger(__name__) @@ -112,6 +112,11 @@ def run(session: Session, job: EnrichmentJob, progress: Progress) -> ImportResul except HTTPException as exc: raise UrlImportError(readable_error(exc)) from exc + # Again, although the router already did: a job can sit queued behind others while + # the disk fills, and every playlist child arrives here without passing the router. + if not capacity.has_room(): + raise UrlImportError(capacity.DISK_FULL_MESSAGE) + with tempfile.TemporaryDirectory(prefix="gam-import-") as workdir: work = Path(workdir) diff --git a/backend/app/generation/run.py b/backend/app/generation/run.py index 80ad9b2..d44a1f8 100644 --- a/backend/app/generation/run.py +++ b/backend/app/generation/run.py @@ -55,7 +55,7 @@ from app.providers.base import ProviderError from app.services import assets as asset_service from app.settings_store import load_fal_key -from app.storage import StorageError, build_storage +from app.storage import StorageError, build_storage, capacity from app.usage import events as usage_events logger = logging.getLogger(__name__) @@ -166,6 +166,11 @@ def run(session: Session, job: EnrichmentJob, progress: Progress) -> GenerationR if state.result is None: if state.queued is None: + # Before submitting, and only then. fal bills when it runs the request, so this + # is the last moment refusing costs nothing; a request already submitted has + # been paid for, and is let try to save its output rather than thrown away. + if not capacity.has_room(): + raise GenerationError(capacity.DISK_FULL_MESSAGE) body = _request_body(session, job, request, progress) progress(STAGE_SUBMITTING, 10, "") state.queued = fal.submit(request.endpoint_id, body, api_key=api_key) diff --git a/backend/app/main.py b/backend/app/main.py index ff6e948..5ce0071 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -34,6 +34,7 @@ from app.routers import providers as providers_router from app.routers import search as search_router from app.routers import settings as settings_router +from app.routers import storage as storage_router from app.routers import tags as tags_router from app.routers import usage as usage_router from app.routers import transcripts as transcripts_router @@ -186,6 +187,7 @@ def me( app.include_router(providers_router.router, prefix="/api/providers", tags=["providers"]) app.include_router(tags_router.router, prefix="/api/tags", tags=["tags"]) app.include_router(usage_router.router, prefix="/api/usage", tags=["usage"]) +app.include_router(storage_router.router, prefix="/api/storage", tags=["storage"]) # Not under /api: these URLs go straight into and