diff --git a/backend/app/render/cleanup.py b/backend/app/render/cleanup.py new file mode 100644 index 0000000..c767995 --- /dev/null +++ b/backend/app/render/cleanup.py @@ -0,0 +1,91 @@ +"""Remove render rows whose video is gone. + +Until `DATA_DIR` was set in the image, renders were written to `/data`, outside the +volume, so every rebuild deleted the files and kept the rows: a project's render list +then offered videos that 404 on play, and "Save to GAM" refused them. This finds those +rows and, with `--apply`, deletes them. + + docker compose exec gvc-backend python -m app.render.cleanup # list only + docker compose exec gvc-backend python -m app.render.cleanup --apply # delete + +Listing is the default because deleting is not undoable, and a mistyped data directory +would make every render look orphaned: run it once to see that the count is the one +you expect. A render's row is the only thing deleted. A copy already saved to GAM lives +there independently and stays. + +Only finished renders are judged. A failed or cancelled one never had a file, and a +running one is still writing it. +""" + +from __future__ import annotations + +import argparse +import shutil +import sys +from dataclasses import dataclass, field +from typing import List, Optional + +from sqlmodel import Session, select + +from app.models.render import RenderJob +from app.render import worker + + +@dataclass +class Orphans: + rows: List[RenderJob] = field(default_factory=list) + kept: int = 0 + + +def find(session: Session) -> Orphans: + found = Orphans() + for job in session.exec(select(RenderJob).where(RenderJob.status == "done")).all(): + path = worker.renders_dir() / job.id / (job.video_path or "") + if job.video_path and path.is_file(): + found.kept += 1 + else: + found.rows.append(job) + return found + + +def remove(session: Session, rows: List[RenderJob]) -> None: + for job in rows: + # A poster or subtitle can outlive the video it went with. + shutil.rmtree(worker.renders_dir() / job.id, ignore_errors=True) + session.delete(job) + session.commit() + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(prog="python -m app.render.cleanup", + description="Remove render rows whose video is gone.") + parser.add_argument("--apply", action="store_true", + help="delete them (without it, only list them)") + args = parser.parse_args(argv) + + from app.database import engine + + with Session(engine) as session: + found = find(session) + print(f"Renders directory: {worker.renders_dir()}") + print(f"{found.kept} finished render(s) have their video; " + f"{len(found.rows)} do not.") + for job in found.rows: + saved = f", saved to GAM as {job.gam_asset_id}" if job.gam_asset_id else "" + print(f" {job.id} project {job.project_id} v{job.version_seq} " + f"{'preview' if job.preview else 'full'} {job.created_at:%Y-%m-%d %H:%M}{saved}") + if not found.rows: + return 0 + if found.kept == 0 and not args.apply: + print("\nNo render has its video. If that's unexpected, check the renders " + "directory above before applying.") + if not args.apply: + print("\nNothing deleted. Run again with --apply to delete these rows.") + return 0 + remove(session, found.rows) + print(f"\nDeleted {len(found.rows)} row(s).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/tests/test_render_cleanup.py b/backend/tests/test_render_cleanup.py new file mode 100644 index 0000000..07787ee --- /dev/null +++ b/backend/tests/test_render_cleanup.py @@ -0,0 +1,61 @@ +"""`python -m app.render.cleanup`: render rows whose video is gone, listed, then removed.""" + +import shutil + +import pytest + +from app import database +from app.models.render import RenderJob +from app.render import cleanup, worker +from tests.test_renders_api import _finished, _project, _start + + +@pytest.fixture +def renders(auth_client, session, engine, monkeypatch): + """Four renders: one with its video, two whose files went with a rebuild (one of + them saved to GAM first), and one that failed and never had a file.""" + monkeypatch.setattr(database, "engine", engine) + _, kept = _finished(auth_client, session) + _, lost = _finished(auth_client, session) + _, saved = _finished(auth_client, session) + for job in (lost, saved): + (worker.renders_dir() / job["id"] / "video.mp4").unlink() + session.get(RenderJob, saved["id"]).gam_asset_id = "gam-9" + failed = _start(auth_client, _project(auth_client)["id"]).json()["data"] + row = session.get(RenderJob, failed["id"]) + row.status = "error" + session.commit() + return {"kept": kept["id"], "lost": lost["id"], "saved": saved["id"], "failed": failed["id"]} + + +def ids(session): + session.expire_all() + return {job.id for job in session.query(RenderJob).all()} + + +def test_by_default_it_only_lists(renders, session, capsys): + assert cleanup.main([]) == 0 + out = capsys.readouterr().out + assert "1 finished render(s) have their video; 2 do not." in out + assert renders["lost"] in out and f"{renders['saved']}" in out and "saved to GAM as gam-9" in out + assert renders["kept"] not in out and renders["failed"] not in out + assert "Nothing deleted" in out + assert ids(session) == set(renders.values()) + + +def test_with_apply_it_deletes_only_the_orphans_and_their_leftovers(renders, session, capsys): + poster = worker.renders_dir() / renders["lost"] / "poster.jpg" + poster.write_bytes(b"x") + assert cleanup.main(["--apply"]) == 0 + assert "Deleted 2 row(s)." in capsys.readouterr().out + assert ids(session) == {renders["kept"], renders["failed"]} + assert not poster.parent.exists() + assert (worker.renders_dir() / renders["kept"] / "video.mp4").is_file() + + +def test_when_no_render_has_its_video_it_says_to_check_the_directory(renders, capsys): + shutil.rmtree(worker.renders_dir()) + cleanup.main([]) + out = capsys.readouterr().out + assert "0 finished render(s) have their video; 3 do not." in out + assert "check the renders directory" in out