diff --git a/.env.example b/.env.example index ba49301..e483f2c 100644 --- a/.env.example +++ b/.env.example @@ -40,5 +40,9 @@ XMLTV_HTTP_TIMEOUT_SECONDS=30 # évite de multiplier les pics RAM/CPU et les écritures concurrentes. XMLTV_SYNC_WORKERS=1 +# Nombre maximal d'arènes de l'allocateur glibc. La valeur 2 évite que la +# décompression 7z conserve durablement son pic RAM après les imports. +MALLOC_ARENA_MAX=2 + # User-Agent envoyé lors du téléchargement des sources XMLTV. #XMLTV_USER_AGENT=tvguide/1.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index d296773..42711fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,27 @@ Ce projet suit [Keep a Changelog](https://keepachangelog.com/fr/1.1.0/) et [SemVer](https://semver.org/lang/fr/). +## [1.16.1] — 2026-07-18 + +### Performance et mémoire + +- Les imports utilisent un pool fixe de workers : les mêmes threads sont + réutilisés au lieu de créer de nouvelles arènes natives à chaque lancement. +- L'image borne glibc à deux arènes (`MALLOC_ARENA_MAX=2`) et restitue les pages + inutilisées avec `malloc_trim(0)` après la collecte des objets d'import. +- Les requêtes de listes utilisent des projections SQL explicites et ne + matérialisent plus les colonnes de détail inutiles. + +### Recherche et SQLite + +- Recherche migrée vers un index SQLite FTS5 Unicode, insensible aux accents et + compatible avec les préfixes ; des triggers maintiennent l'index pendant les + imports atomiques et les suppressions. +- WAL checkpointé puis tronqué après migration et import réussi ; auto-checkpoint + activé et journal résiduel limité à 16 Mio. +- Migration automatique des programmes existants, avec test de mise à niveau et + de synchronisation de l'index. + ## [1.16.0] — 2026-07-18 ### Performance et consommation de ressources diff --git a/Dockerfile b/Dockerfile index f849cd1..dcf52ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ WORKDIR /app ENV PATH="/app/.venv/bin:$PATH" \ PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ + MALLOC_ARENA_MAX=2 \ DATA_DIR=/data \ APP_PORT=8080 diff --git a/README.md b/README.md index e2ee5d2..7238a2b 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ publication prévue. Garde-fous : | `XMLTV_MAX_UNCOMPRESSED_MB` | `500` | Taille maximale après décompression GZIP (protection contre les bombes gzip/zip/xz). | | `XMLTV_HTTP_TIMEOUT_SECONDS` | `30` | Timeout HTTP des téléchargements. | | `XMLTV_SYNC_WORKERS` | `1` | Nombre maximal d'imports XMLTV simultanés. Garder `1` avec SQLite limite la RAM/CPU et évite la concurrence entre writers. | +| `MALLOC_ARENA_MAX` | `2` | Borne les arènes de l'allocateur natif afin que les pics de décompression 7z ne restent pas réservés après l'import. | | `XMLTV_USER_AGENT` | `tvguide/1.0` | User-Agent envoyé aux serveurs XMLTV. | | `FORWARDED_ALLOW_IPS` | `*` | IPs de reverse proxy autorisées à définir les en-têtes `X-Forwarded-*` (voir ci-dessous). | @@ -175,6 +176,28 @@ mv data/backup.db /chemin/sauvegarde/ puis `docker compose up -d`. Les migrations de schéma s'appliquent automatiquement au démarrage (Alembic). +### Migration vers 1.16.1 (index FTS5) + +La mise à jour crée automatiquement un index de recherche plein texte sur les +titres, sous-titres et descriptions existants. Aucune commande SQL manuelle +n'est nécessaire et les sources, chaînes, réglages et programmes sont +conservés. La première ouverture peut prendre plus longtemps selon le nombre +de programmes. + +Avant la mise à jour : + +1. sauvegardez `data/app.db` avec la procédure ci-dessus ; +2. vérifiez que le volume dispose idéalement de **deux fois la taille actuelle + de `app.db`** en espace libre, pour la construction de l'index et son WAL ; +3. lancez `pull` puis `up -d` et suivez le démarrage avec + `docker compose -f compose.ghcr.yml logs -f tvguide`. + +Le fichier WAL est checkpointé et tronqué après la migration puis après chaque +import réussi. L'index FTS5 augmente durablement la taille de la base en échange +d'une recherche qui ne parcourt plus tous les programmes. Pour revenir à +1.16.0, restaurez la sauvegarde réalisée avant migration ; ne démarrez pas +directement l'ancienne image sur une base déjà migrée. + ## Reverse proxy L'application écoute en HTTP simple sur un seul port et honore diff --git a/backend/alembic/versions/b7f4a9c2d1e8_add_programme_fts5.py b/backend/alembic/versions/b7f4a9c2d1e8_add_programme_fts5.py new file mode 100644 index 0000000..6f966d6 --- /dev/null +++ b/backend/alembic/versions/b7f4a9c2d1e8_add_programme_fts5.py @@ -0,0 +1,71 @@ +"""add FTS5 search index for programmes + +Revision ID: b7f4a9c2d1e8 +Revises: 03a3c8d4b621 +Create Date: 2026-07-18 +""" + +from alembic import op + +revision = "b7f4a9c2d1e8" +down_revision = "03a3c8d4b621" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # External-content FTS avoids duplicating the source text in the virtual + # table. The shadow index is synchronized for regular writes and for the + # bulk DELETE/INSERT transaction used by XMLTV imports. + op.execute( + """ + CREATE VIRTUAL TABLE IF NOT EXISTS programmes_fts USING fts5( + title, + subtitle, + description, + content='programmes', + content_rowid='id', + tokenize='unicode61 remove_diacritics 2' + ) + """ + ) + op.execute( + """ + CREATE TRIGGER IF NOT EXISTS programmes_fts_ai AFTER INSERT ON programmes BEGIN + INSERT INTO programmes_fts(rowid, title, subtitle, description) + VALUES (new.id, new.title, new.subtitle, new.description); + END + """ + ) + op.execute( + """ + CREATE TRIGGER IF NOT EXISTS programmes_fts_ad AFTER DELETE ON programmes BEGIN + INSERT INTO programmes_fts(programmes_fts, rowid, title, subtitle, description) + VALUES ('delete', old.id, old.title, old.subtitle, old.description); + END + """ + ) + op.execute( + """ + CREATE TRIGGER IF NOT EXISTS programmes_fts_au + AFTER UPDATE OF title, subtitle, description + ON programmes BEGIN + INSERT INTO programmes_fts(programmes_fts, rowid, title, subtitle, description) + VALUES ('delete', old.id, old.title, old.subtitle, old.description); + INSERT INTO programmes_fts(rowid, title, subtitle, description) + VALUES (new.id, new.title, new.subtitle, new.description); + END + """ + ) + op.execute( + """ + INSERT INTO programmes_fts(programmes_fts) VALUES ('rebuild') + """ + ) + + +def downgrade() -> None: + op.execute("DROP TRIGGER IF EXISTS programmes_fts_au") + op.execute("DROP TRIGGER IF EXISTS programmes_fts_ad") + op.execute("DROP TRIGGER IF EXISTS programmes_fts_ai") + op.execute("DROP TABLE IF EXISTS programmes_fts") diff --git a/backend/app/api/routes/channels.py b/backend/app/api/routes/channels.py index 6a0e1c3..4db5fd7 100644 --- a/backend/app/api/routes/channels.py +++ b/backend/app/api/routes/channels.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, Depends from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, load_only from app.db.session import get_session from app.models import Channel @@ -12,6 +12,21 @@ @router.get("/channels", response_model=list[ChannelOut]) def list_channels(session: Session = Depends(get_session)) -> list[ChannelOut]: rows = session.execute( - select(Channel).where(Channel.enabled.is_(True)).order_by(Channel.display_order, Channel.id) + select(Channel) + .options( + load_only( + Channel.id, + Channel.xmltv_id, + Channel.name, + Channel.custom_name, + Channel.icon_url, + Channel.custom_icon_url, + Channel.display_order, + Channel.enabled, + raiseload=True, + ) + ) + .where(Channel.enabled.is_(True)) + .order_by(Channel.display_order, Channel.id) ).scalars() return [ChannelOut.from_model(c) for c in rows] diff --git a/backend/app/api/routes/programmes.py b/backend/app/api/routes/programmes.py index 376e217..21b953a 100644 --- a/backend/app/api/routes/programmes.py +++ b/backend/app/api/routes/programmes.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy import select -from sqlalchemy.orm import Session, defer +from sqlalchemy.orm import Session, load_only from app.db.base import utcnow from app.db.session import get_session @@ -27,12 +27,37 @@ def _card_columns_only(): - """Avoid loading detail-only JSON/text fields for programme collections.""" - return ( - defer(Programme.raw_metadata), - defer(Programme.year), - defer(Programme.country), - defer(Programme.created_at), + """Exact SQL projection used by every programme-card collection.""" + return load_only( + Programme.id, + Programme.channel_id, + Programme.start_at_utc, + Programme.stop_at_utc, + Programme.title, + Programme.subtitle, + Programme.description, + Programme.category, + Programme.icon_url, + Programme.season_number, + Programme.episode_number, + Programme.rating, + Programme.is_new, + Programme.is_repeat, + raiseload=True, + ) + + +def _channel_card_columns_only(): + return load_only( + Channel.id, + Channel.xmltv_id, + Channel.name, + Channel.custom_name, + Channel.icon_url, + Channel.custom_icon_url, + Channel.display_order, + Channel.enabled, + raiseload=True, ) @@ -40,6 +65,7 @@ def _enabled_channels(session: Session) -> list[Channel]: return list( session.execute( select(Channel) + .options(_channel_card_columns_only()) .where(Channel.enabled.is_(True)) .order_by(Channel.display_order, Channel.id) ).scalars() @@ -70,7 +96,7 @@ def _next_per_channel( ) rows = session.execute( select(Programme) - .options(*_card_columns_only()) + .options(_card_columns_only()) .where(Programme.id.in_(select(subq.c.pid).where(subq.c.rn <= count))) .order_by(Programme.start_at_utc) ).scalars() @@ -122,7 +148,7 @@ def live(at: datetime | None = None, session: Session = Depends(get_session)) -> current_rows = session.execute( select(Programme) - .options(*_card_columns_only()) + .options(_card_columns_only()) .join(Channel, Channel.id == Programme.channel_id) .where( Channel.enabled.is_(True), @@ -164,7 +190,7 @@ def _evening_response( # real start time — not leave the channel empty. rows = session.execute( select(Programme) - .options(*_card_columns_only()) + .options(_card_columns_only()) .join(Channel, Channel.id == Programme.channel_id) .where( Channel.enabled.is_(True), @@ -227,7 +253,7 @@ def grid( channels = _enabled_channels(session) rows = session.execute( select(Programme) - .options(*_card_columns_only()) + .options(_card_columns_only()) .join(Channel, Channel.id == Programme.channel_id) .where( Channel.enabled.is_(True), diff --git a/backend/app/api/routes/search.py b/backend/app/api/routes/search.py index 65d7e27..1fcb583 100644 --- a/backend/app/api/routes/search.py +++ b/backend/app/api/routes/search.py @@ -1,8 +1,9 @@ +import re from datetime import date from fastapi import APIRouter, Depends, Query -from sqlalchemy import func, or_, select -from sqlalchemy.orm import Session, defer +from sqlalchemy import Integer, func, select, text +from sqlalchemy.orm import Session, load_only from app.db.session import get_session from app.models import Channel, Programme @@ -13,9 +14,35 @@ router = APIRouter(tags=["search"]) -def _like_pattern(query: str) -> str: - escaped = query.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - return f"%{escaped.lower()}%" +_SEARCH_TOKEN = re.compile(r"\w+", flags=re.UNICODE) + + +def _fts_query(query: str) -> str | None: + """Build a literal, prefix-enabled FTS query without exposing operators.""" + tokens = _SEARCH_TOKEN.findall(query.casefold()) + if not tokens: + return None + return " AND ".join(f'"{token}"*' for token in tokens) + + +def _card_projection(): + return load_only( + Programme.id, + Programme.channel_id, + Programme.start_at_utc, + Programme.stop_at_utc, + Programme.title, + Programme.subtitle, + Programme.description, + Programme.category, + Programme.icon_url, + Programme.season_number, + Programme.episode_number, + Programme.rating, + Programme.is_new, + Programme.is_repeat, + raiseload=True, + ) @router.get("/search", response_model=list[SearchResult]) @@ -27,24 +54,31 @@ def search( limit: int = Query(default=100, ge=1, le=200), session: Session = Depends(get_session), ) -> list[SearchResult]: - pattern = _like_pattern(q) + match_query = _fts_query(q) + if match_query is None: + return [] + matched = ( + text("SELECT rowid FROM programmes_fts WHERE programmes_fts MATCH :match_query") + .bindparams(match_query=match_query) + .columns(rowid=Integer) + .subquery("matched_programmes") + ) stmt = ( select(Programme, Channel) .options( - defer(Programme.raw_metadata), - defer(Programme.year), - defer(Programme.country), - defer(Programme.created_at), - ) - .join(Channel, Channel.id == Programme.channel_id) - .where( - Channel.enabled.is_(True), - or_( - func.lower(Programme.title).like(pattern, escape="\\"), - func.lower(Programme.subtitle).like(pattern, escape="\\"), - func.lower(Programme.description).like(pattern, escape="\\"), + _card_projection(), + load_only( + Channel.id, + Channel.name, + Channel.custom_name, + Channel.icon_url, + Channel.custom_icon_url, + raiseload=True, ), ) + .join(matched, matched.c.rowid == Programme.id) + .join(Channel, Channel.id == Programme.channel_id) + .where(Channel.enabled.is_(True)) .order_by(Programme.start_at_utc) .limit(limit) ) diff --git a/backend/app/db/session.py b/backend/app/db/session.py index 728e2e1..507cef6 100644 --- a/backend/app/db/session.py +++ b/backend/app/db/session.py @@ -1,11 +1,15 @@ +import logging from collections.abc import Iterator from sqlalchemy import Engine, create_engine, event +from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import StaticPool from app.core.config import get_settings +logger = logging.getLogger(__name__) + _engine: Engine | None = None _session_factory: sessionmaker[Session] | None = None @@ -27,6 +31,11 @@ def _set_sqlite_pragmas(dbapi_connection, _record) -> None: cursor.execute("PRAGMA foreign_keys=ON") cursor.execute("PRAGMA journal_mode=WAL") cursor.execute("PRAGMA busy_timeout=5000") + # Keep SQLite predictable in a small, single-process container. WAL + # pages may grow during an atomic XMLTV import, but are checkpointed + # afterwards and the retained journal is capped at 16 MiB. + cursor.execute("PRAGMA wal_autocheckpoint=1000") + cursor.execute("PRAGMA journal_size_limit=16777216") cursor.close() _session_factory = sessionmaker(bind=_engine, expire_on_commit=False) @@ -54,3 +63,25 @@ def get_session() -> Iterator[Session]: yield session finally: session.close() + + +def checkpoint_database(*, truncate: bool = False) -> tuple[int, int, int] | None: + """Checkpoint the SQLite WAL outside an application transaction. + + Returns SQLite's ``(busy, log_frames, checkpointed_frames)`` tuple. Tests + and custom deployments may use another database backend, in which case + maintenance is simply skipped. + """ + engine = get_engine() + if engine.dialect.name != "sqlite": + return None + mode = "TRUNCATE" if truncate else "PASSIVE" + try: + with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection: + row = connection.exec_driver_sql(f"PRAGMA wal_checkpoint({mode})").one() + return int(row[0]), int(row[1]), int(row[2]) + except SQLAlchemyError: + # Maintenance must never turn a successfully committed import into a + # reported failure. A later auto-checkpoint can safely retry. + logger.warning("SQLite WAL checkpoint failed", exc_info=True) + return None diff --git a/backend/app/jobs/scheduler.py b/backend/app/jobs/scheduler.py index f7cb66a..2fe51a5 100644 --- a/backend/app/jobs/scheduler.py +++ b/backend/app/jobs/scheduler.py @@ -8,6 +8,7 @@ import logging import threading +from concurrent.futures import ThreadPoolExecutor from apscheduler.schedulers.background import BackgroundScheduler from sqlalchemy import or_, select @@ -16,25 +17,45 @@ from app.db.base import utcnow from app.db.session import get_session_factory from app.models import Source +from app.services.memory import reclaim_process_memory from app.services.xmltv import sync_state from app.services.xmltv.importer import sync_source logger = logging.getLogger(__name__) _scheduler: BackgroundScheduler | None = None -_sync_slots: threading.BoundedSemaphore | None = None -_sync_slots_lock = threading.Lock() +_sync_executor: ThreadPoolExecutor | None = None +_sync_executor_lock = threading.Lock() -def _get_sync_slots() -> threading.BoundedSemaphore: - global _sync_slots - if _sync_slots is None: - with _sync_slots_lock: - if _sync_slots is None: +def _get_sync_executor() -> ThreadPoolExecutor: + """Return the fixed worker pool reused by scheduled and manual imports.""" + global _sync_executor + if _sync_executor is None: + with _sync_executor_lock: + if _sync_executor is None: workers = max(1, min(get_settings().xmltv_sync_workers, 8)) - _sync_slots = threading.BoundedSemaphore(workers) - logger.info("XMLTV worker concurrency limited to %d", workers) - return _sync_slots + _sync_executor = ThreadPoolExecutor( + max_workers=workers, thread_name_prefix="xmltv-sync" + ) + logger.info("XMLTV fixed worker pool started with %d worker(s)", workers) + return _sync_executor + + +def _run_sync_job(source_id: int, force: bool) -> None: + session = get_session_factory()() + reclaim_after_job = True + try: + outcome = sync_source(session, source_id, get_settings(), force=force) + reclaim_after_job = not outcome.skipped + sync_state.finish(source_id, error=outcome.error) + except Exception as exc: # defensive: never leave the source claimed + logger.exception("sync worker crashed source_id=%d", source_id) + sync_state.finish(source_id, error=str(exc)) + finally: + session.close() + if reclaim_after_job: + reclaim_process_memory() def run_sync_in_thread(source_id: int, force: bool = False) -> bool: @@ -44,21 +65,11 @@ def run_sync_in_thread(source_id: int, force: bool = False) -> bool: if not sync_state.try_start(source_id): return False - slots = _get_sync_slots() - - def _worker() -> None: - with slots: - session = get_session_factory()() - try: - outcome = sync_source(session, source_id, get_settings(), force=force) - sync_state.finish(source_id, error=outcome.error) - except Exception as exc: # defensive: never leave the slot claimed - logger.exception("sync worker crashed source_id=%d", source_id) - sync_state.finish(source_id, error=str(exc)) - finally: - session.close() - - threading.Thread(target=_worker, name=f"xmltv-sync-{source_id}", daemon=True).start() + try: + _get_sync_executor().submit(_run_sync_job, source_id, force) + except RuntimeError as exc: + sync_state.finish(source_id, error=str(exc)) + return False return True @@ -95,7 +106,10 @@ def start_scheduler() -> None: def stop_scheduler() -> None: - global _scheduler + global _scheduler, _sync_executor if _scheduler is not None: _scheduler.shutdown(wait=False) _scheduler = None + if _sync_executor is not None: + _sync_executor.shutdown(wait=False, cancel_futures=False) + _sync_executor = None diff --git a/backend/app/main.py b/backend/app/main.py index f8cbd45..d581028 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -11,7 +11,7 @@ from app.api.routes import api_router from app.core.config import get_settings from app.core.logging import setup_logging -from app.db.session import init_db +from app.db.session import checkpoint_database, init_db from app.jobs.scheduler import start_scheduler, stop_scheduler logger = logging.getLogger(__name__) @@ -42,6 +42,9 @@ async def lifespan(app: FastAPI): init_db() run_migrations() + # A data migration (notably the initial FTS5 build) can leave a large WAL. + # Reclaim it before accepting traffic. + checkpoint_database(truncate=True) logger.info("Database ready at %s", settings.resolved_database_url) if settings.scheduler_enabled: diff --git a/backend/app/services/memory.py b/backend/app/services/memory.py new file mode 100644 index 0000000..20749d4 --- /dev/null +++ b/backend/app/services/memory.py @@ -0,0 +1,29 @@ +"""Best-effort process memory reclamation after heavyweight background work.""" + +import ctypes +import gc +import logging +import sys + +logger = logging.getLogger(__name__) + + +def reclaim_process_memory() -> None: + """Release unreachable Python objects and return free glibc pages to Linux. + + CPython and native decompression libraries intentionally retain allocator + arenas after a peak. Reusing worker threads prevents new arenas from being + created for every import; ``malloc_trim`` then makes unused pages visible + as free memory to the container. Both operations are safe best efforts. + """ + gc.collect() + if not sys.platform.startswith("linux"): + return + try: + libc = ctypes.CDLL(None) + malloc_trim = libc.malloc_trim + malloc_trim.argtypes = [ctypes.c_size_t] + malloc_trim.restype = ctypes.c_int + malloc_trim(0) + except (AttributeError, OSError): + logger.debug("malloc_trim unavailable on this platform", exc_info=True) diff --git a/backend/app/services/xmltv/importer.py b/backend/app/services/xmltv/importer.py index f8fd677..28177d8 100644 --- a/backend/app/services/xmltv/importer.py +++ b/backend/app/services/xmltv/importer.py @@ -25,6 +25,7 @@ from app.core.config import Settings from app.db.base import utcnow +from app.db.session import checkpoint_database from app.models import Channel, Programme, Source from app.services.xmltv import sync_state from app.services.xmltv.fetcher import FetchError, fetch_source_file, probe_signature @@ -341,6 +342,13 @@ def sync_source( logger.exception("XMLTV sync unexpected failure source_id=%d", source_id) return _record_failure(session, source_id, f"erreur interne : {exc}") + busy, log_frames, checkpointed = checkpoint_database(truncate=True) or (0, 0, 0) + if busy: + logger.warning( + "SQLite WAL checkpoint busy log_frames=%d checkpointed=%d", + log_frames, + checkpointed, + ) logger.info("Imported channels=%d programmes=%d", channel_count, programme_count) logger.info("XMLTV sync completed source=%r", source.name) return SyncOutcome(success=True, channels=channel_count, programmes=programme_count) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index a4db08f..e3935f6 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -209,6 +209,13 @@ def test_search(client, session): # category filter hits = client.get("/api/search", params={"q": "ciné", "category": "Film"}).json() assert [h["title"] for h in hits] == ["Ciné tardif"] + # FTS5 unicode tokenizer: accent-insensitive and prefix searches. + assert [h["title"] for h in client.get("/api/search", params={"q": "cine tard"}).json()] == [ + "Ciné tardif" + ] + assert [h["title"] for h in client.get("/api/search", params={"q": "dujar"}).json()] == [ + "En cours" + ] # date filter (Paris local day) hits = client.get("/api/search", params={"q": "ciné", "date": "2026-09-11"}).json() assert hits == [] @@ -216,6 +223,23 @@ def test_search(client, session): assert client.get("/api/search", params={"q": "fantôme"}).json() == [] # q too short assert client.get("/api/search", params={"q": "a"}).status_code == 422 + assert client.get("/api/search", params={"q": "%%"}).json() == [] + + +def test_fts_index_tracks_programme_updates_and_deletes(client, session): + _seed(session) + programme = session.query(Programme).filter_by(title="Prime TF1").one() + programme.title = "Grand direct" + session.commit() + + assert client.get("/api/search", params={"q": "prime"}).json() == [] + assert [hit["title"] for hit in client.get("/api/search", params={"q": "grand"}).json()] == [ + "Grand direct" + ] + + session.delete(programme) + session.commit() + assert client.get("/api/search", params={"q": "grand"}).json() == [] def test_categories(client, session): @@ -261,6 +285,33 @@ def test_programme_cards_carry_short_description(client, session): assert live[0]["current"]["description_short"] == "Un film avec Jean Dujardin." +def test_programme_lists_use_detail_free_sql_projection(client, session): + from sqlalchemy import event + + from app.db.session import get_engine + + _seed(session) + statements: list[str] = [] + + def capture(_connection, _cursor, statement, _parameters, _context, _many): + if "SELECT programmes.id" in statement: + statements.append(statement) + + engine = get_engine() + event.listen(engine, "before_cursor_execute", capture) + try: + assert client.get("/api/programmes/live").status_code == 200 + finally: + event.remove(engine, "before_cursor_execute", capture) + + assert statements + for statement in statements: + assert "programmes.raw_metadata" not in statement + assert "programmes.year" not in statement + assert "programmes.country" not in statement + assert "programmes.created_at" not in statement + + def test_short_description_truncated_on_word_boundary(): from app.schemas.guide import _shorten diff --git a/backend/tests/test_db.py b/backend/tests/test_db.py new file mode 100644 index 0000000..91f9919 --- /dev/null +++ b/backend/tests/test_db.py @@ -0,0 +1,6 @@ +from sqlalchemy import text + + +def test_sqlite_runtime_limits(session): + assert session.execute(text("PRAGMA wal_autocheckpoint")).scalar_one() == 1000 + assert session.execute(text("PRAGMA journal_size_limit")).scalar_one() == 16 * 1024 * 1024 diff --git a/backend/tests/test_memory.py b/backend/tests/test_memory.py new file mode 100644 index 0000000..a1ae786 --- /dev/null +++ b/backend/tests/test_memory.py @@ -0,0 +1,24 @@ +from app.services import memory + + +def test_reclaim_process_memory_collects_and_trims_on_linux(monkeypatch): + calls: list[object] = [] + + class FakeTrim: + argtypes = None + restype = None + + def __call__(self, padding): + calls.append(padding) + return 1 + + class FakeLibc: + malloc_trim = FakeTrim() + + monkeypatch.setattr(memory.gc, "collect", lambda: calls.append("gc")) + monkeypatch.setattr(memory.sys, "platform", "linux") + monkeypatch.setattr(memory.ctypes, "CDLL", lambda _name: FakeLibc()) + + memory.reclaim_process_memory() + + assert calls == ["gc", 0] diff --git a/backend/tests/test_migrations.py b/backend/tests/test_migrations.py new file mode 100644 index 0000000..c7309c6 --- /dev/null +++ b/backend/tests/test_migrations.py @@ -0,0 +1,63 @@ +import sqlite3 +from datetime import UTC, datetime, timedelta + +from alembic.config import Config as AlembicConfig + +from alembic import command +from app.core.config import get_settings +from app.main import BACKEND_ROOT + + +def _config(database_path) -> AlembicConfig: + config = AlembicConfig(str(BACKEND_ROOT / "alembic.ini")) + config.set_main_option("script_location", str(BACKEND_ROOT / "alembic")) + config.set_main_option("sqlalchemy.url", f"sqlite:///{database_path}") + return config + + +def test_fts_migration_indexes_existing_programmes(tmp_path, monkeypatch): + database = tmp_path / "migration.db" + with monkeypatch.context() as scoped: + scoped.setenv("DATABASE_URL", f"sqlite:///{database}") + get_settings.cache_clear() + config = _config(database) + command.upgrade(config, "03a3c8d4b621") + + now = datetime.now(UTC).replace(tzinfo=None) + now_sql = now.isoformat(sep=" ") + stop_sql = (now + timedelta(hours=1)).isoformat(sep=" ") + with sqlite3.connect(database) as connection: + connection.execute( + """INSERT INTO sources + (id, name, source_type, enabled, sync_interval_hours, channel_count, + programme_count, created_at, updated_at) + VALUES (1, 'source', 'file', 1, 12, 1, 1, ?, ?)""", + (now_sql, now_sql), + ) + connection.execute( + """INSERT INTO channels + (id, source_id, xmltv_id, name, enabled, display_order, created_at, updated_at) + VALUES (1, 1, 'channel', 'Chaîne', 1, 1, ?, ?)""", + (now_sql, now_sql), + ) + connection.execute( + """INSERT INTO programmes + (id, channel_id, start_at_utc, stop_at_utc, title, description, + is_new, is_repeat, created_at) + VALUES (1, 1, ?, ?, 'Cinéma ancien', 'Avec Jean Dujardin', 0, 0, ?)""", + (now_sql, stop_sql, now_sql), + ) + + command.upgrade(config, "head") + + with sqlite3.connect(database) as connection: + rows = connection.execute( + "SELECT rowid FROM programmes_fts WHERE programmes_fts MATCH 'cinema AND dujar*'" + ).fetchall() + assert rows == [(1,)] + plan = connection.execute( + "EXPLAIN QUERY PLAN SELECT rowid FROM programmes_fts " + "WHERE programmes_fts MATCH 'cinema'" + ).fetchall() + assert any("VIRTUAL TABLE INDEX" in row[3] for row in plan) + get_settings.cache_clear() diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index b658f7f..7c832d0 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -1,15 +1,18 @@ +import threading + from app.jobs import scheduler def test_xmltv_worker_concurrency_is_bounded(settings): previous = settings.xmltv_sync_workers - scheduler._sync_slots = None + scheduler.stop_scheduler() settings.xmltv_sync_workers = 1 try: - slots = scheduler._get_sync_slots() - assert slots.acquire(blocking=False) - assert not slots.acquire(blocking=False) - slots.release() + executor = scheduler._get_sync_executor() + assert executor is scheduler._get_sync_executor() + assert executor._max_workers == 1 + thread_ids = [executor.submit(threading.get_ident).result() for _ in range(3)] + assert len(set(thread_ids)) == 1 finally: + scheduler.stop_scheduler() settings.xmltv_sync_workers = previous - scheduler._sync_slots = None diff --git a/compose.ghcr.yml b/compose.ghcr.yml index 49429c4..c3b93f7 100644 --- a/compose.ghcr.yml +++ b/compose.ghcr.yml @@ -4,7 +4,7 @@ # docker compose -f compose.ghcr.yml pull # docker compose -f compose.ghcr.yml up -d # -# Épinglez une version précise en remplaçant `latest` (ex. :1.15.0). +# Épinglez une version précise en remplaçant `latest` (ex. :1.16.1). services: tvguide: @@ -24,6 +24,7 @@ services: XMLTV_MAX_UNCOMPRESSED_MB: ${XMLTV_MAX_UNCOMPRESSED_MB:-500} XMLTV_HTTP_TIMEOUT_SECONDS: ${XMLTV_HTTP_TIMEOUT_SECONDS:-30} XMLTV_SYNC_WORKERS: ${XMLTV_SYNC_WORKERS:-1} + MALLOC_ARENA_MAX: ${MALLOC_ARENA_MAX:-2} volumes: - ./data:/data diff --git a/compose.yml b/compose.yml index d8140ba..0f6b9e6 100644 --- a/compose.yml +++ b/compose.yml @@ -18,6 +18,7 @@ services: XMLTV_MAX_UNCOMPRESSED_MB: ${XMLTV_MAX_UNCOMPRESSED_MB:-500} XMLTV_HTTP_TIMEOUT_SECONDS: ${XMLTV_HTTP_TIMEOUT_SECONDS:-30} XMLTV_SYNC_WORKERS: ${XMLTV_SYNC_WORKERS:-1} + MALLOC_ARENA_MAX: ${MALLOC_ARENA_MAX:-2} volumes: - ./data:/data