Add durable OGM thumbnail coverage pipeline

.env.example

@@ -27,6 +27,11 @@ RESOURCE_REPRESENTATION_DURABLE_STORE=database
 API_RESPONSE_DURABLE_CACHE_STORE=database
 VISUAL_ASSET_DURABLE_STORE=database
 VISUAL_ASSET_CACHE_TTL_SECONDS=0
+OGM_THUMBNAIL_REFRESH_ENABLED=true
+OGM_THUMBNAIL_REFRESH_BATCH_SIZE=500
+OGM_THUMBNAIL_REFRESH_CONCURRENCY=2
+PDF_THUMBNAIL_MAX_BYTES=33554432
+REMOTE_THUMBNAIL_MAX_BYTES=20971520
 
 # Admin / webhook
 ADMIN_USERNAME=admin

Dockerfile

@@ -12,6 +12,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     libcairo2-dev \
     gdal-bin \
     libgdal-dev \
+    poppler-utils \
     curl \
     ca-certificates \
     cron \

backend/app/api/v1/endpoint_modules/resources/thumbnail.py

@@ -11,6 +11,7 @@
 from sqlalchemy.sql import select
 
 from app.api.v1.utils import _get_thumbnail_asset_url, sanitize_for_json
+from app.services.access_policy import is_restricted_resource
 from app.services.cache_service import alias_redirect_cache_control_header
 from app.services.distribution_repository import fetch_distribution_context
 from app.services.iiif_url import is_iiif_info_url
@@ -21,13 +22,16 @@
 from app.services.thumbnail_state_service import (
     ThumbnailState,
     ThumbnailStatePayload,
+    infer_source_type,
     safe_record_thumbnail_state,
 )
 from app.tasks.worker import (
     _generate_cog_thumbnail_bytes,
+    _generate_pdf_thumbnail_bytes,
     _generate_pmtiles_thumbnail_bytes,
     _normalize_thumbnail_image,
     generate_cog_thumbnail,
+    generate_pdf_thumbnail,
     generate_pmtiles_thumbnail,
 )
 from db.models import resources
@@ -448,7 +452,7 @@ async def _get_resource_thumbnail_response(
         raise HTTPException(status_code=404, detail="Resource not found")
 
     # Check for restricted access rights
-    if resource_dict.get("dct_accessrights_s") == "Restricted":
+    if is_restricted_resource(resource_dict):
         await safe_record_thumbnail_state(
             ThumbnailStatePayload(
                 resource_id=id,
@@ -499,6 +503,8 @@ async def _get_resource_thumbnail_response(
                         if image_service._is_cog_url(source_url)
                         else "pmtiles"
                         if image_service._is_pmtiles_url(source_url)
+                        else "pdf"
+                        if infer_source_type(source_url) == "pdf"
                         else "manifest"
                         if image_service._is_manifest_url(source_url)
                         else "remote"
@@ -536,6 +542,7 @@ async def _get_resource_thumbnail_response(
         and not is_iiif_info_url(source_url)
         and not image_service._is_cog_url(source_url)
         and not image_service._is_pmtiles_url(source_url)
+        and infer_source_type(source_url) != "pdf"
         and THUMBNAIL_REQUEST_PROBE_ENABLED
     ):
         fetch_url = image_service._standardize_iiif_url(source_url)
@@ -608,6 +615,31 @@ async def _get_resource_thumbnail_response(
                         state_detail="PMTiles thumbnail generation already queued",
                     )
                 )
+        elif infer_source_type(source_url) == "pdf":
+            if acquire_thumbnail_queue_slot(id, source_url):
+                task = generate_pdf_thumbnail.delay(source_url, id)
+                await safe_record_thumbnail_state(
+                    ThumbnailStatePayload(
+                        resource_id=id,
+                        state=ThumbnailState.QUEUED,
+                        source_type="pdf",
+                        source_url=source_url,
+                        source_hash=image_hash,
+                        queue_task_id=task.id,
+                        state_detail="Queued PDF first-page thumbnail generation",
+                    )
+                )
+            else:
+                await safe_record_thumbnail_state(
+                    ThumbnailStatePayload(
+                        resource_id=id,
+                        state=ThumbnailState.QUEUED,
+                        source_type="pdf",
+                        source_url=source_url,
+                        source_hash=image_hash,
+                        state_detail="PDF thumbnail generation already queued",
+                    )
+                )
         elif image_service._is_manifest_url(source_url) or is_iiif_info_url(source_url):
             image_service._queue_thumbnail_processing(source_url, id)
         else:
@@ -625,6 +657,8 @@ async def _get_resource_thumbnail_response(
                     if image_service._is_cog_url(source_url)
                     else "pmtiles"
                     if image_service._is_pmtiles_url(source_url)
+                    else "pdf"
+                    if infer_source_type(source_url) == "pdf"
                     else "manifest"
                     if image_service._is_manifest_url(source_url)
                     else "remote"
@@ -701,7 +735,7 @@ async def get_resource_thumbnail_no_cache(
 
             resource_dict = sanitize_for_json(dict(row._mapping))
 
-        if resource_dict.get("dct_accessrights_s") == "Restricted":
+        if is_restricted_resource(resource_dict):
             return _svg_placeholder(title="Thumbnail unavailable", subtitle="Restricted resource")
 
         distribution_context = await fetch_distribution_context(id)
@@ -745,6 +779,21 @@ async def get_resource_thumbnail_no_cache(
                 )
             return await _svg_icon_for_resource(resource_dict, variant=variant)
 
+        # For PDF maps: render the first page synchronously for diagnostics.
+        if infer_source_type(source_url) == "pdf":
+            image_bytes = await asyncio.to_thread(_generate_pdf_thumbnail_bytes, source_url)
+            if image_bytes:
+                normalized_bytes, normalized_type = _normalize_thumbnail_image(
+                    image_bytes, "image/png"
+                )
+                if normalized_bytes and normalized_type:
+                    return Response(
+                        content=normalized_bytes,
+                        media_type=normalized_type,
+                        headers={"Cache-Control": "no-store"},
+                    )
+            return await _svg_icon_for_resource(resource_dict, variant=variant)
+
         # Resolve IIIF metadata to an actual image URL when needed.
         if is_iiif_info_url(source_url):
             resolved = await asyncio.to_thread(image_service.get_iiif_image_thumbnail, source_url)

backend/app/services/access_policy.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+ACCESS_RIGHTS_KEYS = (
+    "dct_accessRights_s",
+    "dct_accessrights_s",
+)
+
+
+def resource_access_rights(metadata: Mapping[str, Any] | None) -> str | None:
+    """Return a normalized access-rights value from canonical or legacy field names."""
+    if not metadata:
+        return None
+
+    for key in ACCESS_RIGHTS_KEYS:
+        value = metadata.get(key)
+        if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+            value = next((item for item in value if item is not None), None)
+        if value is None:
+            continue
+        normalized = str(value).strip()
+        if normalized:
+            return normalized
+    return None
+
+
+def is_restricted_resource(metadata: Mapping[str, Any] | None) -> bool:
+    """Return True when a resource is explicitly marked Restricted."""
+    access_rights = resource_access_rights(metadata)
+    return bool(access_rights and access_rights.casefold() == "restricted")

backend/app/services/image_service.py

@@ -5,13 +5,15 @@
 import os
 import re
 from typing import Any, Dict, List, Optional
+from urllib.parse import urlsplit
 
 import aiohttp
 import redis
 import requests
 from dotenv import load_dotenv
 
 from app.security_utils import url_hostname_matches
+from app.services.access_policy import is_restricted_resource
 from app.services.distribution_repository import (
     DistributionContext,
     build_distribution_context,
@@ -43,6 +45,14 @@
 REMOTE_THUMBNAIL_PREFIX = f"remote-thumb-normalized:{THUMBNAIL_CACHE_VERSION}:"
 COG_THUMBNAIL_PREFIX = "cog-thumb:"
 PMTILES_THUMBNAIL_PREFIX = "pmtiles-thumb:"
+PDF_THUMBNAIL_PREFIX = "pdf-thumb:"
+
+DOWNLOAD_REFERENCE_URIS = (
+    "http://schema.org/downloadUrl",
+    "https://schema.org/downloadUrl",
+)
+DIRECT_IMAGE_EXTENSIONS = (".jpg", ".jpeg", ".png", ".gif", ".webp")
+DIRECT_IMAGE_LABELS = ("jpeg", "jpg", "png", "gif", "webp")
 
 # Shared Redis connection pool to avoid creating new connections for each ImageService instance
 _redis_connection_pool = None
@@ -475,6 +485,70 @@ def resolve_thumbnail_source_url(
             return source_url
         return self._clean_external_thumbnail_url(thumbnail_asset_url)
 
+    def _download_source_candidates(
+        self,
+        references: Optional[Dict[str, Any]] = None,
+    ) -> List[tuple[str, str]]:
+        """Return normalized ``(url, label)`` candidates from download distributions."""
+        candidates: List[tuple[str, str]] = []
+
+        if references is None:
+            for uri in DOWNLOAD_REFERENCE_URIS:
+                for record in self.by_uri.get(uri, []):
+                    url = self._clean_external_thumbnail_url(record.url)
+                    if url:
+                        candidates.append((url, str(record.label or "").strip()))
+            legacy_references = self._parse_legacy_references()
+            if legacy_references:
+                candidates.extend(self._download_source_candidates(legacy_references))
+            return candidates
+
+        for uri in DOWNLOAD_REFERENCE_URIS:
+            raw_value = references.get(uri)
+            values = raw_value if isinstance(raw_value, list) else [raw_value]
+            for value in values:
+                label = ""
+                if isinstance(value, str):
+                    raw_url = value
+                elif isinstance(value, dict):
+                    raw_url = value.get("url") or value.get("@id") or value.get("id")
+                    label = str(value.get("label") or value.get("title") or "").strip()
+                else:
+                    continue
+                url = self._clean_external_thumbnail_url(raw_url)
+                if url:
+                    candidates.append((url, label))
+        return candidates
+
+    @staticmethod
+    def _url_path_lower(url: str) -> str:
+        return urlsplit(url).path.lower()
+
+    def _download_image_source_url(
+        self,
+        references: Optional[Dict[str, Any]] = None,
+    ) -> Optional[str]:
+        """Return an explicitly image-bearing download URL, if one is advertised."""
+        for url, label in self._download_source_candidates(references):
+            path = self._url_path_lower(url)
+            normalized_label = label.casefold()
+            if path.endswith(DIRECT_IMAGE_EXTENSIONS) or any(
+                token == normalized_label or token in normalized_label.split()
+                for token in DIRECT_IMAGE_LABELS
+            ):
+                return url
+        return None
+
+    def _download_pdf_source_url(
+        self,
+        references: Optional[Dict[str, Any]] = None,
+    ) -> Optional[str]:
+        """Return a PDF download URL suitable for first-page thumbnail rendering."""
+        for url, label in self._download_source_candidates(references):
+            if self._url_path_lower(url).endswith(".pdf") or "pdf" in label.casefold().split():
+                return url
+        return None
+
     def thumbnail_image_hash_for_source_sync(
         self,
         source_url: str,
@@ -490,6 +564,8 @@ def thumbnail_image_hash_for_source_sync(
                 return hashlib.sha256((COG_THUMBNAIL_PREFIX + source_url).encode()).hexdigest()
             if self._is_pmtiles_url(source_url):
                 return hashlib.sha256((PMTILES_THUMBNAIL_PREFIX + source_url).encode()).hexdigest()
+            if self._is_pdf_url(source_url):
+                return hashlib.sha256((PDF_THUMBNAIL_PREFIX + source_url).encode()).hexdigest()
             if self._is_iiif_info_url(source_url):
                 info_cache_key = f"manifest:{source_url}"
                 cached_info_data = self.cache.get(info_cache_key)
@@ -649,7 +725,7 @@ def get_thumbnail_url(self, *, thumbnail_asset_url: Optional[str] = None) -> Opt
         """
         try:
             # Check for restricted access rights
-            if self.metadata.get("dct_accessrights_s") == "Restricted":
+            if is_restricted_resource(self.metadata):
                 self.logger.info("Skipping thumbnail for restricted item")
                 return None
 
@@ -686,7 +762,7 @@ def get_hot_thumbnail_url(
         over a blocking thumbnail generation path.
         """
         try:
-            if self.metadata.get("dct_accessrights_s") == "Restricted":
+            if is_restricted_resource(self.metadata):
                 return None
 
             doc_id = self.metadata.get("id")
@@ -935,6 +1011,13 @@ def _get_thumbnail_source_url(
             if url := self._first_url(image_key, references=references):
                 return url
 
+        # Some OGM records expose only downloadable derivatives. Use explicit
+        # image files first, then render the first page of a PDF map when needed.
+        if download_image_url := self._download_image_source_url(references=references):
+            return download_image_url
+        if download_pdf_url := self._download_pdf_source_url(references=references):
+            return download_pdf_url
+
         # Return None when no thumbnail source is found
         # This allows the frontend to show a default icon based on resource class
         # (gbl_resourceClass_sm)
@@ -959,6 +1042,12 @@ def _is_pmtiles_url(self, url: str) -> bool:
         url_lower = url.lower()
         return url_lower.endswith(".pmtiles") or ".pmtiles?" in url_lower
 
+    def _is_pdf_url(self, url: str) -> bool:
+        """Check whether a source URL points at a PDF download."""
+        if not url:
+            return False
+        return self._url_path_lower(url).endswith(".pdf")
+
     def _is_manifest_url(self, url: str) -> bool:
         """Check if URL looks like a IIIF manifest URL."""
         return is_iiif_manifest_url(url)

backend/app/services/ogm_harvest/harvest.py

@@ -10,6 +10,7 @@
 from app.services.ogm_harvest.importer import OGMResourceImporter
 from app.services.ogm_harvest.repo_sync import OGMRepoSync
 from app.services.ogm_harvest.repository import OGMHarvestRepository
+from app.services.thumbnail_refresh_service import refresh_thumbnail_cache_for_changed_resources
 
 logger = logging.getLogger(__name__)
 
@@ -77,6 +78,28 @@ async def harvest_repo(
             ogm_run_id=run_id,
             progress_meta={"head_sha": head_sha, "repo_action": sync_result.action},
         )
+        changed_thumbnail_ids = sorted(importer.changed_thumbnail_resource_ids)
+        stats["thumbnail_sources_changed"] = len(changed_thumbnail_ids)
+        if changed_thumbnail_ids:
+            await repo.update_harvest_run(
+                ogm_id=run_id,
+                ogm_stats_json={
+                    **(stats or {}),
+                    "stage": "thumbnail_refresh",
+                    "updated_at": datetime.utcnow().isoformat() + "Z",
+                },
+            )
+            try:
+                stats[
+                    "thumbnail_cache_refresh"
+                ] = await refresh_thumbnail_cache_for_changed_resources(changed_thumbnail_ids)
+            except Exception as exc:
+                logger.warning(
+                    "OGM thumbnail cache refresh failed for repo=%s; continuing. err=%s",
+                    repo_name,
+                    exc,
+                )
+                stats["thumbnail_cache_refresh"] = {"enabled": True, "error": str(exc)}
         await repo.update_harvest_run(
             ogm_id=run_id,
             ogm_stats_json={