diff --git a/Dockerfile b/Dockerfile index cc435eb..bc0402b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.11-slim WORKDIR /app -RUN apt-get update && apt-get install -y \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ gcc \ g++ \ python3-dev \ @@ -14,7 +14,9 @@ RUN apt-get update && apt-get install -y \ libgdal-dev \ curl \ ca-certificates \ + cron \ git \ + tzdata \ && rm -rf /var/lib/apt/lists/* ENV GDAL_VERSION=3.4.1 @@ -28,8 +30,10 @@ ENV UV_HTTP_TIMEOUT=300 COPY backend/pyproject.toml backend/uv.lock ./ COPY backend/scripts ./scripts COPY backend/ ./backend/ +COPY config/crontab ./config/crontab RUN uv pip install -e ./backend --system +RUN chmod +x /app/scripts/start_cron.sh RUN mkdir -p logs static/maps diff --git a/backend/app/api/v1/strong_params.py b/backend/app/api/v1/strong_params.py index 9029fd7..f6b4245 100644 --- a/backend/app/api/v1/strong_params.py +++ b/backend/app/api/v1/strong_params.py @@ -38,6 +38,7 @@ "include_filters[field][]", "exclude_filters[field][]", # Convenience multi-select repo filter (OGM) + "ogm_repo", "ogm_repo[]", ] @@ -72,6 +73,7 @@ "include_filters[field][]", "exclude_filters[field][]", # Convenience multi-select repo filter (OGM) + "ogm_repo", "ogm_repo[]", ] diff --git a/backend/app/elasticsearch/client.py b/backend/app/elasticsearch/client.py index 6af2df3..89e58f1 100644 --- a/backend/app/elasticsearch/client.py +++ b/backend/app/elasticsearch/client.py @@ -73,7 +73,13 @@ async def init_elasticsearch(): if "ogm_repo" not in props: logger.info("Adding missing mapping field: ogm_repo") await es.indices.put_mapping( - index=index_name, properties={"ogm_repo": {"type": "keyword"}} + index=index_name, + properties={ + "ogm_repo": { + "type": "text", + "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}, + } + }, ) except Exception as e: logger.warning(f"Could not ensure mappings for {index_name}: {e}") diff --git a/backend/app/elasticsearch/index.py b/backend/app/elasticsearch/index.py index ef1b3c4..a28e818 100644 --- a/backend/app/elasticsearch/index.py +++ b/backend/app/elasticsearch/index.py @@ -18,7 +18,7 @@ from app.services.language_service import ensure_b1g_language from db.database import database -from db.models import resources +from db.models import ogm_resource_state, resources from .client import es from .suggest import build_suggest_inputs @@ -125,6 +125,29 @@ def to_int(v): return iv if iv is not None else None +def _coerce_ogm_repo_values(value): + """Normalize OGM repo values for exact faceting/filtering.""" + if value in (None, ""): + return [] + if isinstance(value, (list, tuple, set)): + raw_values = value + else: + raw_values = [value] + + repos = [] + seen = set() + for raw in raw_values: + text = str(raw).strip() + if not text: + continue + if text.startswith("ogm_repo:"): + text = text[len("ogm_repo:") :].strip() + if text and text not in seen: + seen.add(text) + repos.append(text) + return repos + + def _calculate_time_period_from_year(year_value): """Calculate the time period bucket for a given year value. @@ -208,7 +231,7 @@ async def index_resources(): await init_elasticsearch() - resource_rows = await database.fetch_all(resources.select()) + resource_rows = await fetch_resources_for_index() processed_resources = await prepare_bulk_data(resource_rows, index_name) if processed_resources: @@ -217,6 +240,42 @@ async def index_resources(): return {"message": "No resources to index"} +async def fetch_resources_for_index(): + """Fetch resource rows and attach current OGM repo memberships.""" + resource_rows = await database.fetch_all(resources.select()) + ogm_rows = await database.fetch_all( + ogm_resource_state.select().with_only_columns( + ogm_resource_state.c.ogm_resource_id, + ogm_resource_state.c.ogm_repo_name, + ogm_resource_state.c.ogm_missing_since, + ) + ) + + repos_by_resource_id = {} + resources_with_ogm_state = set() + for row in ogm_rows: + row_dict = dict(row) + resource_id = str(row_dict.get("ogm_resource_id") or "").strip() + repo_name = str(row_dict.get("ogm_repo_name") or "").strip() + if not resource_id or not repo_name: + continue + resources_with_ogm_state.add(resource_id) + if row_dict.get("ogm_missing_since") is not None: + continue + repos = repos_by_resource_id.setdefault(resource_id, []) + if repo_name not in repos: + repos.append(repo_name) + + indexed_rows = [] + for row in resource_rows: + resource = dict(row) + resource_id = str(resource.get("id")) + if resource_id in resources_with_ogm_state: + resource["ogm_repo"] = repos_by_resource_id.get(resource_id, []) + indexed_rows.append(resource) + return indexed_rows + + async def prepare_bulk_data(resources, index_name): """Prepare resources for indexing (now using individual operations for reliability).""" processed_resources = [] @@ -230,6 +289,7 @@ async def prepare_bulk_data(resources, index_name): async def process_resource(resource_dict): """Process a single resource for indexing.""" processed_dict = {} + explicit_ogm_repo_field = "ogm_repo" in resource_dict date_fields = {"gbl_mdmodified_dt", "b1g_dateAccessioned_s", "b1g_dateRetired_s"} integer_fields = {"gbl_indexYear_im"} @@ -240,7 +300,11 @@ async def process_resource(resource_dict): } for key, value in resource_dict.items(): - if isinstance(value, (list, tuple)): + if key == "ogm_repo": + repos = _coerce_ogm_repo_values(value) + if repos: + processed_dict[key] = repos + elif isinstance(value, (list, tuple)): processed_dict[key] = list(value) elif key in date_fields: processed_dict[key] = _coerce_date(value) @@ -294,10 +358,17 @@ async def process_resource(resource_dict): ensure_b1g_language(processed_dict) - # Derive OGM repo facet/filter field from admin tags. + explicit_ogm_repo_values = _coerce_ogm_repo_values(processed_dict.get("ogm_repo")) + if explicit_ogm_repo_values: + processed_dict["ogm_repo"] = explicit_ogm_repo_values + else: + processed_dict.pop("ogm_repo", None) + + # Derive OGM repo facet/filter field from admin tags when no explicit + # repo state was attached to the index row. # Source-of-truth tag format stored in Postgres: "ogm_repo:" tags = processed_dict.get("b1g_adminTags_sm") - if tags: + if tags and not explicit_ogm_repo_field and "ogm_repo" not in processed_dict: if isinstance(tags, str): tags_list = [tags] elif isinstance(tags, list): @@ -305,16 +376,9 @@ async def process_resource(resource_dict): else: tags_list = [str(tags)] - ogm_repo_values = [] - seen = set() - for t in tags_list: - if not isinstance(t, str): - continue - if t.startswith("ogm_repo:"): - repo_name = t[len("ogm_repo:") :].strip() - if repo_name and repo_name not in seen: - seen.add(repo_name) - ogm_repo_values.append(repo_name) + ogm_repo_values = _coerce_ogm_repo_values( + [t for t in tags_list if isinstance(t, str) and t.startswith("ogm_repo:")] + ) if ogm_repo_values: processed_dict["ogm_repo"] = ogm_repo_values diff --git a/backend/app/elasticsearch/mappings.py b/backend/app/elasticsearch/mappings.py index b1a24dc..c968a64 100644 --- a/backend/app/elasticsearch/mappings.py +++ b/backend/app/elasticsearch/mappings.py @@ -40,7 +40,7 @@ "type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 8191}}, }, - # OpenGeoMetadata repo facet/filter (derived at index-time from b1g_adminTags_sm) + # OpenGeoMetadata repo facet/filter (attached from OGM state at index-time) "ogm_repo": { "type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}, diff --git a/backend/app/services/search_service.py b/backend/app/services/search_service.py index 70ae97c..2ba265c 100644 --- a/backend/app/services/search_service.py +++ b/backend/app/services/search_service.py @@ -509,8 +509,10 @@ def extract_new_style_filters(self, params: Optional[str]) -> tuple[Dict, Dict]: list(raw_params.keys())[:10], ) - # Convenience filters (non-bracket style) for common client use cases. - # Example: ogm_repo[]=edu.stanford.purl&ogm_repo[]=edu.umn + # Convenience filters for common client use cases. + # Examples: ogm_repo=edu.unr, ogm_repo[]=edu.stanford.purl&ogm_repo[]=edu.umn + if "ogm_repo" in raw_params: + include_filters.setdefault("ogm_repo", []).extend(raw_params.get("ogm_repo") or []) if "ogm_repo[]" in raw_params: include_filters.setdefault("ogm_repo", []).extend(raw_params.get("ogm_repo[]") or []) diff --git a/backend/scripts/ogm_importer.py b/backend/scripts/ogm_importer.py index e143f41..0ec091e 100644 --- a/backend/scripts/ogm_importer.py +++ b/backend/scripts/ogm_importer.py @@ -34,6 +34,66 @@ logger = logging.getLogger(__name__) +def derive_repo_alias(repo_name: str) -> Optional[str]: + parts = [p for p in (repo_name or "").split(".") if p] + if len(parts) >= 2 and parts[0] == "edu": + return parts[1] + if parts: + return parts[0] + return None + + +def derive_repo_name_from_path(path: str, ogm_path: Optional[str] = None) -> Optional[str]: + path_obj = Path(path) + candidate_parts = list(path_obj.parts) + + if ogm_path: + try: + candidate_parts = list( + path_obj.resolve(strict=False) + .relative_to(Path(ogm_path).resolve(strict=False)) + .parts + ) + except ValueError: + pass + + for parts in (candidate_parts, list(path_obj.parts)): + if "metadata-aardvark" not in parts: + continue + index = parts.index("metadata-aardvark") + if index > 0: + return parts[index - 1] + + return None + + +def inject_ogm_repo_tags(record: Dict[str, Any], repo_name: Optional[str]) -> Dict[str, Any]: + if not repo_name: + return record + + existing = record.get("b1g_adminTags_sm") + tags: List[str] = [] + if isinstance(existing, list): + tags.extend([str(tag).strip() for tag in existing if str(tag).strip()]) + elif isinstance(existing, str) and existing.strip(): + tags.append(existing.strip()) + + tags.append(f"ogm_repo:{repo_name}") + if alias := derive_repo_alias(repo_name): + tags.append(f"ogm:{alias}") + + deduped: List[str] = [] + seen = set() + for tag in tags: + if tag in seen: + continue + seen.add(tag) + deduped.append(tag) + + record["b1g_adminTags_sm"] = deduped + return record + + class OGMImporter: """Imports OpenGeoMetadata Aardvark records into the database.""" @@ -368,6 +428,16 @@ def import_records(self, limit: Optional[int] = None) -> Dict[str, int]: stats["skipped"] += 1 continue + repo_name = derive_repo_name_from_path(path, self.ogm_path) + if repo_name: + inject_ogm_repo_tags(prepared_record, repo_name) + else: + logger.warning( + "Unable to derive OGM repo name for record %s from path %s", + prepared_record.get("id"), + path, + ) + # Normalize to ensure every model column has a value (or None) normalized_record = {} for col in resources.c: diff --git a/backend/scripts/populate_ogm_repos.py b/backend/scripts/populate_ogm_repos.py index a06c8d8..0c312c6 100644 --- a/backend/scripts/populate_ogm_repos.py +++ b/backend/scripts/populate_ogm_repos.py @@ -21,6 +21,7 @@ import argparse import json import os +import sys from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlparse, urlunparse @@ -31,6 +32,9 @@ # Keep script self-contained: import the SQLAlchemy Table definitions. from db.models import ogm_repos +_BAD_GITHUB_TOKEN_WARNING_SHOWN = False +_REJECTED_GITHUB_TOKENS: set[str] = set() + def _sync_database_url(database_url: str) -> str: # Convert asyncpg URL to sync URL @@ -62,14 +66,40 @@ def _github_headers(token: Optional[str]) -> Dict[str, str]: return headers +def _warn_bad_github_token() -> None: + global _BAD_GITHUB_TOKEN_WARNING_SHOWN + if _BAD_GITHUB_TOKEN_WARNING_SHOWN: + return + + print( + "Warning: configured GitHub token was rejected with 401; " + "retrying public GitHub API requests without authentication.", + file=sys.stderr, + ) + _BAD_GITHUB_TOKEN_WARNING_SHOWN = True + + +def _github_get(url: str, token: Optional[str], **kwargs: Any) -> requests.Response: + effective_token = token + if token and token in _REJECTED_GITHUB_TOKENS: + effective_token = None + + resp = requests.get(url, headers=_github_headers(effective_token), **kwargs) + if effective_token and resp.status_code == 401: + _REJECTED_GITHUB_TOKENS.add(effective_token) + _warn_bad_github_token() + resp = requests.get(url, headers=_github_headers(None), **kwargs) + return resp + + def list_org_repos(org: str, token: Optional[str], per_page: int = 100) -> List[Dict[str, Any]]: repos: List[Dict[str, Any]] = [] page = 1 while True: url = f"https://api.github.com/orgs/{org}/repos" - resp = requests.get( + resp = _github_get( url, - headers=_github_headers(token), + token, params={"per_page": per_page, "page": page}, timeout=30, ) @@ -93,7 +123,7 @@ def repo_has_metadata_aardvark( params = {} if default_branch: params["ref"] = default_branch - resp = requests.get(url, headers=_github_headers(token), params=params, timeout=30) + resp = _github_get(url, token, params=params, timeout=30) if resp.status_code == 200: body = resp.json() return isinstance(body, list) diff --git a/backend/scripts/render_cron_env.py b/backend/scripts/render_cron_env.py index 82bddf5..3342722 100644 --- a/backend/scripts/render_cron_env.py +++ b/backend/scripts/render_cron_env.py @@ -11,11 +11,16 @@ ALLOWED_EXACT_KEYS = { "APPLICATION_URL", "BRIDGE_SYNC_LOCAL_TIMEZONE", + "CRON_LOCAL_TIMEZONE", "DATABASE_URL", "ENDPOINT_CACHE", "IS_DOCKER", "KAMAL_DEST", + "OGM_NIGHTLY_CRON_ENABLED", + "OGM_TRIGGER", + "OPENGEOMETADATA_API_BASE_URL", "PYTHONPATH", + "PYTHON_BIN", "STATIC_MAPS_DIR", "TZ", } @@ -30,9 +35,12 @@ "CELERY_", "CORS_", "ELASTICSEARCH_", + "GITHUB_", "KITHE_BRIDGE_", "LOG_", + "OGM_", "OPENAI_", + "OPENGEOMETADATA_", "POSTGRES_", "RATE_LIMIT_", "REDIS_", diff --git a/backend/scripts/start_cron.sh b/backend/scripts/start_cron.sh index d3261f3..4aaaf12 100755 --- a/backend/scripts/start_cron.sh +++ b/backend/scripts/start_cron.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -CRON_LOCAL_TIMEZONE="${BRIDGE_SYNC_LOCAL_TIMEZONE:-America/Chicago}" +CRON_LOCAL_TIMEZONE="${CRON_LOCAL_TIMEZONE:-${BRIDGE_SYNC_LOCAL_TIMEZONE:-America/Chicago}}" ZONEINFO_PATH="/usr/share/zoneinfo/${CRON_LOCAL_TIMEZONE}" +PYTHON_BIN="${PYTHON_BIN:-python}" if [ -f "${ZONEINFO_PATH}" ]; then ln -snf "${ZONEINFO_PATH}" /etc/localtime @@ -13,15 +14,19 @@ else exit 1 fi +export CRON_LOCAL_TIMEZONE export BRIDGE_SYNC_LOCAL_TIMEZONE="${BRIDGE_SYNC_LOCAL_TIMEZONE:-${CRON_LOCAL_TIMEZONE}}" -/opt/venv/bin/python3 /app/scripts/render_cron_env.py /tmp/cron-container-env.sh +"${PYTHON_BIN}" /app/scripts/render_cron_env.py /tmp/cron-container-env.sh { - printf "ADMIN_USERNAME=%s\n" "${ADMIN_USERNAME}" - printf "ADMIN_PASSWORD=%s\n" "${ADMIN_PASSWORD}" - printf "APPLICATION_URL=%s\n" "${APPLICATION_URL}" + printf "ADMIN_USERNAME=%s\n" "${ADMIN_USERNAME:-}" + printf "ADMIN_PASSWORD=%s\n" "${ADMIN_PASSWORD:-}" + printf "APPLICATION_URL=%s\n" "${APPLICATION_URL:-}" printf "BRIDGE_TRIGGER=%s\n" "${BRIDGE_TRIGGER:-nightly_cron}" + printf "OGM_TRIGGER=%s\n" "${OGM_TRIGGER:-nightly}" + printf "OGM_NIGHTLY_CRON_ENABLED=%s\n" "${OGM_NIGHTLY_CRON_ENABLED:-false}" + printf "PYTHON_BIN=%s\n" "${PYTHON_BIN}" cat /app/config/crontab } | crontab - diff --git a/backend/tests/elasticsearch/test_index.py b/backend/tests/elasticsearch/test_index.py index 420f299..0e74cd7 100644 --- a/backend/tests/elasticsearch/test_index.py +++ b/backend/tests/elasticsearch/test_index.py @@ -8,7 +8,53 @@ @pytest.mark.asyncio -async def test_process_resource_adds_allmaps_overlay_status(monkeypatch): +async def test_fetch_resources_for_index_attaches_active_ogm_repos(monkeypatch): + calls = [] + + async def fake_fetch_all(query): + calls.append(query) + if len(calls) == 1: + return [ + {"id": "unr-record"}, + {"id": "stale-ogm-record"}, + {"id": "non-ogm-record"}, + ] + return [ + { + "ogm_resource_id": "unr-record", + "ogm_repo_name": "edu.unr", + "ogm_missing_since": None, + }, + { + "ogm_resource_id": "unr-record", + "ogm_repo_name": "edu.unr", + "ogm_missing_since": None, + }, + { + "ogm_resource_id": "unr-record", + "ogm_repo_name": "edu.example", + "ogm_missing_since": None, + }, + { + "ogm_resource_id": "stale-ogm-record", + "ogm_repo_name": "edu.unr", + "ogm_missing_since": object(), + }, + ] + + monkeypatch.setattr(index_module.database, "fetch_all", fake_fetch_all) + + rows = await index_module.fetch_resources_for_index() + + assert rows == [ + {"id": "unr-record", "ogm_repo": ["edu.unr", "edu.example"]}, + {"id": "stale-ogm-record", "ogm_repo": []}, + {"id": "non-ogm-record"}, + ] + + +@pytest.fixture +def stub_process_resource_lookups(monkeypatch): async def fake_get_resource_summaries(resource_id): return [] @@ -30,6 +76,9 @@ async def fake_get_allmaps_overlay_status(resource_id): fake_get_allmaps_overlay_status, ) + +@pytest.mark.asyncio +async def test_process_resource_adds_allmaps_overlay_status(stub_process_resource_lookups): indexed = await index_module.process_resource( { "id": "allmaps-map", @@ -39,3 +88,48 @@ async def fake_get_allmaps_overlay_status(resource_id): ) assert indexed["b1g_georeferenced_allmaps_b"] is True + + +@pytest.mark.asyncio +async def test_process_resource_uses_explicit_ogm_repo(stub_process_resource_lookups): + indexed = await index_module.process_resource( + { + "id": "unr-record", + "dct_title_s": "UNR record", + "ogm_repo": "edu.unr", + } + ) + + assert indexed["ogm_repo"] == ["edu.unr"] + + +@pytest.mark.asyncio +async def test_process_resource_prefers_explicit_ogm_repo_over_tags( + stub_process_resource_lookups, +): + indexed = await index_module.process_resource( + { + "id": "unr-record", + "dct_title_s": "UNR record", + "ogm_repo": ["edu.unr"], + "b1g_adminTags_sm": ["ogm_repo:edu.stale", "ogm:stale"], + } + ) + + assert indexed["ogm_repo"] == ["edu.unr"] + + +@pytest.mark.asyncio +async def test_process_resource_does_not_fallback_to_tags_with_empty_explicit_ogm_repo( + stub_process_resource_lookups, +): + indexed = await index_module.process_resource( + { + "id": "stale-ogm-record", + "dct_title_s": "Stale OGM record", + "ogm_repo": [], + "b1g_adminTags_sm": ["ogm_repo:edu.unr", "ogm:unr"], + } + ) + + assert "ogm_repo" not in indexed diff --git a/backend/tests/scripts/test_ogm_importer.py b/backend/tests/scripts/test_ogm_importer.py new file mode 100644 index 0000000..4255354 --- /dev/null +++ b/backend/tests/scripts/test_ogm_importer.py @@ -0,0 +1,16 @@ +from scripts.ogm_importer import derive_repo_name_from_path, inject_ogm_repo_tags + + +def test_derive_repo_name_from_metadata_aardvark_path(tmp_path): + ogm_path = tmp_path / "opengeometadata" + record_path = ogm_path / "edu.unr" / "metadata-aardvark" / "records" / "item.json" + + assert derive_repo_name_from_path(str(record_path), str(ogm_path)) == "edu.unr" + + +def test_inject_ogm_repo_tags_preserves_existing_tags_and_adds_alias(): + record = {"id": "unr-test", "b1g_adminTags_sm": ["curated"]} + + inject_ogm_repo_tags(record, "edu.unr") + + assert record["b1g_adminTags_sm"] == ["curated", "ogm_repo:edu.unr", "ogm:unr"] diff --git a/backend/tests/scripts/test_populate_ogm_repos.py b/backend/tests/scripts/test_populate_ogm_repos.py new file mode 100644 index 0000000..4187777 --- /dev/null +++ b/backend/tests/scripts/test_populate_ogm_repos.py @@ -0,0 +1,64 @@ +import json + +from scripts import populate_ogm_repos + + +class FakeResponse: + def __init__(self, status_code, body): + self.status_code = status_code + self._body = body + self.text = json.dumps(body) + + def json(self): + return self._body + + +def setup_function(): + populate_ogm_repos._BAD_GITHUB_TOKEN_WARNING_SHOWN = False + populate_ogm_repos._REJECTED_GITHUB_TOKENS.clear() + + +def test_list_org_repos_retries_without_rejected_token(monkeypatch, capsys): + responses = [ + FakeResponse(401, {"message": "Bad credentials"}), + FakeResponse(200, [{"name": "edu.example"}]), + FakeResponse(200, []), + ] + calls = [] + + def fake_get(url, headers, params, timeout): + calls.append({"url": url, "headers": headers, "params": params, "timeout": timeout}) + return responses.pop(0) + + monkeypatch.setattr(populate_ogm_repos.requests, "get", fake_get) + + repos = populate_ogm_repos.list_org_repos("OpenGeoMetadata", "bad-token") + + assert repos == [{"name": "edu.example"}] + assert calls[0]["headers"]["Authorization"] == "Bearer bad-token" + assert "Authorization" not in calls[1]["headers"] + assert "Authorization" not in calls[2]["headers"] + assert "configured GitHub token was rejected with 401" in capsys.readouterr().err + + +def test_repo_has_metadata_aardvark_retries_without_rejected_token(monkeypatch): + responses = [ + FakeResponse(401, {"message": "Bad credentials"}), + FakeResponse(200, [{"name": "geoblacklight.json"}]), + ] + calls = [] + + def fake_get(url, headers, params, timeout): + calls.append({"url": url, "headers": headers, "params": params, "timeout": timeout}) + return responses.pop(0) + + monkeypatch.setattr(populate_ogm_repos.requests, "get", fake_get) + + assert ( + populate_ogm_repos.repo_has_metadata_aardvark( + "OpenGeoMetadata", "edu.example", "main", "bad-token" + ) + is True + ) + assert calls[0]["headers"]["Authorization"] == "Bearer bad-token" + assert "Authorization" not in calls[1]["headers"] diff --git a/backend/tests/scripts/test_render_cron_env.py b/backend/tests/scripts/test_render_cron_env.py index c5c5ea6..95d78a5 100644 --- a/backend/tests/scripts/test_render_cron_env.py +++ b/backend/tests/scripts/test_render_cron_env.py @@ -8,8 +8,11 @@ def test_render_cron_env_exports_filters_and_quotes_values(): rendered = render_cron_env_exports( { + "CRON_LOCAL_TIMEZONE": "America/Chicago", "DATABASE_URL": "postgresql://user:p@ss word@db.example/btaa", + "GITHUB_TOKEN": "ghp_example", "REDIS_HOST": "redis.internal", + "OGM_NIGHTLY_CRON_ENABLED": "false", "BRIDGE_SYNC_LOCAL_TIMEZONE": "America/Chicago", "BACKUP_ENABLED": "true", "AWS_ACCESS_KEY_ID": "example-key", @@ -17,8 +20,11 @@ def test_render_cron_env_exports_filters_and_quotes_values(): } ) + assert "export CRON_LOCAL_TIMEZONE=America/Chicago" in rendered assert "export DATABASE_URL='postgresql://user:p@ss word@db.example/btaa'" in rendered + assert "export GITHUB_TOKEN=ghp_example" in rendered assert "export REDIS_HOST=redis.internal" in rendered + assert "export OGM_NIGHTLY_CRON_ENABLED=false" in rendered assert "export BRIDGE_SYNC_LOCAL_TIMEZONE=America/Chicago" in rendered assert "export BACKUP_ENABLED=true" in rendered assert "export AWS_ACCESS_KEY_ID=example-key" in rendered diff --git a/backend/tests/services/test_image_service.py b/backend/tests/services/test_image_service.py index 99ae1d2..546dce9 100644 --- a/backend/tests/services/test_image_service.py +++ b/backend/tests/services/test_image_service.py @@ -992,9 +992,7 @@ def test_get_thumbnail_source_url_checks_dct_references_when_distributions_exist thumbnail_url = "https://images.example.edu/resource-thumb.jpg" metadata = { "id": "resource-with-distributions", - "dct_references_s": json.dumps( - {"http://schema.org/thumbnailUrl": thumbnail_url} - ), + "dct_references_s": json.dumps({"http://schema.org/thumbnailUrl": thumbnail_url}), } distribution_context = SimpleNamespace( by_uri={ diff --git a/backend/tests/services/test_ogm_importer_normalization.py b/backend/tests/services/test_ogm_importer_normalization.py index a373d9f..0717d8b 100644 --- a/backend/tests/services/test_ogm_importer_normalization.py +++ b/backend/tests/services/test_ogm_importer_normalization.py @@ -51,3 +51,15 @@ def test_normalize_record_copies_b1g_publication_state_to_publication_state(): assert normalized["publication_state"] == "published" assert normalized["b1g_publication_state_s"] == "published" + + +def test_normalize_record_injects_repo_tags(): + importer = OGMResourceImporter() + record = { + "id": "unr-test-id", + "b1g_adminTags_sm": ["curated"], + } + + normalized = importer._normalize_record(record, repo_name="edu.unr") + + assert normalized["b1g_adminTags_sm"] == ["curated", "ogm_repo:edu.unr", "ogm:unr"] diff --git a/backend/tests/services/test_ogm_repo_filter_parsing.py b/backend/tests/services/test_ogm_repo_filter_parsing.py index aad2a39..fcd1b3f 100644 --- a/backend/tests/services/test_ogm_repo_filter_parsing.py +++ b/backend/tests/services/test_ogm_repo_filter_parsing.py @@ -3,6 +3,14 @@ from app.services.search_service import SearchService +@pytest.mark.unit +def test_ogm_repo_filter_maps_to_include_filters(): + svc = SearchService() + include_filters, exclude_filters = svc.extract_new_style_filters("ogm_repo=edu.unr") + assert exclude_filters == {} + assert include_filters.get("ogm_repo") == ["edu.unr"] + + @pytest.mark.unit def test_ogm_repo_bracket_filter_maps_to_include_filters(): svc = SearchService() diff --git a/backend/tests/test_kamal_deploy_config.py b/backend/tests/test_kamal_deploy_config.py index 6d4eece..ba99aed 100644 --- a/backend/tests/test_kamal_deploy_config.py +++ b/backend/tests/test_kamal_deploy_config.py @@ -28,3 +28,24 @@ def test_prd_secret_override_keeps_base_secrets(): "config/deploy.prd.yml env.secret replaces the base list; " f"missing inherited secrets: {sorted(missing)}" ) + + +def test_kamal_cron_role_and_crontab_are_wired(): + base_config = _load_deploy_config("config/deploy.yml") + cron_config = base_config["servers"]["cron"] + crontab = (REPO_ROOT / "config/crontab").read_text() + dockerfile = (REPO_ROOT / "Dockerfile").read_text() + + assert "start_cron.sh" in cron_config["cmd"] + assert cron_config["options"]["user"] == "root" + assert base_config["env"]["clear"]["OGM_NIGHTLY_CRON_ENABLED"] == "false" + assert "GITHUB_TOKEN" in base_config["env"]["secret"] + + assert "trigger_ogm_nightly_sync.py" in crontab + assert "OGM_NIGHTLY_CRON_ENABLED" in crontab + assert "generate_sitemap.py" in crontab + assert "prune_generated_api_response_cache.py" in crontab + + assert "cron" in dockerfile + assert "COPY config/crontab ./config/crontab" in dockerfile + assert "start_cron.sh" in dockerfile diff --git a/config/crontab b/config/crontab new file mode 100644 index 0000000..d391c05 --- /dev/null +++ b/config/crontab @@ -0,0 +1,29 @@ +# Cron job definitions for the Kamal cron container. +# +# This file is copied into the image and loaded by backend/scripts/start_cron.sh. +# Cron strips most container env vars; BASH_ENV reloads the startup snapshot +# written by backend/scripts/render_cron_env.py before each job starts. + +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +BASH_ENV=/app/scripts/cron_env.sh +CRON_TZ=America/Chicago +TZ=America/Chicago +PYTHON_BIN=/usr/local/bin/python + +# Daily 2:15 AM local: optional in-container OGM repo refresh + harvest enqueue. +# This is disabled by default because .github/workflows/ogm-nightly-sync.yml is +# currently the active nightly scheduler. Set OGM_NIGHTLY_CRON_ENABLED=true only +# after disabling the GitHub Actions schedule, or production will enqueue twice. +15 2 * * * if [ "${OGM_NIGHTLY_CRON_ENABLED:-false}" = "true" ]; then "$PYTHON_BIN" /app/scripts/trigger_ogm_nightly_sync.py; else echo "$(date -Is) OGM nightly cron disabled; GitHub Actions workflow is primary"; fi >> /proc/1/fd/1 2>> /proc/1/fd/2 + +# Daily 4:15 AM local: regenerate sitemap XML after overnight content refreshes. +15 4 * * * "$PYTHON_BIN" /app/scripts/generate_sitemap.py >> /proc/1/fd/1 2>> /proc/1/fd/2 + +# Daily 4:45 AM local: roll up analytics, create future partitions, and drop +# expired raw partitions. +45 4 * * * "$PYTHON_BIN" /app/backend/scripts/manage_analytics_storage.py --mode maintenance >> /proc/1/fd/1 2>> /proc/1/fd/2 + +# Hourly: prune expired durable API response cache rows so the Postgres L2 cache +# stays bounded even if Redis is cold or search traffic is high. +17 * * * * "$PYTHON_BIN" /app/backend/scripts/prune_generated_api_response_cache.py >> /proc/1/fd/1 2>> /proc/1/fd/2 diff --git a/config/deploy.yml b/config/deploy.yml index 1212ead..cc29e9d 100644 --- a/config/deploy.yml +++ b/config/deploy.yml @@ -17,6 +17,14 @@ servers: hosts: - ogm.geo4lib.app cmd: bash -lc "cd /app/backend && exec celery -A app.tasks.worker worker -E --loglevel=INFO --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --prefetch-multiplier=1" + cron: + hosts: + - ogm.geo4lib.app + cmd: bash -lc "/app/scripts/start_cron.sh" + options: + cpus: 0.25 + memory: 256m + user: root proxy: ssl: true @@ -54,7 +62,12 @@ env: APPLICATION_URL: https://ogm.geo4lib.app OPENGEOMETADATA_API_BASE_URL: https://ogm.geo4lib.app PYTHONPATH: /app/backend + PYTHON_BIN: /usr/local/bin/python IS_DOCKER: "true" + CRON_LOCAL_TIMEZONE: America/Chicago + OGM_TRIGGER: nightly + OGM_NIGHTLY_CRON_ENABLED: "false" + RATE_LIMIT_ENABLED: "false" secret: - ADMIN_USERNAME diff --git a/config/ogm-owned-paths.txt b/config/ogm-owned-paths.txt index 0b0e5b3..700ccb2 100644 --- a/config/ogm-owned-paths.txt +++ b/config/ogm-owned-paths.txt @@ -5,6 +5,8 @@ # and OGM harvesting behavior. Root-level entries document the wider overlay. .github/workflows/ogm-nightly-sync.yml +config/crontab +config/deploy.yml Dockerfile Makefile README.md @@ -13,6 +15,8 @@ docs/ scripts/ backend/scripts/trigger_ogm_nightly_sync.py +backend/scripts/render_cron_env.py +backend/scripts/start_cron.sh backend/scripts/prime_generated_caches.py backend/scripts/start_cache_prime_background.sh backend/scripts/run_migrations.py diff --git a/docs/backend_upstream_sync.md b/docs/backend_upstream_sync.md index 13021f7..b283347 100644 --- a/docs/backend_upstream_sync.md +++ b/docs/backend_upstream_sync.md @@ -115,6 +115,12 @@ python scripts/trigger_ogm_nightly_sync.py Production also includes `.github/workflows/ogm-nightly-sync.yml`, which SSHes to the production host nightly and runs the same in-container script. +Kamal cron support is also downstream-owned here: `config/deploy.yml` defines a +`cron` role, `config/crontab` is copied into the production image, and +`backend/scripts/start_cron.sh` loads it. The OGM nightly cron entry is gated by +`OGM_NIGHTLY_CRON_ENABLED=false` by default so the GitHub Actions workflow remains +the only active nightly scheduler unless production intentionally switches over. + For near-real-time harvesting, configure an OpenGeoMetadata organization webhook: - Payload URL: `https://ogm.geo4lib.app/api/v1/admin/ogm/webhook` diff --git a/docs/deployment.md b/docs/deployment.md index d0630bb..20f710c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -136,6 +136,12 @@ The `.kamal/secrets` file references these secrets: 6. **GITHUB_TOKEN**: GitHub API token for nightly repo discovery and harvest orchestration 7. **OPENAI_API_KEY** / **OPENAI_MODEL**: Optional AI feature configuration +If the nightly OGM workflow fails with `GitHub API error listing repos: 401` +and `Bad credentials`, the deployed `GITHUB_TOKEN` has expired, been revoked, or +was copied incorrectly. Update the secret source used by `.kamal/secrets`, then +reboot or redeploy the app containers so Kamal rewrites the host env file and the +running web/worker/cron roles receive the new value. + ### Nightly OGM Harvest Workflow Secrets The repo also includes `.github/workflows/ogm-nightly-sync.yml`, which SSHes to the @@ -148,6 +154,18 @@ Configure these GitHub Actions secrets for that workflow: 3. **OGM_KAMAL_SSH_USER**: SSH username with Docker access on the host 4. **OGM_KAMAL_SSH_PRIVATE_KEY**: private key matching that SSH user +### Kamal Cron Container + +Kamal also runs a `cron` role from `config/deploy.yml`. The container loads +`config/crontab` through `backend/scripts/start_cron.sh`, which snapshots the +container environment for cron jobs before launching `cron -f`. + +The OGM nightly harvest command is present in `config/crontab`, but it is gated by +`OGM_NIGHTLY_CRON_ENABLED=false` by default because the GitHub Actions workflow +above is currently the active nightly scheduler. To move scheduling fully into +Kamal cron, set `OGM_NIGHTLY_CRON_ENABLED=true` and disable the scheduled +GitHub Actions trigger so production does not enqueue duplicate harvests. + ## Environment Variables ### Application Environment @@ -173,6 +191,9 @@ env: APP_MODE: production APP_ENV: production APPLICATION_URL: https://ogm.geo4lib.app + CRON_LOCAL_TIMEZONE: America/Chicago + OGM_TRIGGER: nightly + OGM_NIGHTLY_CRON_ENABLED: "false" secret: - ADMIN_USERNAME