feat: rename normalized chemistry results to major chemistry results and update related configurations

Author: jirhiker

.github/workflows/tests.yml

@@ -37,8 +37,8 @@ jobs:
 
       - name: Start database (PostGIS)
         run: |
-          docker compose build db_dev
-          docker compose up -d db_dev
+          docker compose build db
+          docker compose up -d db
 
       - name: Wait for database readiness
         run: |
@@ -122,8 +122,8 @@ jobs:
 
       - name: Start database (PostGIS)
         run: |
-          docker compose build db_dev
-          docker compose up -d db_dev
+          docker compose build db
+          docker compose up -d db
 
       - name: Wait for database readiness
         run: |

README.md

@@ -199,12 +199,14 @@ docker compose up --build
 
 Notes:
 * Requires Docker Desktop.
-* By default, spins up two containers: `db_dev` (PostGIS/PostgreSQL) and `app` (FastAPI API service).
-* `db_test` is opt-in via profile: `docker compose --profile test up`.
+* By default, spins up two containers: `db` (PostGIS/PostgreSQL) and `app` (FastAPI API service).
+* `db` initializes both application databases in the same Postgres service:
+  * `ocotilloapi_dev`
+  * `ocotilloapi_test`
 * `alembic upgrade head` runs on app startup after `docker compose up`.
 * Compose uses hardcoded DB names:
   * dev: `ocotilloapi_dev`
-  * test: `ocotilloapi_test`
+  * test: `ocotilloapi_test` (created by init SQL in `docker/db/init/01-create-test-db.sql`)
 * The database listens on port `5432` both inside the container and on your host. Ensure `POSTGRES_PORT=5432` and `POSTGRES_DB=ocotilloapi_dev` in your `.env` to run local commands against the Docker dev DB (e.g., `uv run pytest`, `uv run python -m transfers.transfer`).
 
 #### Staging Data

alembic/versions/b6f7a8b9c0d1_add_normalized_chemistry_results_materialized_view.py

@@ -80,11 +80,11 @@ def _static_analyte_unit_columns() -> str:
     )
 
 
-def _create_normalized_chemistry_results_view() -> str:
+def _create_major_chemistry_results_view() -> str:
     static_columns = _static_analyte_select_columns()
     static_unit_columns = _static_analyte_unit_columns()
     return f"""
-        CREATE MATERIALIZED VIEW ogc_normalized_chemistry_results AS
+        CREATE MATERIALIZED VIEW ogc_major_chemistry_results AS
         WITH latest_location AS (
 {LATEST_LOCATION_CTE}
         ),
@@ -100,7 +100,10 @@ def _create_normalized_chemistry_results_view() -> str:
             FROM "NMA_MajorChemistry" AS mc
             JOIN "NMA_Chemistry_SampleInfo" AS csi
                 ON csi.id = mc.chemistry_sample_info_id
+            JOIN thing AS t
+                ON t.id = csi.thing_id
             WHERE mc."SampleValue" IS NOT NULL
+              AND t.thing_type = 'water well'
         ),
         normalized_rows AS (
             SELECT
@@ -246,7 +249,6 @@ def _create_normalized_chemistry_results_view() -> str:
         JOIN latest_location AS ll ON ll.thing_id = t.id
         JOIN location AS l ON l.id = ll.location_id
         WHERE lr.rn = 1
-          AND t.thing_type = 'water well'
         GROUP BY t.id, ll.location_id, t.name, t.thing_type, l.point
     """
 
@@ -266,29 +268,31 @@ def upgrade() -> None:
     if not required_tables.issubset(existing_tables):
         missing = sorted(t for t in required_tables if t not in existing_tables)
         raise RuntimeError(
-            "Cannot create ogc_normalized_chemistry_results. Missing required tables: "
+            "Cannot create ogc_major_chemistry_results. Missing required tables: "
             + ", ".join(missing)
         )
 
+    op.execute(text("DROP MATERIALIZED VIEW IF EXISTS ogc_major_chemistry_results"))
     op.execute(
         text("DROP MATERIALIZED VIEW IF EXISTS ogc_normalized_chemistry_results")
     )
-    op.execute(text(_create_normalized_chemistry_results_view()))
+    op.execute(text(_create_major_chemistry_results_view()))
     op.execute(
         text(
-            "COMMENT ON MATERIALIZED VIEW ogc_normalized_chemistry_results IS "
+            "COMMENT ON MATERIALIZED VIEW ogc_major_chemistry_results IS "
             "'Latest major-chemistry analyte values per location, pivoted into static analyte columns.'"
         )
     )
     op.execute(
         text(
-            "CREATE UNIQUE INDEX ux_ogc_normalized_chemistry_results_id "
-            "ON ogc_normalized_chemistry_results (id)"
+            "CREATE UNIQUE INDEX ux_ogc_major_chemistry_results_id "
+            "ON ogc_major_chemistry_results (id)"
         )
     )
 
 
 def downgrade() -> None:
+    op.execute(text("DROP MATERIALIZED VIEW IF EXISTS ogc_major_chemistry_results"))
     op.execute(
         text("DROP MATERIALIZED VIEW IF EXISTS ogc_normalized_chemistry_results")
     )

alembic/versions/c7f8a9b0d1e2_add_minor_chemistry_wells_materialized_view.py

@@ -137,7 +137,6 @@ def _create_minor_chemistry_wells_view() -> str:
                 mtc.id AS result_id,
                 COALESCE(mtc.analysis_date::timestamp, csi."CollectionDate") AS observation_datetime,
                 trim(mtc.analyte) AS analyte_name,
-                trim(mtc.symbol) AS symbol_name,
                 mtc.sample_value::double precision AS sample_value,
                 mtc.units AS units
             FROM "NMA_MinorTraceChemistry" AS mtc
@@ -162,15 +161,6 @@ def _create_minor_chemistry_wells_view() -> str:
                     ),
                     ''
                 ) AS analyte_token,
-                NULLIF(
-                    regexp_replace(
-                        lower(trim(coalesce(cr.symbol_name, ''))),
-                        '[^a-z0-9]+',
-                        '',
-                        'g'
-                    ),
-                    ''
-                ) AS symbol_token,
                 cr.sample_value,
                 cr.units
             FROM chemistry_rows AS cr

cli/cli.py

@@ -55,7 +55,7 @@ class SmokePopulation(str, Enum):
     "ogc_avg_tds_wells",
     "ogc_depth_to_water_trend_wells",
     "ogc_water_well_summary",
-    "ogc_normalized_chemistry_results",
+    "ogc_major_chemistry_results",
     "ogc_minor_chemistry_wells",
 )
 

core/pygeoapi-config.yml

@@ -172,7 +172,7 @@ resources:
         table: ogc_water_well_summary
         geom_field: point
 
-  normalized_chemistry_results:
+  major_chemistry_results:
     type: collection
     title: Major Chemistry (Water Wells)
     description: Latest major chemistry analyte values for water wells, represented as static analyte columns.
@@ -192,7 +192,7 @@ resources:
           password: {postgres_password_env}
           search_path: [public]
         id_field: id
-        table: ogc_normalized_chemistry_results
+        table: ogc_major_chemistry_results
         geom_field: point
 
   minor_chemistry_wells:

docker-compose.yml

@@ -1,7 +1,7 @@
 # keep docker-compose.yml in root directory to configure with root .env
 
 services:
-  db_dev:
+  db:
     build:
       context: .
       dockerfile: ./docker/db/Dockerfile
@@ -14,33 +14,13 @@ services:
       - 5432:5432
     volumes:
       - postgres_data_dev:/var/lib/postgresql/data
+      - ./docker/db/init:/docker-entrypoint-initdb.d:ro
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ocotilloapi_dev"]
       interval: 2s
       timeout: 5s
       retries: 20
 
-  db_test:
-    profiles:
-      - test
-    build:
-      context: .
-      dockerfile: ./docker/db/Dockerfile
-    platform: linux/amd64
-    environment:
-      - POSTGRES_USER=${POSTGRES_USER}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - POSTGRES_DB=ocotilloapi_test
-    ports:
-      - 5433:5432
-    volumes:
-      - postgres_data_test:/var/lib/postgresql/data
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ocotilloapi_test"]
-      interval: 2s
-      timeout: 5s
-      retries: 20
-
   app:
     build: 
       context: .
@@ -49,20 +29,19 @@ services:
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_DB=ocotilloapi_dev
-      - POSTGRES_HOST=db_dev
+      - POSTGRES_HOST=db
       - POSTGRES_PORT=5432
       - MODE=${MODE}
       - AUTHENTIK_DISABLE_AUTHENTICATION=${AUTHENTIK_DISABLE_AUTHENTICATION}
     ports:
       - 8000:8000
     depends_on:
-      db_dev:
+      db:
         condition: service_healthy # <-- wait for DB to be ready
     links:
-      - db_dev
+      - db
     volumes:
       - .:/app
 
 volumes:
   postgres_data_dev:
-  postgres_data_test:

docker/db/init/01-create-test-db.sql

@@ -0,0 +1,10 @@
+-- Initialize test database inside the same Postgres service used for dev.
+-- This script runs only when the data directory is first initialized.
+
+CREATE DATABASE ocotilloapi_test;
+
+\connect ocotilloapi_dev
+CREATE EXTENSION IF NOT EXISTS postgis;
+
+\connect ocotilloapi_test
+CREATE EXTENSION IF NOT EXISTS postgis;

tests/__init__.py

@@ -14,13 +14,29 @@
 # limitations under the License.
 # ===============================================================================
 import os
+import socket
 from functools import lru_cache
 
 from dotenv import load_dotenv
 
 # Load .env file BEFORE importing anything else
-# Use override=True to override conflicting shell environment variables
-load_dotenv(override=True)
+# Use override=False so explicit shell environment variables can override .env
+load_dotenv(override=False)
+
+
+def _normalize_test_db_host() -> None:
+    """Fallback docker-compose hostnames to localhost for host-run tests."""
+    for env_name in ("POSTGRES_HOST", "PYGEOAPI_POSTGRES_HOST"):
+        host = (os.environ.get(env_name) or "").strip()
+        if host != "db":
+            continue
+        try:
+            socket.gethostbyname(host)
+        except OSError:
+            os.environ[env_name] = "localhost"
+
+
+_normalize_test_db_host()
 
 # for safety don't test on the production database port
 os.environ["POSTGRES_PORT"] = "5432"

tests/conftest.py

@@ -1,4 +1,5 @@
 import os
+import socket
 
 import pytest
 from alembic import command
@@ -16,7 +17,15 @@
 
 
 def pytest_configure():
-    load_dotenv(override=True)
+    load_dotenv(override=False)
+    for env_name in ("POSTGRES_HOST", "PYGEOAPI_POSTGRES_HOST"):
+        host = (os.environ.get(env_name) or "").strip()
+        if host != "db":
+            continue
+        try:
+            socket.gethostbyname(host)
+        except OSError:
+            os.environ[env_name] = "localhost"
     os.environ.setdefault("POSTGRES_PORT", "54321")
     # NOTE: This hardcoded secret key is for tests only and must NEVER be used in production.
     os.environ.setdefault("SESSION_SECRET_KEY", "test-session-secret-key")