From 61ae0f625d9a6511c366190508548fc4e0896699 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 7 Aug 2025 15:47:47 -0600 Subject: [PATCH 01/16] refactor: removed admin and old schemas feat: added observation/water-chemistry --- admin/__init__.py | 17 --- admin/base.py | 48 ------- admin/user.py | 84 ------------ api/geochronology.py | 2 +- api/geothermal.py | 16 +-- api/observation.py | 17 ++- core/app.py | 12 -- core/lexicon.json | 17 +-- db/observation.py | 8 ++ main.py | 11 -- pyproject.toml | 1 - schemas/__init__.py | 30 ----- schemas/create/__init__.py | 17 --- schemas/create/collabnet.py | 24 ---- schemas/form.py | 123 ------------------ schemas/response/__init__.py | 17 --- schemas/response/chemistry.py | 33 ----- schemas_v2/__init__.py | 11 +- schemas_v2/contact.py | 2 +- .../create => schemas_v2}/geochronology.py | 0 {schemas/create => schemas_v2}/geothermal.py | 0 schemas_v2/group.py | 2 +- schemas_v2/lexicon.py | 2 +- schemas_v2/location.py | 2 +- schemas_v2/observation.py | 4 + schemas_v2/thing.py | 2 +- tests/test_observation.py | 20 +++ uv.lock | 17 +-- 28 files changed, 68 insertions(+), 471 deletions(-) delete mode 100644 admin/__init__.py delete mode 100644 admin/base.py delete mode 100644 admin/user.py delete mode 100644 schemas/__init__.py delete mode 100644 schemas/create/__init__.py delete mode 100644 schemas/create/collabnet.py delete mode 100644 schemas/form.py delete mode 100644 schemas/response/__init__.py delete mode 100644 schemas/response/chemistry.py rename {schemas/create => schemas_v2}/geochronology.py (100%) rename {schemas/create => schemas_v2}/geothermal.py (100%) diff --git a/admin/__init__.py b/admin/__init__.py deleted file mode 100644 index 8e546ddc2..000000000 --- a/admin/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== - -# ============= EOF ============================================= diff --git a/admin/base.py b/admin/base.py deleted file mode 100644 index bba9f65e9..000000000 --- a/admin/base.py +++ /dev/null @@ -1,48 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -from typing import Any -from geoalchemy2.shape import to_shape -from fastadmin import register, SqlAlchemyModelAdmin, WidgetType - -from db.engine import async_database_sessionmaker -from db.location import Location - - -@register(Location, sqlalchemy_sessionmaker=async_database_sessionmaker) -class SampleLocationsAdmin(SqlAlchemyModelAdmin): - """ - Admin interface for SampleLocations. - This class is a placeholder for future implementation. - """ - - list_display = ("name",) - - async def serialize_obj(self, obj: Any, list_view: bool = False) -> dict: - """ - Serialize the SampleLocation object for display. - This method can be customized to include additional fields or formatting. - """ - print(f"Serializing SampleLocation object: {obj}") - return { - "id": obj.id, - "name": obj.name, - "description": obj.description, - "point": to_shape(obj.point).wkt if obj.point else None, - "created_at": obj.created_at.isoformat() if obj.created_at else None, - } - - -# ============= EOF ============================================= diff --git a/admin/user.py b/admin/user.py deleted file mode 100644 index 6a88e5f01..000000000 --- a/admin/user.py +++ /dev/null @@ -1,84 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -import uuid -from typing import Any - -from fastadmin import register, SqlAlchemyModelAdmin, WidgetType -from sqlalchemy import update, select - -from db.engine import async_database_sessionmaker -from db.base import User - - -@register(User, sqlalchemy_sessionmaker=async_database_sessionmaker) -class UserModelAdmin(SqlAlchemyModelAdmin): - list_display = ("id", "username", "is_superuser") - list_display_links = ("id", "username") - list_filter = ("id", "username", "is_superuser") - search_fields = ("username",) - formfield_overrides = { # noqa: RUF012 - "username": (WidgetType.SlugInput, {"required": True}), - "password": (WidgetType.PasswordInput, {"passwordModalForm": True}), - "avatar_url": ( - WidgetType.Upload, - { - "required": False, - # Disable crop image for upload field - # "disableCropImage": True, - }, - ), - } - - async def authenticate( - self, username: str, password: str - ) -> uuid.UUID | int | None: - sessionmaker = self.get_sessionmaker() - async with sessionmaker() as session: - query = select(self.model_cls).filter_by( - username=username, password=password, is_superuser=True - ) - result = await session.scalars(query) - obj = result.first() - if not obj: - return None - return obj.id - - async def change_password(self, id: uuid.UUID | int, password: str) -> None: - sessionmaker = self.get_sessionmaker() - async with sessionmaker() as session: - # use hash password for real usage - query = ( - update(self.model_cls) - .where(User.id.in_([id])) - .values(password=password) - ) - await session.execute(query) - await session.commit() - - async def orm_save_upload_field(self, obj: Any, field: str, base64: str) -> None: - sessionmaker = self.get_sessionmaker() - async with sessionmaker() as session: - # convert base64 to bytes, upload to s3/filestorage, get url and save or save base64 as is to db (don't recomment it) - query = ( - update(self.model_cls) - .where(User.id.in_([obj.id])) - .values(**{field: base64}) - ) - await session.execute(query) - await session.commit() - - -# ============= EOF ============================================= diff --git a/api/geochronology.py b/api/geochronology.py index c2a06b1bf..1a1a5b5ac 100644 --- a/api/geochronology.py +++ b/api/geochronology.py @@ -17,7 +17,7 @@ from fastapi import APIRouter, Depends, status from db import adder from db.engine import get_db_session -from schemas.create.geochronology import CreateGeochronologyAge +from schemas_v2.geochronology import CreateGeochronologyAge from sqlalchemy.orm import Session from sqlalchemy import select diff --git a/api/geothermal.py b/api/geothermal.py index 6e6b6201b..7d6f96395 100644 --- a/api/geothermal.py +++ b/api/geothermal.py @@ -13,11 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from fastapi import APIRouter, Depends, status -from sqlalchemy.orm import Session - -from db import adder -from db.engine import get_db_session +from fastapi import APIRouter # # from db.geothermal import ( @@ -30,16 +26,6 @@ # GeothermalSampleSet, # GeothermalBottomHoleTemperatureHeader, # ) -from schemas.create.geothermal import ( - CreateTemperatureProfile, - CreateTemperatureProfileObservation, - CreateGeothermalSampleSet, - CreateBottomHoleTemperatureHeader, - CreateBottomHoleTemperature, - CreateGeothermalInterval, - CreateThermalConductivity, - CreateHeatFlow, -) router = APIRouter(prefix="/geothermal", tags=["geothermal"]) diff --git a/api/observation.py b/api/observation.py index e7ea8952b..34c2d47ea 100644 --- a/api/observation.py +++ b/api/observation.py @@ -26,7 +26,7 @@ from db.observation import Observation from schemas_v2.observation import ( CreateGroundwaterLevelObservation, - GroundwaterLevelObservationResponse, + GroundwaterLevelObservationResponse, CreateWaterChemistryObservation, ) from services.observation_helper import add_observation from services.query_helper import order_sort_filter @@ -45,6 +45,16 @@ def add_groundwater_level_observation( """ return add_observation(session, obs_data) +@router.post("/water-chemistry", status_code=HTTP_201_CREATED) +def add_water_chemistry_observation( + obs_data: CreateWaterChemistryObservation, + session: session_dependency, +): + """ + Add a new water chemistry observation to the database. + This endpoint is currently a placeholder and does not implement any functionality. + """ + return add_observation(session, obs_data) # # @router.post("/geothermal", status_code=HTTP_201_CREATED) @@ -71,7 +81,6 @@ def get_groundwater_level_observations( thing_id: int | None = None, sensor_id: int | None = None, sample_id: int | None = None, - observed_property: str | None = None, polygon: str | None = None, start_time: datetime | None = None, end_time: datetime | None = None, @@ -83,6 +92,7 @@ def get_groundwater_level_observations( Retrieve all groundwater level observations from the database. """ sql = select(Observation) + sql = sql.where(Observation.observed_property == "groundwater level") if thing_id is not None: sql = sql.join(Sample) sql = sql.where(Sample.thing_id == thing_id) @@ -90,8 +100,7 @@ def get_groundwater_level_observations( sql = sql.where(Observation.sample_id == sample_id) if sensor_id is not None: sql = sql.where(Observation.sensor_id == sensor_id) - if observed_property is not None: - sql = sql.where(Observation.observed_property == observed_property) + if start_time: sql = sql.where(Observation.observation_timestamp >= start_time) if end_time: diff --git a/core/app.py b/core/app.py index aa384e684..5a883240b 100644 --- a/core/app.py +++ b/core/app.py @@ -77,17 +77,6 @@ def init_lexicon(): session.rollback() -def create_superuser(): - from admin.user import User - - with session_ctx() as session: - user = User( - username="admin", - password="admin", - is_superuser=True, - ) - session.add(user) - @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: @@ -96,7 +85,6 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: """ if settings.get_enum("MODE") == "development": init_db() - create_superuser() init_lexicon() yield diff --git a/core/lexicon.json b/core/lexicon.json index 30c81de58..7ed87aed2 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -2,7 +2,11 @@ {"category": "qc_sample", "term": "original", "definition": ""}, {"category": "qc_sample", "term": "duplicate", "definition": ""}, - + {"category": "units", "term": "dimensionless", "definition": ""}, + + {"category": "observed_property", "term": "groundwater level", "definition": "groundwater level measurement" }, + {"category": "observed_property", "term": "pH", "definition": "pH"}, + {"category": "observed_property", "term": "Alkalinity as CaCO3", "definition": "Alkalinity as CaCO3"}, {"term": "groundwater", "definition": "groundwater sample from a well", "category": "sample_type"}, @@ -29,7 +33,6 @@ {"term": "continuous", "definition": "continuous sampling", "category": "collection_method"}, - {"term": "groundwater level", "definition": "groundwater level measurement", "category": "observed_property"}, {"term": "United States", "definition": "United States of America", "category": "country"}, {"term": "Canada", "definition": "Canada", "category": "country"}, @@ -63,16 +66,6 @@ {"term": "Home", "definition": "Primary", "category": "phone_type"}, {"term": "Mobile", "definition": "Primary", "category": "phone_type"}, - - {"term": "TDS", "definition": "Total Dissolved Solids", "category": "water_chemistry"}, - {"term": "Na", "definition": "Sodium", "category": "water_chemistry"}, - {"term": "Cl", "definition": "Chloride", "category": "water_chemistry"}, - {"term": "Ca", "definition": "Calcium", "category": "water_chemistry"}, - {"term": "Mg", "definition": "Magnesium", "category": "water_chemistry"}, - {"term": "SO4", "definition": "Sulfate", "category": "water_chemistry"}, - {"term": "HCO3", "definition": "Bicarbonate", "category": "water_chemistry"}, - - {"term": "ft", "definition": "feet", "category": "unit"}, {"term": "ftbgs", "definition": "feet below ground surface", "category": "unit"}, {"term": "F", "definition": "Fahrenheit", "category": "unit"}, diff --git a/db/observation.py b/db/observation.py index dd0953378..1b3bce799 100644 --- a/db/observation.py +++ b/db/observation.py @@ -92,6 +92,14 @@ class Observation(Base, AuditMixin, ReleaseMixin): doc="Temperature of the geothermal observation in degrees Celsius", ) + # water chemistry + value = mapped_column( + Float, + nullable=True, + ) + units = lexicon_term() + + sensor = relationship("Sensor") sample = relationship("Sample") diff --git a/main.py b/main.py index 2c8ee8576..a7377d68f 100644 --- a/main.py +++ b/main.py @@ -2,16 +2,10 @@ from fastapi_pagination import add_pagination -os.environ["ADMIN_USER_MODEL"] = "User" -os.environ["ADMIN_USER_MODEL_USERNAME_FIELD"] = "username" -os.environ["ADMIN_SECRET_KEY"] = "secret" - from starlette.middleware.cors import CORSMiddleware from core.app import app -from fastadmin import fastapi_app as admin_app - from api.group import router as group_router from api.contact import router as contact_router from api.location import router as location_router @@ -50,11 +44,6 @@ app.include_router(search_router) app.include_router(thing_router) -from admin.user import * # noqa: F401, F403 -from admin.base import * # noqa: F401, F403 - -app.mount("/admin", admin_app) - app.add_middleware( CORSMiddleware, diff --git a/pyproject.toml b/pyproject.toml index c778fbb9b..1bf38f5e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ dependencies = [ "cryptography==45.0.5", "dnspython==2.7.0", "email-validator==2.2.0", - "fastadmin==0.2.22", "fastapi==0.116.1", "fastapi-pagination==0.13.3", "frozenlist==1.7.0", diff --git a/schemas/__init__.py b/schemas/__init__.py deleted file mode 100644 index fc2bd4889..000000000 --- a/schemas/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -from datetime import datetime - -from pydantic import BaseModel, ConfigDict - - -class ORMBaseModel(BaseModel): - id: int # every ORM model should have an id field - created_at: datetime - model_config = ConfigDict( - from_attributes=True, - populate_by_name=True, - ) - - -# ============= EOF ============================================= diff --git a/schemas/create/__init__.py b/schemas/create/__init__.py deleted file mode 100644 index 8e546ddc2..000000000 --- a/schemas/create/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== - -# ============= EOF ============================================= diff --git a/schemas/create/collabnet.py b/schemas/create/collabnet.py deleted file mode 100644 index 2987188c1..000000000 --- a/schemas/create/collabnet.py +++ /dev/null @@ -1,24 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -from pydantic import BaseModel - - -class CreateCollaborativeNetworkWell(BaseModel): - well_id: int - actively_monitored: bool = False - - -# ============= EOF ============================================= diff --git a/schemas/form.py b/schemas/form.py deleted file mode 100644 index f6874e8dd..000000000 --- a/schemas/form.py +++ /dev/null @@ -1,123 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -from datetime import date -from typing import List - -from pydantic import BaseModel - -from schemas_v2.location import LocationResponse - - -class WFLocation(BaseModel): - """ - A class representing a geographic location. - This class is used to validate and process location data. - """ - - point: str # Assuming point is a string representation of a geographic point (e.g., 'POINT(-105.0 40.0)') - # You can add more fields as necessary, such as latitude, longitude, etc. - - -class WFContact(BaseModel): - """ - A class representing a contact information. - This class is used to validate and process contact data. - """ - - name: str - phone: str # Assuming phone is a string representation of a phone number - email: str # Assuming email is a string representation of an email address - # You can add more fields as necessary, such as address, etc. - - -class WFWell(BaseModel): - ose_pod_id: str | None = None # OSE POD well number, optional - api_id: str | None = None # API well number, optional - usgs_id: str | None = None # USGS well number, optional - - well_depth: float | None = None # Depth of the well in feet, optional - hole_depth: float | None = None # Depth of the hole in feet, optional - casing_diameter: float | None = None # Diameter of the casing in inches, optional - casing_depth: float | None = None # Depth of the casing in feet, optional - casing_description: str | None = None # Description of the casing, optional - construction_notes: str | None = None # Construction notes, optional - - -class WFGroup(BaseModel): - name: str - description: str | None = None - - -class WellForm(BaseModel): - """ - A class representing a form for well data submission. - This class is used to validate and process well data submissions. - """ - - location: WFLocation - well: WFWell - groups: List[WFGroup] | None = None # Optional group field - - # Define the fields for the well form - # site_id: str - # depth_to_water_ftbgs: float - # date_measured: str # ISO format date string - # time_measured: str # ISO format time string - # level_status: str - # data_quality: str - # measuring_agency: str - # data_source: str - # measurement_method: str - # measured_by: str - # site_notes: str = None # Optional field - # public_release: bool = True # Default to True - - -class WellFormResponse(BaseModel): - """ - A class representing the response for a well form submission. - This class is used to structure the response data after a successful submission. - """ - - location: LocationResponse - # You can add more fields to the response as necessary, such as status messages, etc. - - -class GroundwaterLevelForm(BaseModel): - """ - A class representing a form for groundwater level data submission. - This class is used to validate and process groundwater level data submissions. - """ - - well_id: int # ID of the well - depth_to_water_bgs: float # Depth to water below ground surface - measurement_date: date # ISO format date string - notes: str | None = None # Optional notes field - - -class GroundwaterLevelFormResponse(BaseModel): - """ - A class representing the response for a groundwater level form submission. - This class is used to structure the response data after a successful submission. - """ - - well_id: int - depth_to_water_bgs: float # Depth to water below ground surface - measurement_date: date # ISO format date string - notes: str | None = None # Optional notes field - - -# ============= EOF ============================================= diff --git a/schemas/response/__init__.py b/schemas/response/__init__.py deleted file mode 100644 index 8e546ddc2..000000000 --- a/schemas/response/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== - -# ============= EOF ============================================= diff --git a/schemas/response/chemistry.py b/schemas/response/chemistry.py deleted file mode 100644 index 84739b0e0..000000000 --- a/schemas/response/chemistry.py +++ /dev/null @@ -1,33 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -from schemas import ORMBaseModel - - -class WaterChemistryAnalysisSetResponse(ORMBaseModel): - well_id: int - laboratory: str - # collection_timestamp: str # ISO 8601 format - # analyses: list # List of WaterChemistryAnalysisResponse objects - # id: int # Unique identifier for the analysis set - - -class WaterChemistryAnalysisResponse(ORMBaseModel): - analysis_set_id: int - value: float - unit: str - - -# ============= EOF ============================================= diff --git a/schemas_v2/__init__.py b/schemas_v2/__init__.py index a117e7097..271869ad5 100644 --- a/schemas_v2/__init__.py +++ b/schemas_v2/__init__.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from pydantic import BaseModel +from datetime import datetime + +from pydantic import BaseModel, ConfigDict class ResourceNotFoundResponse(BaseModel): @@ -21,3 +23,10 @@ class ResourceNotFoundResponse(BaseModel): # ============= EOF ============================================= +class ORMBaseModel(BaseModel): + id: int # every ORM model should have an id field + created_at: datetime + model_config = ConfigDict( + from_attributes=True, + populate_by_name=True, + ) diff --git a/schemas_v2/contact.py b/schemas_v2/contact.py index 95eb4ad80..7b4cdb53a 100644 --- a/schemas_v2/contact.py +++ b/schemas_v2/contact.py @@ -21,7 +21,7 @@ from phonenumbers import NumberParseException from pydantic import field_validator, BaseModel -from schemas import ORMBaseModel +from schemas_v2 import ORMBaseModel from schemas_v2.thing import ThingResponse """ diff --git a/schemas/create/geochronology.py b/schemas_v2/geochronology.py similarity index 100% rename from schemas/create/geochronology.py rename to schemas_v2/geochronology.py diff --git a/schemas/create/geothermal.py b/schemas_v2/geothermal.py similarity index 100% rename from schemas/create/geothermal.py rename to schemas_v2/geothermal.py diff --git a/schemas_v2/group.py b/schemas_v2/group.py index e4cb9c4d2..e0b689f83 100644 --- a/schemas_v2/group.py +++ b/schemas_v2/group.py @@ -15,7 +15,7 @@ # =============================================================================== from pydantic import BaseModel -from schemas import ORMBaseModel +from schemas_v2 import ORMBaseModel # -------- CREATE ---------- diff --git a/schemas_v2/lexicon.py b/schemas_v2/lexicon.py index d304e8426..6ae8561ea 100644 --- a/schemas_v2/lexicon.py +++ b/schemas_v2/lexicon.py @@ -15,7 +15,7 @@ # =============================================================================== from pydantic import BaseModel -from schemas import ORMBaseModel +from schemas_v2 import ORMBaseModel # -------- CREATE ---------- diff --git a/schemas_v2/location.py b/schemas_v2/location.py index af48e8f9d..82df72dc7 100644 --- a/schemas_v2/location.py +++ b/schemas_v2/location.py @@ -18,7 +18,7 @@ from pydantic import BaseModel, field_validator from shapely import wkt -from schemas import ORMBaseModel +from schemas_v2 import ORMBaseModel """ REFACTOR TODO diff --git a/schemas_v2/observation.py b/schemas_v2/observation.py index 043909b1f..7d2ce58e9 100644 --- a/schemas_v2/observation.py +++ b/schemas_v2/observation.py @@ -38,6 +38,10 @@ class CreateGroundwaterLevelObservation(CreateBaseObservation): level_status: str +class CreateWaterChemistryObservation(CreateBaseObservation): + value: float + units: str + # # # class CreateGroundwaterLevelObservation(ChildObservationModel, GroundwaterLevelMixin): diff --git a/schemas_v2/thing.py b/schemas_v2/thing.py index 7d8285ad0..b326078f6 100644 --- a/schemas_v2/thing.py +++ b/schemas_v2/thing.py @@ -18,7 +18,7 @@ from pydantic import BaseModel, model_validator -from schemas import ORMBaseModel +from schemas_v2 import ORMBaseModel from schemas_v2.location import LocationResponse diff --git a/tests/test_observation.py b/tests/test_observation.py index cd43afba2..3bf974e09 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -19,6 +19,26 @@ # ============= Post tests ================= +def test_add_water_chemistry_observation(location, thing, sample, sensor): + response = client.post( + "/observation/water-chemistry", + json={ + "observation_timestamp": "2025-01-01T00:00:00Z", + "release_status": "draft", + "value": 7.5, + "units": "dimensionless", + "sample_id": sample.id, + "sensor_id": sensor.id, + "observed_property": "pH", + }, + ) + data = response.json() + assert response.status_code == 201 + + assert data["value"] == 7.5 + assert data["units"] == "dimensionless" + + def test_add_groundwater_observation(location, thing, sample, sensor): response = client.post( "/observation/groundwater-level", diff --git a/uv.lock b/uv.lock index 86469e801..ef121befc 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.13" [[package]] @@ -406,19 +406,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/ee/bf0adb559ad3c786f12bcbc9296b3f5675f529199bef03e2df281fa1fadb/email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631", size = 33521, upload-time = "2024-06-20T11:30:28.248Z" }, ] -[[package]] -name = "fastadmin" -version = "0.2.22" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "asgiref" }, - { name = "pyjwt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/50/2a/e5443dbf1eba8d9799862b87edcb5a8e3c8d837923aa6617222ce8f5747b/fastadmin-0.2.22.tar.gz", hash = "sha256:51703a9e0a892ce2baeceb43cafef7f5ecbbc70ee17dd3f93f25277f81bdcbe0", size = 1660674, upload-time = "2025-04-22T11:14:43.391Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/80/26/cb8393ce0c179c29b15a7ce214b22d798a547f505f4c4b0b0a11a580f2c6/fastadmin-0.2.22-py3-none-any.whl", hash = "sha256:4e8e1341a5097d9da7f2a8c5b925de8bf16253822e7b857a5d430d3b026251b8", size = 1679535, upload-time = "2025-04-22T11:14:41.578Z" }, -] - [[package]] name = "fastapi" version = "0.116.1" @@ -845,7 +832,6 @@ dependencies = [ { name = "cryptography" }, { name = "dnspython" }, { name = "email-validator" }, - { name = "fastadmin" }, { name = "fastapi" }, { name = "fastapi-pagination" }, { name = "frozenlist" }, @@ -945,7 +931,6 @@ requires-dist = [ { name = "cryptography", specifier = "==45.0.5" }, { name = "dnspython", specifier = "==2.7.0" }, { name = "email-validator", specifier = "==2.2.0" }, - { name = "fastadmin", specifier = "==0.2.22" }, { name = "fastapi", specifier = "==0.116.1" }, { name = "fastapi-pagination", specifier = "==0.13.3" }, { name = "frozenlist", specifier = "==1.7.0" }, From dc8f1321cf1de36ffd4792d1fb00ff6c4a38bad7 Mon Sep 17 00:00:00 2001 From: jirhiker Date: Thu, 7 Aug 2025 21:48:03 +0000 Subject: [PATCH 02/16] Formatting changes --- api/observation.py | 5 ++++- core/app.py | 1 - db/observation.py | 1 - schemas_v2/observation.py | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/observation.py b/api/observation.py index 34c2d47ea..550b526a6 100644 --- a/api/observation.py +++ b/api/observation.py @@ -26,7 +26,8 @@ from db.observation import Observation from schemas_v2.observation import ( CreateGroundwaterLevelObservation, - GroundwaterLevelObservationResponse, CreateWaterChemistryObservation, + GroundwaterLevelObservationResponse, + CreateWaterChemistryObservation, ) from services.observation_helper import add_observation from services.query_helper import order_sort_filter @@ -45,6 +46,7 @@ def add_groundwater_level_observation( """ return add_observation(session, obs_data) + @router.post("/water-chemistry", status_code=HTTP_201_CREATED) def add_water_chemistry_observation( obs_data: CreateWaterChemistryObservation, @@ -56,6 +58,7 @@ def add_water_chemistry_observation( """ return add_observation(session, obs_data) + # # @router.post("/geothermal", status_code=HTTP_201_CREATED) # def add_geothermal_observation( diff --git a/core/app.py b/core/app.py index 5a883240b..e25ec199d 100644 --- a/core/app.py +++ b/core/app.py @@ -77,7 +77,6 @@ def init_lexicon(): session.rollback() - @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: """ diff --git a/db/observation.py b/db/observation.py index 1b3bce799..695ace935 100644 --- a/db/observation.py +++ b/db/observation.py @@ -99,7 +99,6 @@ class Observation(Base, AuditMixin, ReleaseMixin): ) units = lexicon_term() - sensor = relationship("Sensor") sample = relationship("Sample") diff --git a/schemas_v2/observation.py b/schemas_v2/observation.py index 7d2ce58e9..d9639b9b8 100644 --- a/schemas_v2/observation.py +++ b/schemas_v2/observation.py @@ -42,6 +42,7 @@ class CreateWaterChemistryObservation(CreateBaseObservation): value: float units: str + # # # class CreateGroundwaterLevelObservation(ChildObservationModel, GroundwaterLevelMixin): From 870036a1e38a994848c39c591256bd970944f6d0 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 7 Aug 2025 23:18:52 -0600 Subject: [PATCH 03/16] merge --- .../versions/66ac1af4ba69_initial_migration.py | 4 ++-- api/observation.py | 4 ++-- core/app.py | 2 +- db/observation.py | 6 +++--- migration/migration2.py | 2 +- schemas_v2/observation.py | 18 +++++++++--------- services/observation_helper.py | 2 +- tests/test_observation.py | 6 +++--- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/alembic/versions/66ac1af4ba69_initial_migration.py b/alembic/versions/66ac1af4ba69_initial_migration.py index 97a31f5d5..e07bb5807 100644 --- a/alembic/versions/66ac1af4ba69_initial_migration.py +++ b/alembic/versions/66ac1af4ba69_initial_migration.py @@ -402,7 +402,7 @@ def upgrade() -> None: # op.create_table('observation', # sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), # sa.Column('series_id', sa.Integer(), nullable=False), - # sa.Column('observation_timestamp', sa.TIMESTAMP(), nullable=False), + # sa.Column('observation_datetime', sa.TIMESTAMP(), nullable=False), # sa.Column('observation_type', sa.String(length=100), nullable=True), # sa.Column('depth_to_water', sa.Float(), nullable=True), # sa.Column('measuring_point_height', sa.Float(), nullable=True), @@ -415,7 +415,7 @@ def upgrade() -> None: # sa.ForeignKeyConstraint(['observation_type'], ['lexicon_term.term'], ), # sa.ForeignKeyConstraint(['release_status'], ['lexicon_term.term'], ), # sa.ForeignKeyConstraint(['series_id'], ['series.id'], ondelete='CASCADE'), - # sa.PrimaryKeyConstraint('id', 'observation_timestamp') + # sa.PrimaryKeyConstraint('id', 'observation_datetime') # ) # ### end Alembic commands ### diff --git a/api/observation.py b/api/observation.py index 550b526a6..3023ae66b 100644 --- a/api/observation.py +++ b/api/observation.py @@ -105,9 +105,9 @@ def get_groundwater_level_observations( sql = sql.where(Observation.sensor_id == sensor_id) if start_time: - sql = sql.where(Observation.observation_timestamp >= start_time) + sql = sql.where(Observation.observation_datetime >= start_time) if end_time: - sql = sql.where(Observation.observation_timestamp <= end_time) + sql = sql.where(Observation.observation_datetime <= end_time) sql = order_sort_filter(sql, Observation, sort, order, filter_) return paginate(query=sql, conn=session) diff --git a/core/app.py b/core/app.py index e25ec199d..02dab1bb8 100644 --- a/core/app.py +++ b/core/app.py @@ -45,7 +45,7 @@ def init_hypertables(): # Create hypertables for time-series data with session_ctx() as session: session.execute( - text("select create_hypertable('observation', 'observation_timestamp');") + text("select create_hypertable('observation', 'observation_datetime');") ) # session.commit() diff --git a/db/observation.py b/db/observation.py index 8c8d0438f..c76fa69f2 100644 --- a/db/observation.py +++ b/db/observation.py @@ -33,7 +33,7 @@ class Observation(Base, AuditMixin, ReleaseMixin): __table_args__ = ( PrimaryKeyConstraint( "id", - "observation_timestamp", + "observation_datetime", ), {}, ) @@ -54,8 +54,8 @@ class Observation(Base, AuditMixin, ReleaseMixin): nullable=False, ) - observation_timestamp = mapped_column( - TIMESTAMP(timezone=True), nullable=False, doc="Timestamp of the observation" + observation_datetime = mapped_column( + DateTime(timezone=True), nullable=False, doc="Timestamp of the observation" ) observed_property = lexicon_term() diff --git a/migration/migration2.py b/migration/migration2.py index 2db797d0a..341fb5f89 100644 --- a/migration/migration2.py +++ b/migration/migration2.py @@ -109,7 +109,7 @@ def migrate_water_levels(session, limit=800): for row in group.itertuples(): obs = Observation() obs.series = series - obs.observation_timestamp = datetime.fromisoformat(row.DateMeasured) + obs.observation_datetime = datetime.fromisoformat(row.DateMeasured) # print("rw", row.DateMeasured, row.TimeMeasured) gwl_obs = GroundwaterLevelObservation() gwl_obs.observation = obs diff --git a/schemas_v2/observation.py b/schemas_v2/observation.py index 2706771d5..4d46ea3b2 100644 --- a/schemas_v2/observation.py +++ b/schemas_v2/observation.py @@ -28,25 +28,25 @@ class ValidateObservation(BaseModel): - @field_validator("observation_timestamp", check_fields=False) - def convert_observation_timestamp_to_utc( - observation_timestamp: AwareDatetime, + @field_validator("observation_datetime", check_fields=False) + def convert_observation_datetime_to_utc( + observation_datetime: AwareDatetime, ) -> AwareDatetime: """ Convert observation_timestamp to UTC timezone if it's not already. This runs after the Annotated validator PastDatetime() is run. """ if ( - observation_timestamp is not None - and observation_timestamp.tzinfo != timezone.utc + observation_datetime is not None + and observation_datetime.tzinfo != timezone.utc ): - return observation_timestamp.astimezone(timezone.utc) - return observation_timestamp + return observation_datetime.astimezone(timezone.utc) + return observation_datetime # -------- CREATE ---------- class CreateBaseObservation(ValidateObservation): - observation_timestamp: Annotated[AwareDatetime, PastDatetime()] + observation_datetime: Annotated[AwareDatetime, PastDatetime()] sample_id: int sensor_id: int observed_property: str @@ -87,7 +87,7 @@ class BaseObservationResponse(BaseModel): id: int sample_id: int sensor_id: int - observation_timestamp: AwareDatetime + observation_datetime: AwareDatetime observed_property: str created_at: AwareDatetime release_status: str diff --git a/services/observation_helper.py b/services/observation_helper.py index be195e838..bbfebb596 100644 --- a/services/observation_helper.py +++ b/services/observation_helper.py @@ -29,7 +29,7 @@ def add_observation(session: Session, data: BaseModel) -> Base: # if 'sample_id' not in data: # sample = Sample(thing_id=thing_id, # collection_method=data.get('collection_method', 'manual'), - # collection_timestamp=data.get('observation_timestamp')) + # collection_timestamp=data.get('observation_datetime')) # session.add(sample) # data['sample'] = sample # else: diff --git a/tests/test_observation.py b/tests/test_observation.py index 3bf974e09..c4110f516 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -23,7 +23,7 @@ def test_add_water_chemistry_observation(location, thing, sample, sensor): response = client.post( "/observation/water-chemistry", json={ - "observation_timestamp": "2025-01-01T00:00:00Z", + "observation_datetime": "2025-01-01T00:00:00Z", "release_status": "draft", "value": 7.5, "units": "dimensionless", @@ -43,7 +43,7 @@ def test_add_groundwater_observation(location, thing, sample, sensor): response = client.post( "/observation/groundwater-level", json={ - "observation_timestamp": "2025-01-01T00:00:00Z", + "observation_datetime": "2025-01-01T00:00:00Z", "release_status": "draft", "depth_to_water": 101, "measuring_point_height": 53, @@ -65,7 +65,7 @@ def test_add_groundwater_observation(location, thing, sample, sensor): # "/observation/geothermal", # json={ # "observation_id": 1, -# "observation_timestamp": "2025-01-01T00:00:00Z", +# "observation_datetime": "2025-01-01T00:00:00Z", # "depth": 100, # "temperature": 25.5, # }, From 12e63204ff0ac2843835129eb040d60383959377 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 7 Aug 2025 23:20:34 -0600 Subject: [PATCH 04/16] refactor: ORMBaseModel use AwareDatetime --- schemas_v2/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schemas_v2/__init__.py b/schemas_v2/__init__.py index 271869ad5..d5e003db1 100644 --- a/schemas_v2/__init__.py +++ b/schemas_v2/__init__.py @@ -15,18 +15,18 @@ # =============================================================================== from datetime import datetime -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, AwareDatetime class ResourceNotFoundResponse(BaseModel): detail: str -# ============= EOF ============================================= class ORMBaseModel(BaseModel): id: int # every ORM model should have an id field - created_at: datetime + created_at: AwareDatetime model_config = ConfigDict( from_attributes=True, populate_by_name=True, ) +# ============= EOF ============================================= From 3aaeedaf12e377bb2aec250c1074afa73819330a Mon Sep 17 00:00:00 2001 From: jirhiker Date: Fri, 8 Aug 2025 05:20:50 +0000 Subject: [PATCH 05/16] Formatting changes --- schemas_v2/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schemas_v2/__init__.py b/schemas_v2/__init__.py index d5e003db1..668fd8005 100644 --- a/schemas_v2/__init__.py +++ b/schemas_v2/__init__.py @@ -29,4 +29,6 @@ class ORMBaseModel(BaseModel): from_attributes=True, populate_by_name=True, ) + + # ============= EOF ============================================= From e423baea3d519f0f56645f4cdbb66bfebdca210e Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 7 Aug 2025 23:23:01 -0600 Subject: [PATCH 06/16] fix: import error --- db/observation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/observation.py b/db/observation.py index c76fa69f2..72b122d0e 100644 --- a/db/observation.py +++ b/db/observation.py @@ -18,7 +18,7 @@ Integer, TIMESTAMP, PrimaryKeyConstraint, - Float, + Float, DateTime, ) from sqlalchemy.orm import mapped_column, relationship From aec2859eeb7c2a3ed841b089739ae323aa98c9a5 Mon Sep 17 00:00:00 2001 From: jirhiker Date: Fri, 8 Aug 2025 05:23:30 +0000 Subject: [PATCH 07/16] Formatting changes --- db/observation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/observation.py b/db/observation.py index 72b122d0e..55f3f383f 100644 --- a/db/observation.py +++ b/db/observation.py @@ -18,7 +18,8 @@ Integer, TIMESTAMP, PrimaryKeyConstraint, - Float, DateTime, + Float, + DateTime, ) from sqlalchemy.orm import mapped_column, relationship From d4aca7afb06ba31c677f876879ed4bfb89e63b39 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 7 Aug 2025 23:35:21 -0600 Subject: [PATCH 08/16] feat: allow observation to be added using sample_id or field_sample_id --- schemas_v2/observation.py | 3 ++- services/observation_helper.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/schemas_v2/observation.py b/schemas_v2/observation.py index 4d46ea3b2..7a49b7326 100644 --- a/schemas_v2/observation.py +++ b/schemas_v2/observation.py @@ -47,7 +47,8 @@ def convert_observation_datetime_to_utc( # -------- CREATE ---------- class CreateBaseObservation(ValidateObservation): observation_datetime: Annotated[AwareDatetime, PastDatetime()] - sample_id: int + sample_id: int | None = None + field_sample_id: str | None = None sensor_id: int observed_property: str release_status: str diff --git a/services/observation_helper.py b/services/observation_helper.py index bbfebb596..74a258521 100644 --- a/services/observation_helper.py +++ b/services/observation_helper.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== from pydantic import BaseModel +from sqlalchemy import select from sqlalchemy.orm import Session from db import Base, Observation, Sample @@ -34,7 +35,15 @@ def add_observation(session: Session, data: BaseModel) -> Base: # data['sample'] = sample # else: # raise ValueError('Cannot specify both thing_id and sample_id') - + if 'field_sample_id' in data: + field_sample_id = data.pop('field_sample_id') + data.pop('sample_id', None) # Ensure sample_id is not set if field_sample_id is used + + sql = select(Sample).where(Sample.field_sample_id == field_sample_id) + sample = session.scalar(sql) + if not sample: + raise ValueError(f'Sample with id {field_sample_id} does not exist') + data['sample'] = sample obj = Observation(**data) session.add(obj) From dab7b984c9dfc2b30ea0cd222cc79461264804fa Mon Sep 17 00:00:00 2001 From: jirhiker Date: Fri, 8 Aug 2025 05:35:40 +0000 Subject: [PATCH 09/16] Formatting changes --- services/observation_helper.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/services/observation_helper.py b/services/observation_helper.py index 74a258521..43b46abd4 100644 --- a/services/observation_helper.py +++ b/services/observation_helper.py @@ -35,15 +35,17 @@ def add_observation(session: Session, data: BaseModel) -> Base: # data['sample'] = sample # else: # raise ValueError('Cannot specify both thing_id and sample_id') - if 'field_sample_id' in data: - field_sample_id = data.pop('field_sample_id') - data.pop('sample_id', None) # Ensure sample_id is not set if field_sample_id is used + if "field_sample_id" in data: + field_sample_id = data.pop("field_sample_id") + data.pop( + "sample_id", None + ) # Ensure sample_id is not set if field_sample_id is used - sql = select(Sample).where(Sample.field_sample_id == field_sample_id) + sql = select(Sample).where(Sample.field_sample_id == field_sample_id) sample = session.scalar(sql) if not sample: - raise ValueError(f'Sample with id {field_sample_id} does not exist') - data['sample'] = sample + raise ValueError(f"Sample with id {field_sample_id} does not exist") + data["sample"] = sample obj = Observation(**data) session.add(obj) From 0e558af75746d7686f78c017f30f87055592324a Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 8 Aug 2025 14:37:09 -0600 Subject: [PATCH 10/16] refactor: lexicon updates. migration2 updates --- api/lexicon.py | 24 +++++- core/lexicon.json | 16 ++-- db/lexicon.py | 18 ++-- db/observation.py | 2 +- migration/migration2.py | 184 ++++++++++++++++++++-------------------- schemas_v2/lexicon.py | 28 +++--- 6 files changed, 153 insertions(+), 119 deletions(-) diff --git a/api/lexicon.py b/api/lexicon.py index 240cb6996..4cb5377e2 100644 --- a/api/lexicon.py +++ b/api/lexicon.py @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from fastapi import APIRouter, Depends -from fastapi import status +from fastapi import APIRouter, Depends, Query, status from fastapi_pagination.ext.sqlalchemy import paginate from sqlalchemy import select @@ -30,6 +29,7 @@ LexiconCategoryResponse, ) from services.lexicon import add_lexicon_term +from services.query_helper import simple_all_getter, paginated_all_getter router = APIRouter( prefix="/lexicon", @@ -106,6 +106,9 @@ def get_lexicon_terms( session: session_dependency, category: str | None = None, term: str | None = None, + sort: str = None, + order: str = None, + filter_: str = Query(alias="filter", default=None), ) -> CustomPage[LexiconTermResponse]: """ Endpoint to retrieve lexicon terms. @@ -120,7 +123,22 @@ def get_lexicon_terms( if term: sql = sql.where(Lexicon.term.ilike(f"%{term}%")) + sql = order_sort_filter( + sql, Lexicon, sort=sort, order=order, filter_=filter_ + ) return paginate(query=sql, conn=session) - + # return paginated_all_getter(session, sql, filter_) + +@router.get("/category") +def get_lexicon_categories( + session: session_dependency, + sort: str = None, + order: str = None, + filter_: str = Query(alias="filter", default=None), +) -> CustomPage[LexiconCategoryResponse]: + """ + Endpoint to retrieve lexicon categories. + """ + return paginated_all_getter(session, Category, sort, order, filter_) # ============= EOF ============================================= diff --git a/core/lexicon.json b/core/lexicon.json index 7ed87aed2..c484c8294 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -8,16 +8,22 @@ {"category": "observed_property", "term": "pH", "definition": "pH"}, {"category": "observed_property", "term": "Alkalinity as CaCO3", "definition": "Alkalinity as CaCO3"}, + {"category": "release_status", "term": "draft", "definition": "draft version"}, + {"category": "release_status", "term": "provisional", "definition": "provisional version"}, + {"category": "release_status", "term": "final", "definition": "final version"}, + {"category": "release_status", "term": "published", "definition": "published version"}, + {"category": "release_status", "term": "archived", "definition": "archived version"}, + {"category": "release_status", "term": "public", "definition": "public version"}, + {"category": "release_status", "term": "private", "definition": "private version"}, + + + {"term": "groundwater", "definition": "groundwater sample from a well", "category": "sample_type"}, {"term": "water well", "definition": "a hole drill into the ground to access groundwater", "category": "thing_type"}, {"term": "spring", "definition": "a natural discharge of groundwater at the surface", "category": "thing_type"}, - {"term": "draft", "definition": "draft version", "category": "release_status"}, - {"term": "provisional", "definition": "provisional version", "category": "release_status"}, - {"term": "final", "definition": "final version", "category": "release_status"}, - {"term": "published", "definition": "published version", "category": "release_status"}, - {"term": "archived", "definition": "archived version", "category": "release_status"}, + {"term": "dry", "definition": "well is dry", "category": "level_status"}, {"term": "normal", "definition": "normal well water level status", "category": "level_status"}, diff --git a/db/lexicon.py b/db/lexicon.py index d89821dae..fe4fb20cd 100644 --- a/db/lexicon.py +++ b/db/lexicon.py @@ -29,11 +29,14 @@ class Lexicon(Base, AutoBaseMixin): term = mapped_column(String(100), unique=True, nullable=False) definition = mapped_column(String(255), nullable=False) - # category_id = mapped_column(Integer, nullable=False) - # category = mapped_column(String(255), nullable=True) + # categories = relationship( + # "Category", + # secondary="lexicon_term_category_association", + # ) + # categories = relationship("TermCategoryAssociation") def __repr__(self): - return f"" + return f"" class Category(Base, AutoBaseMixin): @@ -46,12 +49,7 @@ class Category(Base, AutoBaseMixin): name = mapped_column(String(100), unique=True, nullable=False) description = mapped_column(String(255), nullable=True) - # terms = relationship( - # "lexicon", - # backref="category", - # cascade="all, delete-orphan", - # lazy="dynamic" - # ) + def __repr__(self): return f"" @@ -73,7 +71,7 @@ class TermCategoryAssociation(Base, AutoBaseMixin): nullable=False, ) - term = relationship("Lexicon") + term = relationship("Lexicon", backref="categories") category = relationship("Category") def __repr__(self): diff --git a/db/observation.py b/db/observation.py index 55f3f383f..de680894d 100644 --- a/db/observation.py +++ b/db/observation.py @@ -91,7 +91,7 @@ class Observation(Base, AuditMixin, ReleaseMixin): doc="Temperature of the geothermal observation in degrees Celsius", ) - # water chemistry + # general observations value = mapped_column( Float, nullable=True, diff --git a/migration/migration2.py b/migration/migration2.py index 341fb5f89..d87eee65d 100644 --- a/migration/migration2.py +++ b/migration/migration2.py @@ -28,10 +28,10 @@ # from db.observation.groundwaterlevel import GroundwaterLevelObservation from db.observation import Observation -from db.series.groundwaterlevel import GroundwaterLevelSeries -from db.series.series import Series +# from db.series.groundwaterlevel import GroundwaterLevelSeries +# from db.series.series import Series from services.lexicon import add_lexicon_term -from services.thing_helper import add_well +from services.thing_helper import add_thing TRANSFORMERS = {} @@ -61,97 +61,99 @@ def make_location(row): ) return Location( - # name=row_dict["PointID"], + name=row.PointID, point=transformed_point.wkt, + release_status='public' if row.PublicRelease else 'private', # visible=row_dict["PublicRelease"], ) - -def migrate_water_levels(session, limit=800): - wd = pd.read_csv("./migration/data/water_levels.csv") - p = pd.read_csv("./migration/data/welldata.csv") - # get first 100 rows - pointids = p["PointID"].unique()[:limit] - - wd = wd[wd["PointID"].isin(pointids)] - - gwd = wd.groupby(["PointID"]) - - sensor = Sensor() - sensor.name = '"manual gwl measurement. needs to be replaced with measurementmethod(?) e.g. steel tape, eprobe, etc."' - sensor.description = "Groundwater level manual measurement" - session.add(sensor) - session.commit() - - for index, group in gwd: - - # add a series - # add a groundwater level series - thing = session.query(Thing).filter_by(name=index[0]).first() - print("Processing PointID:", index, thing) - if not thing: - continue - - print("found thing:", index, thing.id) - series = Series(name="Groundwater Level Series") - series.observed_property = "groundwater level" - series.unit = "ft" - - series.sensor = sensor - series.thing = thing - - groundwater_level_series = GroundwaterLevelSeries() - groundwater_level_series.series = series - - session.add(series) - session.add(groundwater_level_series) - - for row in group.itertuples(): - obs = Observation() - obs.series = series - obs.observation_datetime = datetime.fromisoformat(row.DateMeasured) - # print("rw", row.DateMeasured, row.TimeMeasured) - gwl_obs = GroundwaterLevelObservation() - gwl_obs.observation = obs - gwl_obs.depth_to_water = row.DepthToWater - gwl_obs.measuring_point_height = row.MPHeight - session.add(obs) - session.add(gwl_obs) - - session.commit() - # break - - # print(group) - # print('--------------------------------------------') - # break - # for index, row in group: - # print(index, row) - # print(row.PointID, row.TimeMeasured) - # print(row.PointID, row.WaterLevel, row.WaterLevelDate) - # if pd.isna(row.WaterLevel) or pd.isna(row.WaterLevelDate): - # continue - # - # obs = add_groundwater_level_observation( - # session, - # { - # "point_id": row.PointID, - # "water_level": row.WaterLevel, - # "water_level_date": row.WaterLevelDate, - # }, - # ) - # print(obs) - - # print(index, row) - - # obs = Observation() +# +# def migrate_water_levels(session, limit=800): +# wd = pd.read_csv("./migration/data/water_levels.csv") +# p = pd.read_csv("./migration/data/welldata.csv") +# # get first 100 rows +# pointids = p["PointID"].unique()[:limit] +# +# wd = wd[wd["PointID"].isin(pointids)] +# +# gwd = wd.groupby(["PointID"]) +# +# sensor = Sensor() +# sensor.name = '"manual gwl measurement. needs to be replaced with measurementmethod(?) e.g. steel tape, eprobe, etc."' +# sensor.description = "Groundwater level manual measurement" +# session.add(sensor) +# session.commit() +# +# for index, group in gwd: +# +# # add a series +# # add a groundwater level series +# thing = session.query(Thing).filter_by(name=index[0]).first() +# print("Processing PointID:", index, thing) +# if not thing: +# continue +# +# print("found thing:", index, thing.id) +# series = Series(name="Groundwater Level Series") +# series.observed_property = "groundwater level" +# series.unit = "ft" +# +# series.sensor = sensor +# series.thing = thing +# +# groundwater_level_series = GroundwaterLevelSeries() +# groundwater_level_series.series = series +# +# session.add(series) +# session.add(groundwater_level_series) +# +# for row in group.itertuples(): +# obs = Observation() +# obs.series = series +# obs.observation_datetime = datetime.fromisoformat(row.DateMeasured) +# # print("rw", row.DateMeasured, row.TimeMeasured) +# gwl_obs = GroundwaterLevelObservation() +# gwl_obs.observation = obs +# gwl_obs.depth_to_water = row.DepthToWater +# gwl_obs.measuring_point_height = row.MPHeight +# session.add(obs) +# session.add(gwl_obs) +# +# session.commit() +# # break +# +# # print(group) +# # print('--------------------------------------------') +# # break +# # for index, row in group: +# # print(index, row) +# # print(row.PointID, row.TimeMeasured) +# # print(row.PointID, row.WaterLevel, row.WaterLevelDate) +# # if pd.isna(row.WaterLevel) or pd.isna(row.WaterLevelDate): +# # continue +# # +# # obs = add_groundwater_level_observation( +# # session, +# # { +# # "point_id": row.PointID, +# # "water_level": row.WaterLevel, +# # "water_level_date": row.WaterLevelDate, +# # }, +# # ) +# # print(obs) +# +# # print(index, row) +# +# # obs = Observation() ADDED = [] def migrate_wells(session, limit=1000): - wdf = pd.read_csv("./migration/data/welldata.csv") - ldf = pd.read_csv("./migration/data/location.csv") + wdf = pd.read_csv("./data/welldata.csv") + # wdf = pd.read_csv("./migration/data/welldata.csv") + ldf = pd.read_csv("./data/location.csv") wdf = wdf.replace(pd.NA, None) wdf = wdf.replace({np.nan: None}) @@ -177,15 +179,17 @@ def migrate_wells(session, limit=1000): location = make_location(row) session.add(location) - well = add_well( + well = add_thing( session, { "name": row.PointID, "hole_depth": row.HoleDepth, "well_depth": row.WellDepth, - "casing_diameter": row.CasingDiameter, - "casing_depth": row.CasingDepth, - "casing_description": row.CasingDescription, + "well_casing_diameter": row.CasingDiameter, + "well_casing_depth": row.CasingDepth, + "well_casing_description": row.CasingDescription, + "thing_type": "water well", + "release_status": "public" if row.PublicRelease else "private", }, ) wt = row.Meaning @@ -200,7 +204,7 @@ def migrate_wells(session, limit=1000): assoc = LocationThingAssociation() assoc.location = location - assoc.thing = well.thing + assoc.thing = well session.add(assoc) # break @@ -219,6 +223,6 @@ def migrate_wells(session, limit=1000): # reset_db() with session_ctx() as sess: migrate_wells(sess) - migrate_water_levels(sess) + # migrate_water_levels(sess) # ============= EOF ============================================= diff --git a/schemas_v2/lexicon.py b/schemas_v2/lexicon.py index 6ae8561ea..7aaa3b4b1 100644 --- a/schemas_v2/lexicon.py +++ b/schemas_v2/lexicon.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== from pydantic import BaseModel +from typing import List, Optional from schemas_v2 import ORMBaseModel @@ -52,16 +53,6 @@ class CreateTriple(BaseModel): # -------- RESPONSE ---------- -class LexiconTermResponse(ORMBaseModel): - """ - Pydantic model for the response of a lexicon term. - This model can be extended to include additional fields as needed. - """ - - term: str - definition: str - category: str | None = None - class LexiconCategoryResponse(ORMBaseModel): """ @@ -74,6 +65,23 @@ class LexiconCategoryResponse(ORMBaseModel): description: str | None = None # terms: list[LexiconTermResponse] | None = None +class LexiconTermCategoryResponse(ORMBaseModel): + """ + Pydantic model for the response of a lexicon term category association. + This model can be extended to include additional fields as needed. + """ + + category: LexiconCategoryResponse + +class LexiconTermResponse(ORMBaseModel): + """ + Pydantic model for the response of a lexicon term. + This model can be extended to include additional fields as needed. + """ + + term: str + definition: str + categories: List[LexiconTermCategoryResponse] | None = None # -------- UPDATE ---------- # ============= EOF ============================================= From 82f1464714b6ee9f719bc2ce9c7b090f250d5faa Mon Sep 17 00:00:00 2001 From: jirhiker Date: Fri, 8 Aug 2025 20:37:29 +0000 Subject: [PATCH 11/16] Formatting changes --- api/lexicon.py | 20 ++++++++++---------- db/lexicon.py | 2 -- migration/migration2.py | 4 +++- schemas_v2/lexicon.py | 4 ++++ 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/api/lexicon.py b/api/lexicon.py index 4cb5377e2..aa128957f 100644 --- a/api/lexicon.py +++ b/api/lexicon.py @@ -106,9 +106,9 @@ def get_lexicon_terms( session: session_dependency, category: str | None = None, term: str | None = None, - sort: str = None, - order: str = None, - filter_: str = Query(alias="filter", default=None), + sort: str = None, + order: str = None, + filter_: str = Query(alias="filter", default=None), ) -> CustomPage[LexiconTermResponse]: """ Endpoint to retrieve lexicon terms. @@ -123,22 +123,22 @@ def get_lexicon_terms( if term: sql = sql.where(Lexicon.term.ilike(f"%{term}%")) - sql = order_sort_filter( - sql, Lexicon, sort=sort, order=order, filter_=filter_ - ) + sql = order_sort_filter(sql, Lexicon, sort=sort, order=order, filter_=filter_) return paginate(query=sql, conn=session) # return paginated_all_getter(session, sql, filter_) + @router.get("/category") def get_lexicon_categories( - session: session_dependency, - sort: str = None, - order: str = None, - filter_: str = Query(alias="filter", default=None), + session: session_dependency, + sort: str = None, + order: str = None, + filter_: str = Query(alias="filter", default=None), ) -> CustomPage[LexiconCategoryResponse]: """ Endpoint to retrieve lexicon categories. """ return paginated_all_getter(session, Category, sort, order, filter_) + # ============= EOF ============================================= diff --git a/db/lexicon.py b/db/lexicon.py index fe4fb20cd..869bd03a5 100644 --- a/db/lexicon.py +++ b/db/lexicon.py @@ -29,7 +29,6 @@ class Lexicon(Base, AutoBaseMixin): term = mapped_column(String(100), unique=True, nullable=False) definition = mapped_column(String(255), nullable=False) - # categories = relationship( # "Category", # secondary="lexicon_term_category_association", @@ -49,7 +48,6 @@ class Category(Base, AutoBaseMixin): name = mapped_column(String(100), unique=True, nullable=False) description = mapped_column(String(255), nullable=True) - def __repr__(self): return f"" diff --git a/migration/migration2.py b/migration/migration2.py index d87eee65d..da486480a 100644 --- a/migration/migration2.py +++ b/migration/migration2.py @@ -28,6 +28,7 @@ # from db.observation.groundwaterlevel import GroundwaterLevelObservation from db.observation import Observation + # from db.series.groundwaterlevel import GroundwaterLevelSeries # from db.series.series import Series from services.lexicon import add_lexicon_term @@ -63,10 +64,11 @@ def make_location(row): return Location( name=row.PointID, point=transformed_point.wkt, - release_status='public' if row.PublicRelease else 'private', + release_status="public" if row.PublicRelease else "private", # visible=row_dict["PublicRelease"], ) + # # def migrate_water_levels(session, limit=800): # wd = pd.read_csv("./migration/data/water_levels.csv") diff --git a/schemas_v2/lexicon.py b/schemas_v2/lexicon.py index 7aaa3b4b1..b72079582 100644 --- a/schemas_v2/lexicon.py +++ b/schemas_v2/lexicon.py @@ -54,6 +54,7 @@ class CreateTriple(BaseModel): # -------- RESPONSE ---------- + class LexiconCategoryResponse(ORMBaseModel): """ Pydantic model for the response of a lexicon category. @@ -65,6 +66,7 @@ class LexiconCategoryResponse(ORMBaseModel): description: str | None = None # terms: list[LexiconTermResponse] | None = None + class LexiconTermCategoryResponse(ORMBaseModel): """ Pydantic model for the response of a lexicon term category association. @@ -73,6 +75,7 @@ class LexiconTermCategoryResponse(ORMBaseModel): category: LexiconCategoryResponse + class LexiconTermResponse(ORMBaseModel): """ Pydantic model for the response of a lexicon term. @@ -83,5 +86,6 @@ class LexiconTermResponse(ORMBaseModel): definition: str categories: List[LexiconTermCategoryResponse] | None = None + # -------- UPDATE ---------- # ============= EOF ============================================= From 39f76ab3d86ffae075b384f0aedcf202ad6cdc85 Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 8 Aug 2025 14:38:28 -0600 Subject: [PATCH 12/16] refactor: deleted schemas folder --- schemas/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 schemas/__init__.py diff --git a/schemas/__init__.py b/schemas/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 3d312c57e601148c8364798c7f7ce8d31e5e5777 Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 8 Aug 2025 14:39:13 -0600 Subject: [PATCH 13/16] refactor: renamed schemas_v2 to schemas --- api/asset.py | 2 +- api/author.py | 2 +- api/contact.py | 2 +- api/geochronology.py | 2 +- api/geospatial.py | 2 +- api/group.py | 6 +++--- api/lexicon.py | 2 +- api/location.py | 4 ++-- api/observation.py | 2 +- api/publication.py | 2 +- api/sample.py | 4 ++-- api/sensor.py | 2 +- api/thing.py | 4 ++-- {schemas_v2 => schemas}/__init__.py | 0 {schemas_v2 => schemas}/asset.py | 0 {schemas_v2 => schemas}/contact.py | 4 ++-- {schemas_v2 => schemas}/geochronology.py | 0 {schemas_v2 => schemas}/geothermal.py | 0 {schemas_v2 => schemas}/group.py | 2 +- {schemas_v2 => schemas}/lexicon.py | 2 +- {schemas_v2 => schemas}/location.py | 2 +- {schemas_v2 => schemas}/observation.py | 0 {schemas_v2 => schemas}/publication.py | 0 {schemas_v2 => schemas}/sample.py | 0 {schemas_v2 => schemas}/sensor.py | 0 {schemas_v2 => schemas}/series.py | 0 {schemas_v2 => schemas}/thing.py | 4 ++-- services/people_helper.py | 2 +- services/publication_helper.py | 2 +- services/thing_helper.py | 2 +- services/validation/well.py | 2 +- tests/test_sample.py | 2 +- 32 files changed, 30 insertions(+), 30 deletions(-) rename {schemas_v2 => schemas}/__init__.py (100%) rename {schemas_v2 => schemas}/asset.py (100%) rename {schemas_v2 => schemas}/contact.py (98%) rename {schemas_v2 => schemas}/geochronology.py (100%) rename {schemas_v2 => schemas}/geothermal.py (100%) rename {schemas_v2 => schemas}/group.py (97%) rename {schemas_v2 => schemas}/lexicon.py (98%) rename {schemas_v2 => schemas}/location.py (98%) rename {schemas_v2 => schemas}/observation.py (100%) rename {schemas_v2 => schemas}/publication.py (100%) rename {schemas_v2 => schemas}/sample.py (100%) rename {schemas_v2 => schemas}/sensor.py (100%) rename {schemas_v2 => schemas}/series.py (100%) rename {schemas_v2 => schemas}/thing.py (98%) diff --git a/api/asset.py b/api/asset.py index 8f92af0e6..63d31b7fd 100644 --- a/api/asset.py +++ b/api/asset.py @@ -27,7 +27,7 @@ from core.dependencies import session_dependency from db import Thing from db.asset import Asset, AssetThingAssociation -from schemas_v2.asset import AssetResponse, CreateAsset, UpdateAsset +from schemas.asset import AssetResponse, CreateAsset, UpdateAsset from services.crud_helper import model_patcher router = APIRouter(prefix="/asset", tags=["asset"]) diff --git a/api/author.py b/api/author.py index 5704bac81..1761dec6e 100644 --- a/api/author.py +++ b/api/author.py @@ -19,7 +19,7 @@ from db.engine import get_db_session from db.publication import Author -from schemas_v2.publication import PublicationResponse +from schemas.publication import PublicationResponse router = APIRouter( prefix="/author", diff --git a/api/contact.py b/api/contact.py index 33bda0a16..38ddc254e 100644 --- a/api/contact.py +++ b/api/contact.py @@ -28,7 +28,7 @@ from core.dependencies import session_dependency from db import ThingContactAssociation, Thing from db.contact import Contact, Email, Phone, Address -from schemas_v2.contact import ( +from schemas.contact import ( CreateContact, PhoneResponse, EmailResponse, diff --git a/api/geochronology.py b/api/geochronology.py index 1a1a5b5ac..d32f57f34 100644 --- a/api/geochronology.py +++ b/api/geochronology.py @@ -17,7 +17,7 @@ from fastapi import APIRouter, Depends, status from db import adder from db.engine import get_db_session -from schemas_v2.geochronology import CreateGeochronologyAge +from schemas.geochronology import CreateGeochronologyAge from sqlalchemy.orm import Session from sqlalchemy import select diff --git a/api/geospatial.py b/api/geospatial.py index 12c85df69..48b1fb887 100644 --- a/api/geospatial.py +++ b/api/geospatial.py @@ -22,7 +22,7 @@ # from starlette.responses import FileResponse from core.dependencies import session_dependency -from schemas_v2.thing import FeatureCollectionResponse +from schemas.thing import FeatureCollectionResponse from services.geospatial_helper import create_shapefile, get_thing_features router = APIRouter(prefix="/geospatial", tags=["geospatial"]) diff --git a/api/group.py b/api/group.py index 02db32e41..12d056d6a 100644 --- a/api/group.py +++ b/api/group.py @@ -23,9 +23,9 @@ from db import adder from db.engine import get_db_session from db.group import Group, GroupThingAssociation -from schemas_v2.group import UpdateGroup -from schemas_v2.location import CreateGroup, CreateGroupThing -from schemas_v2.thing import GroupResponse +from schemas.group import UpdateGroup +from schemas.location import CreateGroup, CreateGroupThing +from schemas.thing import GroupResponse from services.crud_helper import model_patcher from services.query_helper import ( simple_get_by_id, diff --git a/api/lexicon.py b/api/lexicon.py index aa128957f..ab0acf1ae 100644 --- a/api/lexicon.py +++ b/api/lexicon.py @@ -21,7 +21,7 @@ from core.dependencies import session_dependency from db.engine import get_db_session from db.lexicon import Category, LexiconTriple, Lexicon, TermCategoryAssociation -from schemas_v2.lexicon import ( +from schemas.lexicon import ( CreateLexiconTerm, CreateLexiconCategory, CreateTriple, diff --git a/api/location.py b/api/location.py index 384aa577c..1d63b23c5 100644 --- a/api/location.py +++ b/api/location.py @@ -26,8 +26,8 @@ from db import adder from db.location import Location from db.engine import get_db_session -from schemas_v2.location import CreateLocation, LocationResponse, UpdateLocation -from schemas_v2.thing import LocationWellResponse +from schemas.location import CreateLocation, LocationResponse, UpdateLocation +from schemas.thing import LocationWellResponse from services.geospatial_helper import make_within_wkt from services.query_helper import make_query, order_sort_filter from services.crud_helper import model_patcher diff --git a/api/observation.py b/api/observation.py index 3023ae66b..326226f38 100644 --- a/api/observation.py +++ b/api/observation.py @@ -24,7 +24,7 @@ from core.dependencies import session_dependency from db import Sample from db.observation import Observation -from schemas_v2.observation import ( +from schemas.observation import ( CreateGroundwaterLevelObservation, GroundwaterLevelObservationResponse, CreateWaterChemistryObservation, diff --git a/api/publication.py b/api/publication.py index 3da758ec5..53ffe69d6 100644 --- a/api/publication.py +++ b/api/publication.py @@ -15,7 +15,7 @@ # =============================================================================== from db.engine import get_db_session from fastapi import APIRouter, Depends, status -from schemas_v2.publication import PublicationResponse, CreatePublication +from schemas.publication import PublicationResponse, CreatePublication from services.publication_helper import add_publication from sqlalchemy.orm import Session diff --git a/api/sample.py b/api/sample.py index 9f6c486a1..4ce23af88 100644 --- a/api/sample.py +++ b/api/sample.py @@ -24,8 +24,8 @@ from db import adder from db.engine import get_db_session from db.sample import Sample -from schemas_v2 import ResourceNotFoundResponse -from schemas_v2.sample import SampleResponse, CreateSample, UpdateSample +from schemas import ResourceNotFoundResponse +from schemas.sample import SampleResponse, CreateSample, UpdateSample from services.query_helper import paginated_all_getter, simple_get_by_id from services.crud_helper import model_patcher diff --git a/api/sensor.py b/api/sensor.py index 2425d6971..a7f6b6ccb 100644 --- a/api/sensor.py +++ b/api/sensor.py @@ -25,7 +25,7 @@ from db import adder, Observation from db.engine import get_db_session from db.sensor import Sensor -from schemas_v2.sensor import SensorResponse, CreateSensor +from schemas.sensor import SensorResponse, CreateSensor from services.query_helper import order_sort_filter router = APIRouter(prefix="/sensor", tags=["sensor"]) diff --git a/api/thing.py b/api/thing.py index bfcab09b3..a335d4dc4 100644 --- a/api/thing.py +++ b/api/thing.py @@ -31,8 +31,8 @@ from db.location import LocationThingAssociation, Location from db.thing import Thing, WellScreen from db.thing import ThingIdLink -from schemas_v2.location import LocationResponse, UpdateLocation -from schemas_v2.thing import ( +from schemas.location import LocationResponse, UpdateLocation +from schemas.thing import ( CreateThingIdLink, CreateWell, CreateWellScreen, diff --git a/schemas_v2/__init__.py b/schemas/__init__.py similarity index 100% rename from schemas_v2/__init__.py rename to schemas/__init__.py diff --git a/schemas_v2/asset.py b/schemas/asset.py similarity index 100% rename from schemas_v2/asset.py rename to schemas/asset.py diff --git a/schemas_v2/contact.py b/schemas/contact.py similarity index 98% rename from schemas_v2/contact.py rename to schemas/contact.py index 645bb8ba1..522abf153 100644 --- a/schemas_v2/contact.py +++ b/schemas/contact.py @@ -20,8 +20,8 @@ from phonenumbers import NumberParseException from pydantic import field_validator, BaseModel, AwareDatetime -from schemas_v2 import ORMBaseModel -from schemas_v2.thing import ThingResponse +from schemas import ORMBaseModel +from schemas.thing import ThingResponse """ REFACTOR TODO diff --git a/schemas_v2/geochronology.py b/schemas/geochronology.py similarity index 100% rename from schemas_v2/geochronology.py rename to schemas/geochronology.py diff --git a/schemas_v2/geothermal.py b/schemas/geothermal.py similarity index 100% rename from schemas_v2/geothermal.py rename to schemas/geothermal.py diff --git a/schemas_v2/group.py b/schemas/group.py similarity index 97% rename from schemas_v2/group.py rename to schemas/group.py index e0b689f83..e4cb9c4d2 100644 --- a/schemas_v2/group.py +++ b/schemas/group.py @@ -15,7 +15,7 @@ # =============================================================================== from pydantic import BaseModel -from schemas_v2 import ORMBaseModel +from schemas import ORMBaseModel # -------- CREATE ---------- diff --git a/schemas_v2/lexicon.py b/schemas/lexicon.py similarity index 98% rename from schemas_v2/lexicon.py rename to schemas/lexicon.py index b72079582..f9ade729e 100644 --- a/schemas_v2/lexicon.py +++ b/schemas/lexicon.py @@ -16,7 +16,7 @@ from pydantic import BaseModel from typing import List, Optional -from schemas_v2 import ORMBaseModel +from schemas import ORMBaseModel # -------- CREATE ---------- diff --git a/schemas_v2/location.py b/schemas/location.py similarity index 98% rename from schemas_v2/location.py rename to schemas/location.py index 82df72dc7..af48e8f9d 100644 --- a/schemas_v2/location.py +++ b/schemas/location.py @@ -18,7 +18,7 @@ from pydantic import BaseModel, field_validator from shapely import wkt -from schemas_v2 import ORMBaseModel +from schemas import ORMBaseModel """ REFACTOR TODO diff --git a/schemas_v2/observation.py b/schemas/observation.py similarity index 100% rename from schemas_v2/observation.py rename to schemas/observation.py diff --git a/schemas_v2/publication.py b/schemas/publication.py similarity index 100% rename from schemas_v2/publication.py rename to schemas/publication.py diff --git a/schemas_v2/sample.py b/schemas/sample.py similarity index 100% rename from schemas_v2/sample.py rename to schemas/sample.py diff --git a/schemas_v2/sensor.py b/schemas/sensor.py similarity index 100% rename from schemas_v2/sensor.py rename to schemas/sensor.py diff --git a/schemas_v2/series.py b/schemas/series.py similarity index 100% rename from schemas_v2/series.py rename to schemas/series.py diff --git a/schemas_v2/thing.py b/schemas/thing.py similarity index 98% rename from schemas_v2/thing.py rename to schemas/thing.py index b326078f6..047c94f11 100644 --- a/schemas_v2/thing.py +++ b/schemas/thing.py @@ -18,8 +18,8 @@ from pydantic import BaseModel, model_validator -from schemas_v2 import ORMBaseModel -from schemas_v2.location import LocationResponse +from schemas import ORMBaseModel +from schemas.location import LocationResponse # -------- CREATE ---------- diff --git a/services/people_helper.py b/services/people_helper.py index a7e6b04b1..3f250ffaa 100644 --- a/services/people_helper.py +++ b/services/people_helper.py @@ -14,7 +14,7 @@ # limitations under the License. # =============================================================================== from db.contact import Contact, Email, Phone, Address, ThingContactAssociation -from schemas_v2.contact import CreateContact +from schemas.contact import CreateContact from sqlalchemy.orm import Session diff --git a/services/publication_helper.py b/services/publication_helper.py index ed042e2d1..9ae346acd 100644 --- a/services/publication_helper.py +++ b/services/publication_helper.py @@ -14,7 +14,7 @@ # limitations under the License. # =============================================================================== from db.publication import Author, Publication, AuthorPublicationAssociation -from schemas_v2.publication import CreatePublication +from schemas.publication import CreatePublication from sqlalchemy.orm import Session from sqlalchemy import select diff --git a/services/thing_helper.py b/services/thing_helper.py index 046dbab86..c768a68c5 100644 --- a/services/thing_helper.py +++ b/services/thing_helper.py @@ -19,7 +19,7 @@ from sqlalchemy.orm import Session from db import LocationThingAssociation, Thing, Base, Location -from schemas_v2.location import LocationResponse +from schemas.location import LocationResponse from db.group import Group, GroupThingAssociation from services.geospatial_helper import make_within_wkt from services.query_helper import make_query, order_sort_filter diff --git a/services/validation/well.py b/services/validation/well.py index d8e213dda..edd890a3e 100644 --- a/services/validation/well.py +++ b/services/validation/well.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from schemas_v2.thing import CreateWellScreen +from schemas.thing import CreateWellScreen from services.validation import get_category diff --git a/tests/test_sample.py b/tests/test_sample.py index 20e21aa38..2e351050a 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -17,7 +17,7 @@ from db.engine import session_ctx from db.sample import Sample -from schemas_v2.sample import ValidateSample +from schemas.sample import ValidateSample from tests import client # ============= module & function fixtures ======================================= From e1a3f54288e05659bc2b499fc3dc50b127454157 Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 8 Aug 2025 15:19:44 -0600 Subject: [PATCH 14/16] refactor: display sorting via "categories" --- api/lexicon.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/lexicon.py b/api/lexicon.py index ab0acf1ae..058abb084 100644 --- a/api/lexicon.py +++ b/api/lexicon.py @@ -29,7 +29,7 @@ LexiconCategoryResponse, ) from services.lexicon import add_lexicon_term -from services.query_helper import simple_all_getter, paginated_all_getter +from services.query_helper import simple_all_getter, paginated_all_getter, order_sort_filter router = APIRouter( prefix="/lexicon", @@ -123,6 +123,11 @@ def get_lexicon_terms( if term: sql = sql.where(Lexicon.term.ilike(f"%{term}%")) + # If sort is 'categories', we do not apply sorting or filtering + if sort == 'categories': + sort = None + order = None + sql = order_sort_filter(sql, Lexicon, sort=sort, order=order, filter_=filter_) return paginate(query=sql, conn=session) # return paginated_all_getter(session, sql, filter_) From efccb86d57c7e4cc395b490798afed47a95e168f Mon Sep 17 00:00:00 2001 From: jirhiker Date: Fri, 8 Aug 2025 21:20:03 +0000 Subject: [PATCH 15/16] Formatting changes --- api/lexicon.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api/lexicon.py b/api/lexicon.py index 058abb084..dea5b0226 100644 --- a/api/lexicon.py +++ b/api/lexicon.py @@ -29,7 +29,11 @@ LexiconCategoryResponse, ) from services.lexicon import add_lexicon_term -from services.query_helper import simple_all_getter, paginated_all_getter, order_sort_filter +from services.query_helper import ( + simple_all_getter, + paginated_all_getter, + order_sort_filter, +) router = APIRouter( prefix="/lexicon", @@ -124,7 +128,7 @@ def get_lexicon_terms( sql = sql.where(Lexicon.term.ilike(f"%{term}%")) # If sort is 'categories', we do not apply sorting or filtering - if sort == 'categories': + if sort == "categories": sort = None order = None From efaf64987d98e4a9ba40b718abdc12a69576e921 Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 8 Aug 2025 16:00:20 -0600 Subject: [PATCH 16/16] ai: added lexicon test coverage --- tests/test_lexicon_pagination.py | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/test_lexicon_pagination.py diff --git a/tests/test_lexicon_pagination.py b/tests/test_lexicon_pagination.py new file mode 100644 index 000000000..7838d0f7a --- /dev/null +++ b/tests/test_lexicon_pagination.py @@ -0,0 +1,39 @@ +# ============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from tests import client + + +def test_get_lexicon_terms_sort_categories_branch(): + """ + Ensure the special-case branch (sort == 'categories') in GET /lexicon is exercised. + It should not apply sorting/filtering and still return a valid pagination payload. + """ + resp = client.get("/lexicon", params={"sort": "categories"}) + assert resp.status_code == 200 + data = resp.json() + # fastapi-pagination returns a Page-like object with these keys + assert "items" in data + assert "total" in data + + +def test_get_lexicon_categories_endpoint(): + """Basic smoke test that categories endpoint returns a paginated payload.""" + resp = client.get("/lexicon/category") + assert resp.status_code == 200 + data = resp.json() + assert "items" in data + # Should have at least one category from init_lexicon and/or previous tests + assert isinstance(data["items"], list)