From da6f7885fa01e3dcf0c5c981fbe6e028dc746b88 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 7 Aug 2025 14:50:56 -0600 Subject: [PATCH 1/6] fix: ensure sample_date is UTC in storage and response for samples Data must be timezone aware. If it is not UTC it gets converted to UTC. --- api/sample.py | 4 +++- db/sample.py | 4 +++- schemas_v2/sample.py | 37 ++++++++++++++++++++++--------------- tests/conftest.py | 2 +- tests/test_sample.py | 12 ++---------- 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/api/sample.py b/api/sample.py index 8b69e71d5..9f6c486a1 100644 --- a/api/sample.py +++ b/api/sample.py @@ -60,7 +60,9 @@ def database_error_handler( # ============= Post ============================================= @router.post("", status_code=HTTP_201_CREATED) -def add_sample(sample_data: CreateSample, session: session_dependency): +def add_sample( + sample_data: CreateSample, session: session_dependency +) -> SampleResponse: """ Endpoint to add a sample. """ diff --git a/db/sample.py b/db/sample.py index 6a0098f33..1e6c749b8 100644 --- a/db/sample.py +++ b/db/sample.py @@ -48,7 +48,9 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin): # Sample Attributes sample_date: Mapped[datetime.datetime] = mapped_column( - DateTime, nullable=False, comment="Date and time of sample collection." + DateTime(timezone=True), + nullable=False, + comment="Date and time of sample collection.", ) # REFACTOR TODO: update with enum/restricted values sample_matrix: Mapped[Optional[str]] = mapped_column( diff --git a/schemas_v2/sample.py b/schemas_v2/sample.py index e520ade7d..35df8872d 100644 --- a/schemas_v2/sample.py +++ b/schemas_v2/sample.py @@ -13,8 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from datetime import datetime, timezone -from pydantic import BaseModel, field_validator, model_validator +from datetime import timezone +from pydantic import ( + BaseModel, + field_validator, + model_validator, + AwareDatetime, + PastDatetime, +) +from typing import Annotated from typing_extensions import Self @@ -30,16 +37,6 @@ class ValidateSample(BaseModel): Validator for Sample data for Create and Update schemas. """ - @field_validator("sample_date", check_fields=False) - def validate_sample_date(cls, sample_date: datetime | None) -> datetime | None: - """ - Validate that the sample_date is not in the future. - """ - if sample_date is not None: - if sample_date > datetime.now(tz=timezone.utc): - raise ValueError(f"Sample date {sample_date} cannot be in the future.") - return sample_date - # # REFACTOR TODO: is below ground negative or positive? the combine this with validate_sample_bottom defined below # @field_validator("sample_bottom", check_fields=False) # def validate_sample_bottom(cls, sample_bottom: float | None, values) -> float | None: @@ -73,13 +70,23 @@ def validate_top_and_bottom(self) -> Self: ) return self + @field_validator("sample_date", check_fields=False) + def convert_sample_date_to_utc(sample_date: AwareDatetime) -> AwareDatetime: + """ + Convert sample_date to UTC timezone if it's not already. This runs after + the Annotated validator PastDatetime() is run. + """ + if sample_date is not None and sample_date.tzinfo != timezone.utc: + return sample_date.astimezone(timezone.utc) + return sample_date + # -------- CREATE ---------- class CreateSample(ValidateSample): thing_id: int sample_type: str field_sample_id: str - sample_date: datetime + sample_date: Annotated[AwareDatetime, PastDatetime()] release_status: str sampler_name: str # REFACTOR TODO: update with enum/restricted values qc_sample: str = "Original" @@ -111,7 +118,7 @@ class UpdateSample(ValidateSample): thing_id: int = None # REFACTOR TODO: should users be able to change this? sample_type: str = None field_sample_id: str = None - sample_date: datetime = None + sample_date: Annotated[AwareDatetime, PastDatetime()] = None release_status: str = None sampler_name: str = None # REFACTOR TODO: update with enum/restricted values qc_sample: str = None @@ -138,7 +145,7 @@ class SampleResponse(BaseModel): thing_id: int sample_type: str field_sample_id: str - sample_date: datetime + sample_date: AwareDatetime release_status: str sampler_name: str qc_sample: str diff --git a/tests/conftest.py b/tests/conftest.py index 7957ee379..62b1aebed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,7 +49,7 @@ def sensor(): def sample(thing, sensor): with session_ctx() as session: sample = Sample( - sample_date="2025-01-01T00:00:00", + sample_date="2025-01-01T00:00:00Z", thing_id=thing.id, sample_type="groundwater", sampler_name="Test Sampler", diff --git a/tests/test_sample.py b/tests/test_sample.py index 2d30e92d7..20e21aa38 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -51,14 +51,6 @@ def second_sample(thing, sensor): # ============== Custom validators ================================================= -def test_validate_sample_date(): - invalid_sample_date = "3500-01-01T00:00:00Z" - try: - invalid_sample = ValidateSample(sample_date=invalid_sample_date) - except ValueError as e: - assert str(e) == f"Sample date {invalid_sample_date} is not valid." - - def test_validate_sample_top_and_bottom(): for i in range(2): sample_top = 10.0 if i == 0 else None @@ -103,7 +95,7 @@ def test_add_sample(thing, sensor): assert data["thing_id"] == payload["thing_id"] assert data["sample_type"] == payload["sample_type"] assert data["field_sample_id"] == payload["field_sample_id"] - assert data["sample_date"] == payload["sample_date"][:-1] + assert data["sample_date"] == payload["sample_date"] assert data["release_status"] == payload["release_status"] assert data["sampler_name"] == payload["sampler_name"] assert data["qc_sample"] == payload["qc_sample"] @@ -200,7 +192,7 @@ def test_patch_sample(sample): assert data["id"] == sample.id assert data["sampler_name"] == new_sampler_name - assert data["sample_date"] == new_sample_date[:-1] + assert data["sample_date"] == new_sample_date assert data["sample_method"] == new_sample_method # rollback after updating the sample From e7460f36881220dad1d7c2fbd6c51fdc321d3aef Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 7 Aug 2025 15:14:23 -0600 Subject: [PATCH 2/6] feat: make effective_start/end tz aware, default start to UTC when made --- db/location.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/db/location.py b/db/location.py index 486d09fc4..57aac7b70 100644 --- a/db/location.py +++ b/db/location.py @@ -19,7 +19,6 @@ Integer, String, ForeignKey, - Boolean, DateTime, func, Text, @@ -56,8 +55,13 @@ class LocationThingAssociation(Base, AutoBaseMixin): Integer, ForeignKey("thing.id", ondelete="CASCADE"), primary_key=True ) - effective_start = Column(DateTime, nullable=False, server_default=func.now()) - effective_end = Column(DateTime, nullable=True) + # REFACTOR TODO: when refactoring/updating location/thing schemas and tests, ensure timezone is UTC + effective_start = Column( + DateTime(timezone=True), + nullable=False, + server_default=func.timezone("UTC", func.now()), + ) + effective_end = Column(DateTime(timezone=True), nullable=True) location = relationship("Location") thing = relationship("Thing") From a3904cc9d3e1012d50c7427d13937264381b668e Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 7 Aug 2025 15:18:15 -0600 Subject: [PATCH 3/6] fix: ensure AuditMixin created_at is tz aware and defaults to UTC --- db/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/db/base.py b/db/base.py index 7fff27cba..32c2c561f 100644 --- a/db/base.py +++ b/db/base.py @@ -57,7 +57,11 @@ def release_status(self): class AuditMixin: @declared_attr def created_at(self): - return Column(DateTime, nullable=False, server_default=func.now()) + return Column( + DateTime(timezone=True), + nullable=False, + server_default=func.timezone("UTC", func.now()), + ) class AutoBaseMixin(AuditMixin): From 6c9c0f565c421209e350329897613e67de418cae Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 7 Aug 2025 15:21:38 -0600 Subject: [PATCH 4/6] fix: make created_at responses tz aware --- schemas/__init__.py | 6 ++---- schemas_v2/asset.py | 6 ++---- schemas_v2/contact.py | 5 ++--- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/schemas/__init__.py b/schemas/__init__.py index fc2bd4889..153a92dd5 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -13,14 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from datetime import datetime - -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, AwareDatetime class ORMBaseModel(BaseModel): id: int # every ORM model should have an id field - created_at: datetime + created_at: AwareDatetime model_config = ConfigDict( from_attributes=True, populate_by_name=True, diff --git a/schemas_v2/asset.py b/schemas_v2/asset.py index 1ee7be652..e31c2190c 100644 --- a/schemas_v2/asset.py +++ b/schemas_v2/asset.py @@ -13,9 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from datetime import datetime - -from pydantic import BaseModel +from pydantic import BaseModel, AwareDatetime class BaseAsset(BaseModel): @@ -42,7 +40,7 @@ class AssetResponse(BaseAsset): # storage_path: str # mime_type: str # size: int - created_at: datetime + created_at: AwareDatetime storage_service: str diff --git a/schemas_v2/contact.py b/schemas_v2/contact.py index 95eb4ad80..bd6b3578f 100644 --- a/schemas_v2/contact.py +++ b/schemas_v2/contact.py @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from datetime import datetime from typing import Optional, List import phonenumbers from email_validator import validate_email, EmailNotValidError from phonenumbers import NumberParseException -from pydantic import field_validator, BaseModel +from pydantic import field_validator, BaseModel, AwareDatetime from schemas import ORMBaseModel from schemas_v2.thing import ThingResponse @@ -193,7 +192,7 @@ class ContactResponse(ORMBaseModel): id: int name: str role: str - created_at: datetime + created_at: AwareDatetime emails: List[EmailResponse] = [] phones: List[PhoneResponse] = [] addresses: List[AddressResponse] = [] From cef3eaa943e18825db58a7c3096a2455e0be1698 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 7 Aug 2025 15:23:23 -0600 Subject: [PATCH 5/6] fix: make observation db model and schema dt fields tz aware And default to UTC --- db/observation.py | 6 ++---- schemas_v2/observation.py | 11 +++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/db/observation.py b/db/observation.py index dd0953378..63f9468e3 100644 --- a/db/observation.py +++ b/db/observation.py @@ -16,13 +16,11 @@ from sqlalchemy import ( ForeignKey, Integer, - DateTime, TIMESTAMP, PrimaryKeyConstraint, - ForeignKeyConstraint, Float, ) -from sqlalchemy.orm import declared_attr, mapped_column, relationship +from sqlalchemy.orm import mapped_column, relationship from db.base import Base, AuditMixin, ReleaseMixin, lexicon_term @@ -57,7 +55,7 @@ class Observation(Base, AuditMixin, ReleaseMixin): ) observation_timestamp = mapped_column( - TIMESTAMP, nullable=False, doc="Timestamp of the observation" + TIMESTAMP(timezone=True), nullable=False, doc="Timestamp of the observation" ) observed_property = lexicon_term() diff --git a/schemas_v2/observation.py b/schemas_v2/observation.py index 043909b1f..46c31d4b6 100644 --- a/schemas_v2/observation.py +++ b/schemas_v2/observation.py @@ -13,9 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from datetime import datetime - -from pydantic import BaseModel +from pydantic import BaseModel, AwareDatetime, PastDatetime +from typing import Annotated # class GeothermalMixin: @@ -25,7 +24,7 @@ # -------- CREATE ---------- class CreateBaseObservation(BaseModel): - observation_timestamp: datetime + observation_timestamp: Annotated[AwareDatetime, PastDatetime()] sample_id: int sensor_id: int observed_property: str @@ -61,9 +60,9 @@ class BaseObservationResponse(BaseModel): id: int sample_id: int sensor_id: int - observation_timestamp: datetime + observation_timestamp: AwareDatetime observed_property: str - created_at: datetime + created_at: AwareDatetime release_status: str From a2104bebc565cb45eab295b3929644966959d7c9 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 7 Aug 2025 15:30:42 -0600 Subject: [PATCH 6/6] feat: ensure observation_timestamp is UTC --- schemas_v2/observation.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/schemas_v2/observation.py b/schemas_v2/observation.py index 46c31d4b6..35f05e8d7 100644 --- a/schemas_v2/observation.py +++ b/schemas_v2/observation.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from pydantic import BaseModel, AwareDatetime, PastDatetime +from datetime import timezone +from pydantic import BaseModel, AwareDatetime, PastDatetime, field_validator from typing import Annotated @@ -22,8 +23,29 @@ # temperature: float +# -------- VALIDATE ------- + + +class ValidateObservation(BaseModel): + + @field_validator("observation_timestamp", check_fields=False) + def convert_observation_timestamp_to_utc( + observation_timestamp: AwareDatetime, + ) -> AwareDatetime: + """ + Convert observation_timestamp to UTC timezone if it's not already. This runs after + the Annotated validator PastDatetime() is run. + """ + if ( + observation_timestamp is not None + and observation_timestamp.tzinfo != timezone.utc + ): + return observation_timestamp.astimezone(timezone.utc) + return observation_timestamp + + # -------- CREATE ---------- -class CreateBaseObservation(BaseModel): +class CreateBaseObservation(ValidateObservation): observation_timestamp: Annotated[AwareDatetime, PastDatetime()] sample_id: int sensor_id: int