NO TICKET: update all datetime fields to be timezone aware in the database and schemas

api/sample.py

@@ -60,7 +60,9 @@ def database_error_handler(
 
 # ============= Post =============================================
 @router.post("", status_code=HTTP_201_CREATED)
-def add_sample(sample_data: CreateSample, session: session_dependency):
+def add_sample(
+    sample_data: CreateSample, session: session_dependency
+) -> SampleResponse:
     """
     Endpoint to add a sample.
     """

db/base.py

@@ -57,7 +57,11 @@ def release_status(self):
 class AuditMixin:
     @declared_attr
     def created_at(self):
-        return Column(DateTime, nullable=False, server_default=func.now())
+        return Column(
+            DateTime(timezone=True),
+            nullable=False,
+            server_default=func.timezone("UTC", func.now()),
+        )
 
 
 class AutoBaseMixin(AuditMixin):

db/location.py

@@ -19,7 +19,6 @@
     Integer,
     String,
     ForeignKey,
-    Boolean,
     DateTime,
     func,
     Text,
@@ -56,8 +55,13 @@ class LocationThingAssociation(Base, AutoBaseMixin):
         Integer, ForeignKey("thing.id", ondelete="CASCADE"), primary_key=True
     )
 
-    effective_start = Column(DateTime, nullable=False, server_default=func.now())
-    effective_end = Column(DateTime, nullable=True)
+    # REFACTOR TODO: when refactoring/updating location/thing schemas and tests, ensure timezone is UTC
+    effective_start = Column(
+        DateTime(timezone=True),
+        nullable=False,
+        server_default=func.timezone("UTC", func.now()),
+    )
+    effective_end = Column(DateTime(timezone=True), nullable=True)
 
     location = relationship("Location")
     thing = relationship("Thing")

db/observation.py

@@ -16,13 +16,11 @@
 from sqlalchemy import (
     ForeignKey,
     Integer,
-    DateTime,
     TIMESTAMP,
     PrimaryKeyConstraint,
-    ForeignKeyConstraint,
     Float,
 )
-from sqlalchemy.orm import declared_attr, mapped_column, relationship
+from sqlalchemy.orm import mapped_column, relationship
 
 from db.base import Base, AuditMixin, ReleaseMixin, lexicon_term
 
@@ -57,7 +55,7 @@ class Observation(Base, AuditMixin, ReleaseMixin):
     )
 
     observation_timestamp = mapped_column(
-        TIMESTAMP, nullable=False, doc="Timestamp of the observation"
+        TIMESTAMP(timezone=True), nullable=False, doc="Timestamp of the observation"
     )
     observed_property = lexicon_term()
 

db/sample.py

@@ -48,7 +48,9 @@ class Sample(Base, AutoBaseMixin, ReleaseMixin):
 
     # Sample Attributes
     sample_date: Mapped[datetime.datetime] = mapped_column(
-        DateTime, nullable=False, comment="Date and time of sample collection."
+        DateTime(timezone=True),
+        nullable=False,
+        comment="Date and time of sample collection.",
     )
     # REFACTOR TODO: update with enum/restricted values
     sample_matrix: Mapped[Optional[str]] = mapped_column(

schemas/__init__.py

@@ -13,14 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-from datetime import datetime
-
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, AwareDatetime
 
 
 class ORMBaseModel(BaseModel):
     id: int  # every ORM model should have an id field
-    created_at: datetime
+    created_at: AwareDatetime
     model_config = ConfigDict(
         from_attributes=True,
         populate_by_name=True,

schemas_v2/asset.py

@@ -13,9 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-from datetime import datetime
-
-from pydantic import BaseModel
+from pydantic import BaseModel, AwareDatetime
 
 
 class BaseAsset(BaseModel):
@@ -42,7 +40,7 @@ class AssetResponse(BaseAsset):
     # storage_path: str
     # mime_type: str
     # size: int
-    created_at: datetime
+    created_at: AwareDatetime
     storage_service: str
 
 

schemas_v2/contact.py

@@ -13,13 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-from datetime import datetime
 from typing import Optional, List
 
 import phonenumbers
 from email_validator import validate_email, EmailNotValidError
 from phonenumbers import NumberParseException
-from pydantic import field_validator, BaseModel
+from pydantic import field_validator, BaseModel, AwareDatetime
 
 from schemas import ORMBaseModel
 from schemas_v2.thing import ThingResponse
@@ -193,7 +192,7 @@ class ContactResponse(ORMBaseModel):
     id: int
     name: str
     role: str
-    created_at: datetime
+    created_at: AwareDatetime
     emails: List[EmailResponse] = []
     phones: List[PhoneResponse] = []
     addresses: List[AddressResponse] = []

schemas_v2/observation.py

@@ -13,19 +13,40 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-from datetime import datetime
-
-from pydantic import BaseModel
+from datetime import timezone
+from pydantic import BaseModel, AwareDatetime, PastDatetime, field_validator
+from typing import Annotated
 
 
 # class GeothermalMixin:
 #     depth: float
 #     temperature: float
 
 
+# -------- VALIDATE -------
+
+
+class ValidateObservation(BaseModel):
+
+    @field_validator("observation_timestamp", check_fields=False)
+    def convert_observation_timestamp_to_utc(
+        observation_timestamp: AwareDatetime,
+    ) -> AwareDatetime:
+        """
+        Convert observation_timestamp to UTC timezone if it's not already. This runs after
+        the Annotated validator PastDatetime() is run.
+        """
+        if (
+            observation_timestamp is not None
+            and observation_timestamp.tzinfo != timezone.utc
+        ):
+            return observation_timestamp.astimezone(timezone.utc)
+        return observation_timestamp
+
+
 # -------- CREATE ----------
-class CreateBaseObservation(BaseModel):
-    observation_timestamp: datetime
+class CreateBaseObservation(ValidateObservation):
+    observation_timestamp: Annotated[AwareDatetime, PastDatetime()]
     sample_id: int
     sensor_id: int
     observed_property: str
@@ -61,9 +82,9 @@ class BaseObservationResponse(BaseModel):
     id: int
     sample_id: int
     sensor_id: int
-    observation_timestamp: datetime
+    observation_timestamp: AwareDatetime
     observed_property: str
-    created_at: datetime
+    created_at: AwareDatetime
     release_status: str
 
 

schemas_v2/sample.py

@@ -13,8 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-from datetime import datetime, timezone
-from pydantic import BaseModel, field_validator, model_validator
+from datetime import timezone
+from pydantic import (
+    BaseModel,
+    field_validator,
+    model_validator,
+    AwareDatetime,
+    PastDatetime,
+)
+from typing import Annotated
 from typing_extensions import Self
 
 
@@ -30,16 +37,6 @@ class ValidateSample(BaseModel):
     Validator for Sample data for Create and Update schemas.
     """
 
-    @field_validator("sample_date", check_fields=False)
-    def validate_sample_date(cls, sample_date: datetime | None) -> datetime | None:
-        """
-        Validate that the sample_date is not in the future.
-        """
-        if sample_date is not None:
-            if sample_date > datetime.now(tz=timezone.utc):
-                raise ValueError(f"Sample date {sample_date} cannot be in the future.")
-        return sample_date
-
     # # REFACTOR TODO: is below ground negative or positive? the combine this with validate_sample_bottom defined below
     # @field_validator("sample_bottom", check_fields=False)
     # def validate_sample_bottom(cls, sample_bottom: float | None, values) -> float | None:
@@ -73,13 +70,23 @@ def validate_top_and_bottom(self) -> Self:
             )
         return self
 
+    @field_validator("sample_date", check_fields=False)
+    def convert_sample_date_to_utc(sample_date: AwareDatetime) -> AwareDatetime:
+        """
+        Convert sample_date to UTC timezone if it's not already. This runs after
+        the Annotated validator PastDatetime() is run.
+        """
+        if sample_date is not None and sample_date.tzinfo != timezone.utc:
+            return sample_date.astimezone(timezone.utc)
+        return sample_date
+
 
 # -------- CREATE ----------
 class CreateSample(ValidateSample):
     thing_id: int
     sample_type: str
     field_sample_id: str
-    sample_date: datetime
+    sample_date: Annotated[AwareDatetime, PastDatetime()]
     release_status: str
     sampler_name: str  # REFACTOR TODO: update with enum/restricted values
     qc_sample: str = "Original"
@@ -111,7 +118,7 @@ class UpdateSample(ValidateSample):
     thing_id: int = None  # REFACTOR TODO: should users be able to change this?
     sample_type: str = None
     field_sample_id: str = None
-    sample_date: datetime = None
+    sample_date: Annotated[AwareDatetime, PastDatetime()] = None
     release_status: str = None
     sampler_name: str = None  # REFACTOR TODO: update with enum/restricted values
     qc_sample: str = None
@@ -138,7 +145,7 @@ class SampleResponse(BaseModel):
     thing_id: int
     sample_type: str
     field_sample_id: str
-    sample_date: datetime
+    sample_date: AwareDatetime
     release_status: str
     sampler_name: str
     qc_sample: str

tests/conftest.py

@@ -49,7 +49,7 @@ def sensor():
 def sample(thing, sensor):
     with session_ctx() as session:
         sample = Sample(
-            sample_date="2025-01-01T00:00:00",
+            sample_date="2025-01-01T00:00:00Z",
             thing_id=thing.id,
             sample_type="groundwater",
             sampler_name="Test Sampler",

tests/test_sample.py

@@ -51,14 +51,6 @@ def second_sample(thing, sensor):
 # ============== Custom validators =================================================
 
 
-def test_validate_sample_date():
-    invalid_sample_date = "3500-01-01T00:00:00Z"
-    try:
-        invalid_sample = ValidateSample(sample_date=invalid_sample_date)
-    except ValueError as e:
-        assert str(e) == f"Sample date {invalid_sample_date} is not valid."
-
-
 def test_validate_sample_top_and_bottom():
     for i in range(2):
         sample_top = 10.0 if i == 0 else None
@@ -103,7 +95,7 @@ def test_add_sample(thing, sensor):
     assert data["thing_id"] == payload["thing_id"]
     assert data["sample_type"] == payload["sample_type"]
     assert data["field_sample_id"] == payload["field_sample_id"]
-    assert data["sample_date"] == payload["sample_date"][:-1]
+    assert data["sample_date"] == payload["sample_date"]
     assert data["release_status"] == payload["release_status"]
     assert data["sampler_name"] == payload["sampler_name"]
     assert data["qc_sample"] == payload["qc_sample"]
@@ -200,7 +192,7 @@ def test_patch_sample(sample):
 
     assert data["id"] == sample.id
     assert data["sampler_name"] == new_sampler_name
-    assert data["sample_date"] == new_sample_date[:-1]
+    assert data["sample_date"] == new_sample_date
     assert data["sample_method"] == new_sample_method
 
     # rollback after updating the sample