From 8bcbf95d1878ba0eb8dfb38f483a6e811ddf5577 Mon Sep 17 00:00:00 2001 From: Anton Butsyk Date: Tue, 12 May 2026 16:39:00 +0300 Subject: [PATCH] [MCC-1496700] Python get_datasets() pagination support --- README.md | 3 +- dataconnect/__init__.py | 4 +- dataconnect/client.py | 6 +- dataconnect/service/base.py | 4 +- dataconnect/service/default.py | 49 ++---- dataconnect/service/validators.py | 38 +++++ tests/test_get_datasets_paginated.py | 217 +++++++++++++++++++++++++++ 7 files changed, 281 insertions(+), 40 deletions(-) create mode 100644 tests/test_get_datasets_paginated.py diff --git a/README.md b/README.md index b6a4e0c..8577f70 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ with DataConnectClient.connect( studies = client.get_studies(search_study_name="ACME") - datasets = client.get_datasets(study_environment_uuid=UUID("cec9f2a7-07ba-4fa8-bfcf-34fbc5d56793")) + pagination = client.get_datasets(study_environment_uuid=UUID("cec9f2a7-07ba-4fa8-bfcf-34fbc5d56793")) + datasets = pagination.items ``` ## Development diff --git a/dataconnect/__init__.py b/dataconnect/__init__.py index 664a36d..6f9f0a0 100644 --- a/dataconnect/__init__.py +++ b/dataconnect/__init__.py @@ -11,7 +11,7 @@ ServerError, ValidationError, ) -from dataconnect.models import DatasetVersion, Study, StudyEnvironment +from dataconnect.models import DatasetVersion, PaginatedResponse, Pagination, Study, StudyEnvironment __all__ = [ # Client @@ -20,6 +20,8 @@ "Study", "StudyEnvironment", "DatasetVersion", + "PaginatedResponse", + "Pagination", # Exceptions — catch these in user application code "DataConnectError", "AuthenticationError", diff --git a/dataconnect/client.py b/dataconnect/client.py index 6702a97..eb9df21 100644 --- a/dataconnect/client.py +++ b/dataconnect/client.py @@ -12,7 +12,7 @@ import pandas as pd -from dataconnect.models import Dataset, DatasetVersion, Study +from dataconnect.models import Dataset, DatasetVersion, PaginatedResponse, Study from dataconnect.service import DataConnectService, DefaultDataConnectService _DEFAULT_HOST = "enodia-gateway.platform.imedidata.com" @@ -67,7 +67,7 @@ def get_datasets( search_dataset_name: str = "", page: int = 1, page_size: int = 50, - ) -> list[Dataset]: + ) -> PaginatedResponse[Dataset]: """List datasets for a study environment. Args: @@ -77,7 +77,7 @@ def get_datasets( page_size: Number of results per page. Returns: - A list of :class:`Dataset` items matching the criteria. + A :class:`PaginatedResponse` of :class:`Dataset` items matching the criteria. """ return self._service.get_datasets( study_environment_uuid=study_environment_uuid, diff --git a/dataconnect/service/base.py b/dataconnect/service/base.py index 6709ed9..cf068fb 100644 --- a/dataconnect/service/base.py +++ b/dataconnect/service/base.py @@ -7,7 +7,7 @@ import pandas as pd -from dataconnect.models import Dataset, DatasetVersion, Study +from dataconnect.models import Dataset, DatasetVersion, PaginatedResponse, Study class DataConnectService(ABC): @@ -23,7 +23,7 @@ def get_datasets( search_dataset_name: str = "", page: int = 1, page_size: int = 50, - ) -> list[Dataset]: ... + ) -> PaginatedResponse[Dataset]: ... @abstractmethod def get_dataset_versions(self, dataset_uuid: UUID) -> list[DatasetVersion]: ... diff --git a/dataconnect/service/default.py b/dataconnect/service/default.py index bb3cf1c..61d41fd 100644 --- a/dataconnect/service/default.py +++ b/dataconnect/service/default.py @@ -2,13 +2,11 @@ from __future__ import annotations -from datetime import UTC, datetime from uuid import UUID import pandas as pd -from dataconnect.exceptions import ValidationError -from dataconnect.models import Dataset, DatasetVersion, Study +from dataconnect.models import Dataset, DatasetVersion, PaginatedResponse, Pagination, Study from dataconnect.service.base import DataConnectService from dataconnect.service.error_handler import translate_error from dataconnect.service.mappers import ( @@ -17,6 +15,7 @@ resource_to_fetched_data, resource_to_study, ) +from dataconnect.service.validators import validate_positive_int, validate_uuid from dataconnect.transport.base import Transport from dataconnect.transport.errors import TransportError from dataconnect.transport.models import ResourceQuery @@ -69,20 +68,7 @@ def get_dataset_versions(self, dataset_uuid: UUID) -> list[DatasetVersion]: Raises: ValidationError: If *dataset_uuid* is not a valid, non-zero UUID. """ - # Input validation: ensure callers pass a UUID - if not isinstance(dataset_uuid, UUID): - raise ValidationError( - error_code="VAL_C_DATASET_UUID", - message="dataset_uuid must be a valid UUID", - timestamp=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), - ) - - if dataset_uuid.int == 0: - raise ValidationError( - error_code="VAL_C_DATASET_UUID", - message="dataset_uuid must not be empty", - timestamp=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), - ) + validate_uuid(dataset_uuid, field_name="dataset_uuid", error_code="VAL_C_DATASET_UUID") request = ResourceQuery(action=_ACTION_LIST_DATASET_VERSIONS).append_body({"dataset_uuid": str(dataset_uuid)}) @@ -124,7 +110,7 @@ def get_datasets( search_dataset_name: str = "", page: int = 1, page_size: int = 50, - ) -> list[Dataset]: + ) -> PaginatedResponse[Dataset]: """List datasets for a study environment. Args: @@ -134,21 +120,11 @@ def get_datasets( page_size: Number of results per page. Returns: - A list of :class:`Dataset` items matching the criteria. + A :class:`PaginatedResponse` of :class:`Dataset` items matching the criteria. """ - if not isinstance(study_environment_uuid, UUID): - raise ValidationError( - error_code="VAL_C_STUDY_ENV_UUID", - message="study_environment_uuid must be a valid UUID.", - timestamp=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), - ) - - if study_environment_uuid.int == 0: - raise ValidationError( - error_code="VAL_C_STUDY_ENV_UUID", - message="study_environment_uuid must not be empty.", - timestamp=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), - ) + validate_uuid(study_environment_uuid, field_name="study_environment_uuid", error_code="VAL_C_STUDY_ENV_UUID") + validate_positive_int(page, field_name="page", error_code="VAL_C_PAGE") + validate_positive_int(page_size, field_name="page_size", error_code="VAL_C_PAGE_SIZE") request = ResourceQuery(action=_ACTION_LIST_DATASETS).append_body( { @@ -161,7 +137,14 @@ def get_datasets( try: resources = self._transport.list_resources(request) - return [resource_to_dataset(r) for r in resources] + items = [resource_to_dataset(r) for r in resources] + total_records = resources[0].total_records if resources else 0 + total_pages = (total_records + page_size - 1) // page_size if page_size > 0 else 0 + return PaginatedResponse( + total_records=total_records, + pagination=Pagination(page=page, page_size=page_size, total_pages=total_pages), + items=items, + ) except TransportError as ex: raise translate_error(ex) from ex diff --git a/dataconnect/service/validators.py b/dataconnect/service/validators.py index 5909d36..9daef24 100644 --- a/dataconnect/service/validators.py +++ b/dataconnect/service/validators.py @@ -2,6 +2,9 @@ from __future__ import annotations +from datetime import UTC, datetime +from uuid import UUID + from dataconnect.exceptions import ValidationError @@ -13,3 +16,38 @@ def validate_search_study_name(search_study_name: str | None) -> None: if not isinstance(search_study_name, str): raise ValidationError("search_study_name must be a string") + + +def validate_uuid(value: object, *, field_name: str, error_code: str) -> None: + """Ensure *value* is a non-zero UUID. + + Raises: + ValidationError: If *value* is not a UUID instance or is the nil UUID. + """ + if not isinstance(value, UUID): + raise ValidationError( + error_code=error_code, + message=f"{field_name} must be a valid UUID.", + timestamp=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + ) + + if value.int == 0: + raise ValidationError( + error_code=error_code, + message=f"{field_name} must not be empty.", + timestamp=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + ) + + +def validate_positive_int(value: object, *, field_name: str, error_code: str) -> None: + """Ensure *value* is an integer >= 1. + + Raises: + ValidationError: If *value* is not an int or is less than 1. + """ + if not isinstance(value, int) or value < 1: + raise ValidationError( + error_code=error_code, + message=f"{field_name} must be a positive integer.", + timestamp=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + ) diff --git a/tests/test_get_datasets_paginated.py b/tests/test_get_datasets_paginated.py new file mode 100644 index 0000000..235140e --- /dev/null +++ b/tests/test_get_datasets_paginated.py @@ -0,0 +1,217 @@ +"""Tests for get_datasets returning PaginatedResponse[Dataset].""" + +from __future__ import annotations + +import json +from uuid import UUID + +import pytest + +from dataconnect.exceptions import ValidationError +from dataconnect.models import Dataset, PaginatedResponse, Pagination +from dataconnect.service.default import DefaultDataConnectService +from dataconnect.transport.errors import TransportError +from dataconnect.transport.models import DataRef, ResourceInfo, ResourceQuery + + +class _FakeTransport: + def __init__( + self, + resources: list[ResourceInfo] | None = None, + error: Exception | None = None, + ) -> None: + self._resources = resources or [] + self._error = error + self.last_request: ResourceQuery | None = None + + def list_resources(self, request: ResourceQuery) -> list[ResourceInfo]: + self.last_request = request + if self._error is not None: + raise self._error + return self._resources + + def do_get(self, request: ResourceQuery) -> None: + pass + + def close(self) -> None: + return None + + +def _dataset_resource(payload: dict[str, str], total_records: int = 1) -> ResourceInfo: + return ResourceInfo( + descriptor=b"", + endpoints=[DataRef(ticket=json.dumps(payload).encode("utf-8"))], + total_records=total_records, + schema_bytes=b"", + ) + + +_STUDY_ENV_UUID = UUID("4d1fd10d-5b57-4fd8-a436-f4ec59ce2e4a") + + +class TestGetDatasetsReturnsPaginatedResponse: + """Verify get_datasets returns a PaginatedResponse[Dataset].""" + + def test_returns_paginated_response_with_items(self) -> None: + payload = { + "dataset_uuid": "073410b6-79be-3e7d-ae37-92f6e054013e", + "study_uuid": "64a98a9b-1512-44c8-92af-e4cab0183670", + "study_env_uuid": "4d1fd10d-5b57-4fd8-a436-f4ec59ce2e4a", + "dataset_name": "labs", + } + transport = _FakeTransport(resources=[_dataset_resource(payload, total_records=3)]) + service = DefaultDataConnectService(transport) + + result = service.get_datasets(study_environment_uuid=_STUDY_ENV_UUID) + + assert isinstance(result, PaginatedResponse) + assert result.total_records == 3 + assert result.items == [ + Dataset( + dataset_uuid="073410b6-79be-3e7d-ae37-92f6e054013e", + study_uuid="64a98a9b-1512-44c8-92af-e4cab0183670", + study_env_uuid="4d1fd10d-5b57-4fd8-a436-f4ec59ce2e4a", + dataset_name="labs", + ) + ] + + def test_pagination_metadata_uses_request_params(self) -> None: + payload = { + "dataset_uuid": "aaa", + "study_uuid": "bbb", + "study_env_uuid": "ccc", + "dataset_name": "vitals", + } + transport = _FakeTransport(resources=[_dataset_resource(payload, total_records=100)]) + service = DefaultDataConnectService(transport) + + result = service.get_datasets( + study_environment_uuid=_STUDY_ENV_UUID, + page=2, + page_size=25, + ) + + assert result.pagination == Pagination(page=2, page_size=25, total_pages=4) + + def test_total_pages_rounds_up(self) -> None: + payload = { + "dataset_uuid": "aaa", + "study_uuid": "bbb", + "study_env_uuid": "ccc", + "dataset_name": "vitals", + } + # 51 records / 25 per page = 3 pages (rounds up) + transport = _FakeTransport(resources=[_dataset_resource(payload, total_records=51)]) + service = DefaultDataConnectService(transport) + + result = service.get_datasets( + study_environment_uuid=_STUDY_ENV_UUID, + page=1, + page_size=25, + ) + + assert result.pagination.total_pages == 3 + + def test_empty_resources_returns_zero_totals(self) -> None: + transport = _FakeTransport(resources=[]) + service = DefaultDataConnectService(transport) + + result = service.get_datasets(study_environment_uuid=_STUDY_ENV_UUID) + + assert isinstance(result, PaginatedResponse) + assert result.total_records == 0 + assert result.items == [] + assert result.pagination == Pagination(page=1, page_size=50, total_pages=0) + + def test_builds_correct_request_body(self) -> None: + transport = _FakeTransport(resources=[]) + service = DefaultDataConnectService(transport) + + service.get_datasets( + study_environment_uuid=_STUDY_ENV_UUID, + search_dataset_name="vitals", + page=3, + page_size=10, + ) + + assert transport.last_request is not None + assert transport.last_request.action == "datasets.list" + body = json.loads(transport.last_request.body) + assert body == { + "study_environment_uuid": str(_STUDY_ENV_UUID), + "search_dataset_name": "vitals", + "page": 3, + "page_size": 10, + } + + def test_raises_validation_error_for_invalid_uuid(self) -> None: + transport = _FakeTransport(resources=[]) + service = DefaultDataConnectService(transport) + + with pytest.raises(ValidationError, match="study_environment_uuid must be a valid UUID"): + service.get_datasets(study_environment_uuid="not-a-uuid") # type: ignore[arg-type] + + def test_raises_validation_error_for_zero_uuid(self) -> None: + transport = _FakeTransport(resources=[]) + service = DefaultDataConnectService(transport) + + with pytest.raises(ValidationError, match="study_environment_uuid must not be empty"): + service.get_datasets(study_environment_uuid=UUID(int=0)) + + @pytest.mark.parametrize("page", [0, -1, -100]) + def test_raises_validation_error_for_invalid_page(self, page: int) -> None: + transport = _FakeTransport(resources=[]) + service = DefaultDataConnectService(transport) + + with pytest.raises(ValidationError, match="page must be a positive integer"): + service.get_datasets(study_environment_uuid=_STUDY_ENV_UUID, page=page) + + def test_raises_validation_error_for_non_int_page(self) -> None: + transport = _FakeTransport(resources=[]) + service = DefaultDataConnectService(transport) + + with pytest.raises(ValidationError, match="page must be a positive integer"): + service.get_datasets(study_environment_uuid=_STUDY_ENV_UUID, page="2") # type: ignore[arg-type] + + @pytest.mark.parametrize("page_size", [0, -1, -50]) + def test_raises_validation_error_for_invalid_page_size(self, page_size: int) -> None: + transport = _FakeTransport(resources=[]) + service = DefaultDataConnectService(transport) + + with pytest.raises(ValidationError, match="page_size must be a positive integer"): + service.get_datasets(study_environment_uuid=_STUDY_ENV_UUID, page_size=page_size) + + def test_raises_validation_error_for_non_int_page_size(self) -> None: + transport = _FakeTransport(resources=[]) + service = DefaultDataConnectService(transport) + + with pytest.raises(ValidationError, match="page_size must be a positive integer"): + service.get_datasets(study_environment_uuid=_STUDY_ENV_UUID, page_size="25") # type: ignore[arg-type] + + def test_translates_transport_errors(self) -> None: + transport = _FakeTransport(error=TransportError(error_code="CONN", message="cannot connect")) + service = DefaultDataConnectService(transport) + + with pytest.raises(Exception, match="cannot connect"): + service.get_datasets(study_environment_uuid=_STUDY_ENV_UUID) + + def test_multiple_items_returned(self) -> None: + resources = [ + _dataset_resource( + {"dataset_uuid": "aaa", "study_uuid": "bbb", "study_env_uuid": "ccc", "dataset_name": "labs"}, + total_records=2, + ), + _dataset_resource( + {"dataset_uuid": "ddd", "study_uuid": "eee", "study_env_uuid": "fff", "dataset_name": "vitals"}, + total_records=2, + ), + ] + transport = _FakeTransport(resources=resources) + service = DefaultDataConnectService(transport) + + result = service.get_datasets(study_environment_uuid=_STUDY_ENV_UUID, page_size=10) + + assert len(result.items) == 2 + assert result.total_records == 2 + assert result.items[0].dataset_name == "labs" + assert result.items[1].dataset_name == "vitals"