Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ dependency.
## Installation

```bash
pip install dataconnect # core (pyarrow + pydantic + httpx)
pip install dataconnect # core (pyarrow)
pip install dataconnect[pandas] # + pandas for .to_pandas() on results
Comment thread
butsyk-mdsol marked this conversation as resolved.
```

Expand All @@ -25,7 +25,8 @@ Requires **Python ≥ 3.13**.
## Quick start

```python
import pyarrow as pa
from uuid import UUID

from dataconnect import DataConnectClient


Expand All @@ -36,6 +37,9 @@ with DataConnectClient.connect(
) as client:

studies = client.get_studies(search_study_name="ACME")

datasets = client.get_datasets(study_environment_uuid=UUID("cec9f2a7-07ba-4fa8-bfcf-34fbc5d56793"))

```
## Development

Expand Down
27 changes: 26 additions & 1 deletion dataconnect/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from types import TracebackType
from uuid import UUID

from dataconnect.models import DatasetVersion, Study
from dataconnect.models import Dataset, DatasetVersion, Study
from dataconnect.service import DataConnectService, DefaultDataConnectService

_DEFAULT_HOST = "enodia-gateway.platform.imedidata.com"
Expand Down Expand Up @@ -51,6 +51,31 @@ def get_dataset_versions(self, dataset_uuid: UUID) -> list[DatasetVersion]:
"""List the dataset versions the client is authorized to access."""
return self._service.get_dataset_versions(dataset_uuid)

def get_datasets(
self,
study_environment_uuid: UUID,
search_dataset_name: str = "",
page: int = 1,
page_size: int = 50,
) -> list[Dataset]:
"""List datasets for a study environment.

Args:
study_environment_uuid: UUID of the study environment (required).
search_dataset_name: Full or partial dataset name filter.
page: Page number for paginated results.
page_size: Number of results per page.

Returns:
A list of :class:`Dataset` items matching the criteria.
"""
return self._service.get_datasets(
study_environment_uuid=study_environment_uuid,
search_dataset_name=search_dataset_name,
page=page,
page_size=page_size,
)

# Lifecycle

def close(self) -> None:
Expand Down
31 changes: 31 additions & 0 deletions dataconnect/models.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
from __future__ import annotations

from dataclasses import dataclass, field
from typing import Generic, TypeVar
from uuid import UUID

T = TypeVar("T")


@dataclass(frozen=True)
class StudyEnvironment:
Expand All @@ -24,3 +27,31 @@ class DatasetVersion:
dataset_uuid: UUID
dataset_name: str
dataset_version: str


@dataclass(frozen=True)
class Dataset:
"""A dataset belonging to a study environment."""

dataset_uuid: str
study_uuid: str
study_env_uuid: str
Comment thread
butsyk-mdsol marked this conversation as resolved.
dataset_name: str


Comment thread
butsyk-mdsol marked this conversation as resolved.
Comment thread
butsyk-mdsol marked this conversation as resolved.
@dataclass(frozen=True)
class Pagination:
"""Server-side pagination metadata."""

page: int
page_size: int
total_pages: int


@dataclass
class PaginatedResponse(Generic[T]): # noqa: UP046
"""A paginated collection returned by list endpoints."""

total_records: int
pagination: Pagination
items: list[T]
Comment thread
butsyk-mdsol marked this conversation as resolved.
11 changes: 10 additions & 1 deletion dataconnect/service/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from abc import ABC, abstractmethod
from uuid import UUID

from dataconnect.models import DatasetVersion, Study
from dataconnect.models import Dataset, DatasetVersion, Study


class DataConnectService(ABC):
Expand All @@ -14,6 +14,15 @@ class DataConnectService(ABC):
@abstractmethod
def get_studies(self, search_study_name: str | None = None) -> list[Study]: ...

@abstractmethod
def get_datasets(
self,
study_environment_uuid: UUID,
search_dataset_name: str = "",
page: int = 1,
page_size: int = 50,
) -> list[Dataset]: ...

@abstractmethod
def get_dataset_versions(self, dataset_uuid: UUID) -> list[DatasetVersion]: ...

Expand Down
48 changes: 46 additions & 2 deletions dataconnect/service/default.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
ServerError,
ValidationError,
)
from dataconnect.models import DatasetVersion, Study
from dataconnect.models import Dataset, DatasetVersion, Study
from dataconnect.service.base import DataConnectService
from dataconnect.service.mappers import resource_to_dataset_version, resource_to_study
from dataconnect.service.mappers import resource_to_dataset, resource_to_dataset_version, resource_to_study
from dataconnect.service.validators import validate_search_study_name
from dataconnect.transport.base import Transport
from dataconnect.transport.errors import (
Expand All @@ -32,6 +32,7 @@

# Server action identifiers
_ACTION_LIST_STUDIES = "studies.list"
_ACTION_LIST_DATASETS = "datasets.list"
_ACTION_LIST_DATASET_VERSIONS = "dataset_versions.list"


Expand Down Expand Up @@ -100,6 +101,49 @@ def get_dataset_versions(self, dataset_uuid: UUID) -> list[DatasetVersion]:
except (IndexError, KeyError, TypeError, ValueError) as ex:
raise ValidationError(f"Unexpected dataset versions response format: {ex}") from ex

def get_datasets(
self,
study_environment_uuid: UUID,
search_dataset_name: str = "",
page: int = 1,
page_size: int = 50,
Comment thread
butsyk-mdsol marked this conversation as resolved.
) -> list[Dataset]:
Comment thread
butsyk-mdsol marked this conversation as resolved.
"""List datasets for a study environment.

Args:
study_environment_uuid: UUID of the study environment (required).
search_dataset_name: Full or partial dataset name filter.
page: Page number for paginated results.
page_size: Number of results per page.

Returns:
A list of :class:`Dataset` items matching the criteria.
"""
Comment thread
butsyk-mdsol marked this conversation as resolved.
if not isinstance(study_environment_uuid, UUID):
raise ValidationError("study_environment_uuid must be a valid UUID")

if study_environment_uuid.int == 0:
raise ValidationError("study_environment_uuid must not be empty")

Comment thread
butsyk-mdsol marked this conversation as resolved.
request = ResourceQuery(action=_ACTION_LIST_DATASETS).append_body(
{
"study_environment_uuid": str(study_environment_uuid),
"search_dataset_name": search_dataset_name,
"page": page,
"page_size": page_size,
}
)

Comment thread
butsyk-mdsol marked this conversation as resolved.
try:
resources = self._transport.list_resources(request)
except TransportError as ex:
raise _translate_error(ex) from ex

try:
return [resource_to_dataset(r) for r in resources]
except (IndexError, KeyError, TypeError, ValueError) as ex:
raise ValidationError(f"Unexpected datasets response format: {ex}") from ex
Comment thread
butsyk-mdsol marked this conversation as resolved.

def close(self) -> None:

try:
Expand Down
18 changes: 17 additions & 1 deletion dataconnect/service/mappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from uuid import UUID

from dataconnect.exceptions import NotFoundError
from dataconnect.models import DatasetVersion, Study, StudyEnvironment
from dataconnect.models import Dataset, DatasetVersion, Study, StudyEnvironment
from dataconnect.transport.models import ResourceInfo


Expand Down Expand Up @@ -45,3 +45,19 @@ def resource_to_dataset_version(resource: ResourceInfo) -> DatasetVersion:
dataset_name=data["dataset_name"],
dataset_version=data["dataset_version"],
)


def resource_to_dataset(resource: ResourceInfo) -> Dataset:
"""Parse a transport-layer ``ResourceInfo`` into a ``Dataset`` domain object."""

if not resource or not resource.endpoints or not resource.endpoints[0].ticket:
raise NotFoundError("Invalid resource: missing endpoints or ticket")

data = json.loads(resource.endpoints[0].ticket.decode("utf-8"))

return Dataset(
dataset_uuid=data.get("dataset_uuid", ""),
study_uuid=data.get("study_uuid", ""),
study_env_uuid=data.get("study_env_uuid", ""),
Comment thread
butsyk-mdsol marked this conversation as resolved.
dataset_name=data.get("dataset_name", ""),
Comment thread
butsyk-mdsol marked this conversation as resolved.
)
1 change: 1 addition & 0 deletions dataconnect/transport/arrow_flight/transport.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ def _to_resource_info(info: flight.FlightInfo) -> ResourceInfo:
# Maps service-layer action names to the flight_type value the Arrow Flight server expects.
_ACTION_FLIGHT_TYPE: dict[str, str] = {
"studies.list": "STUDIES",
"datasets.list": "DATASETS",
"dataset_versions.list": "VERSIONS",
}

Expand Down
58 changes: 56 additions & 2 deletions tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,22 @@
import pytest

from dataconnect.client import DataConnectClient
from dataconnect.models import DatasetVersion, Study
from dataconnect.models import Dataset, DatasetVersion, Study


class _FakeService:
def __init__(self, studies: list[Study] | None = None, versions: list[DatasetVersion] | None = None) -> None:
def __init__(
self,
studies: list[Study] | None = None,
versions: list[DatasetVersion] | None = None,
datasets: list[Dataset] | None = None,
) -> None:
self._studies = studies or []
self._versions = versions or []
self._datasets = datasets or []
self.closed = 0
self.last_dataset_uuid: UUID | None = None
self.last_get_datasets_kwargs: dict[str, object] | None = None

def get_studies(self) -> list[Study]:
return self._studies
Expand All @@ -24,6 +31,10 @@ def get_dataset_versions(self, dataset_uuid: UUID) -> list[DatasetVersion]:
self.last_dataset_uuid = dataset_uuid
return self._versions

def get_datasets(self, **kwargs: object) -> list[Dataset]:
self.last_get_datasets_kwargs = kwargs
return self._datasets

def close(self) -> None:
self.closed += 1

Expand Down Expand Up @@ -144,3 +155,46 @@ def test_close_delegates_to_service() -> None:
client.close()

assert service.was_closed


def test_get_datasets_forwards_arguments_to_service() -> None:
datasets = [
Dataset(
dataset_uuid="073410b6-79be-3e7d-ae37-92f6e054013e",
study_uuid="64a98a9b-1512-44c8-92af-e4cab0183670",
study_env_uuid="4d1fd10d-5b57-4fd8-a436-f4ec59ce2e4a",
dataset_name="labs",
)
]
service = _FakeService(datasets=datasets)
client = DataConnectClient(service)

result = client.get_datasets(
study_environment_uuid=UUID("4d1fd10d-5b57-4fd8-a436-f4ec59ce2e4a"),
search_dataset_name="labs",
page=2,
page_size=10,
)

assert result == datasets
assert service.last_get_datasets_kwargs == {
"study_environment_uuid": UUID("4d1fd10d-5b57-4fd8-a436-f4ec59ce2e4a"),
"search_dataset_name": "labs",
"page": 2,
"page_size": 10,
}


def test_get_datasets_uses_defaults() -> None:
service = _FakeService()
client = DataConnectClient(service)

result = client.get_datasets(study_environment_uuid=UUID("11111111-1111-1111-1111-111111111111"))

assert result == []
assert service.last_get_datasets_kwargs == {
"study_environment_uuid": UUID("11111111-1111-1111-1111-111111111111"),
"search_dataset_name": "",
"page": 1,
"page_size": 50,
}
Loading
Loading