Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion dataconnect/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
ServerError,
ValidationError,
)
from dataconnect.models import DatasetVersion, PaginatedResponse, Pagination, Study, StudyEnvironment
from dataconnect.models import DatasetVersion, DryPublishResult, PaginatedResponse, Pagination, Study, StudyEnvironment

__all__ = [
# Client
Expand All @@ -20,6 +20,7 @@
"Study",
"StudyEnvironment",
"DatasetVersion",
"DryPublishResult",
"PaginatedResponse",
"Pagination",
# Exceptions — catch these in user application code
Expand Down
21 changes: 20 additions & 1 deletion dataconnect/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

import pandas as pd

from dataconnect.models import Dataset, DatasetVersion, PaginatedResponse, StudiesResult
from dataconnect.models import Dataset, DatasetVersion, DryPublishResult, PaginatedResponse, StudiesResult
from dataconnect.service import DataConnectService, DefaultDataConnectService

_DEFAULT_HOST = "enodia-gateway.platform.imedidata.com"
Expand Down Expand Up @@ -87,6 +87,25 @@ def get_datasets(
page_size=page_size,
)

def dry_publish(
self,
project_token: str,
dataset_name: str,
key_columns: list[str],
source_datasets: list[UUID],
data: pd.DataFrame,
datetime_formats: dict[str, str] | None = None,
) -> DryPublishResult:

return self._service.dry_publish(
project_token=project_token,
dataset_name=dataset_name,
key_columns=key_columns,
source_datasets=source_datasets,
data=data,
datetime_formats=datetime_formats,
)

# Lifecycle

def close(self) -> None:
Expand Down
21 changes: 21 additions & 0 deletions dataconnect/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
from typing import Generic, TypeVar
from uuid import UUID

import pandas as pd

T = TypeVar("T")


Expand Down Expand Up @@ -61,3 +63,22 @@ class PaginatedResponse(Generic[T]): # noqa: UP046
total_records: int
pagination: Pagination
items: list[T]


@dataclass
class DryPublishResult:
"""Result of a dry publish operation, including validation status and details."""

status: bool
is_schema_valid: bool | None = None
is_config_valid: bool | None = None
is_dataset_valid: bool | None = None
errors: list[str] = field(default_factory=list)
invalid_datetime_formats: dict[str, str] = field(default_factory=dict)
dataset_name: str | None = None
dataset_version: int | None = None
no_of_columns: int | None = None
valid_record_count: int | None = None
duplicate_record_count: int | None = None
invalid_record_count: int | None = None
invalid_records: pd.DataFrame | None = None
13 changes: 12 additions & 1 deletion dataconnect/service/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import pandas as pd

from dataconnect.models import Dataset, DatasetVersion, PaginatedResponse, StudiesResult
from dataconnect.models import Dataset, DatasetVersion, DryPublishResult, PaginatedResponse, StudiesResult


class DataConnectService(ABC):
Expand Down Expand Up @@ -35,5 +35,16 @@ def fetch_data(
first_n_rows: int | None = None,
) -> pd.DataFrame: ...

@abstractmethod
def dry_publish(
self,
project_token: str,
dataset_name: str,
key_columns: list[str],
source_datasets: list[UUID],
data: pd.DataFrame,
datetime_formats: dict[str, str] | None = None,
) -> DryPublishResult: ...

@abstractmethod
def close(self) -> None: ...
71 changes: 69 additions & 2 deletions dataconnect/service/default.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,24 @@

from __future__ import annotations

import json
from uuid import UUID

import pandas as pd

from dataconnect.models import Dataset, DatasetVersion, PaginatedResponse, Pagination, StudiesResult
from dataconnect.models import Dataset, DatasetVersion, DryPublishResult, PaginatedResponse, Pagination, StudiesResult
from dataconnect.service.base import DataConnectService
from dataconnect.service.error_handler import translate_error
from dataconnect.service.mappers import (
dry_publish_response_to_domain,
resource_to_dataset,
resource_to_dataset_version,
resource_to_fetched_data,
resource_to_study,
)
from dataconnect.transport.base import Transport
from dataconnect.transport.errors import TransportError
from dataconnect.transport.models import DatasetTicket, ResourceQuery
from dataconnect.transport.models import DatasetTicket, PublishRequest, ResourceQuery

# Server action identifiers
_ACTION_LIST_STUDIES = "studies.list"
Expand Down Expand Up @@ -138,6 +140,71 @@ def get_datasets(
except TransportError as ex:
raise translate_error(ex) from ex

def dry_publish(
self,
project_token: str,
dataset_name: str,
key_columns: list[str],
source_datasets: list[UUID],
data: pd.DataFrame,
datetime_formats: dict[str, str] | None = None,
) -> DryPublishResult:
"""Validate a dataset against the server without committing any changes.

Encodes the publish configuration as a JSON ``input_config`` string,
sets ``is_dry_publish: True`` so the server treats the call as a
validation-only run, then delegates to the transport layer.

The server response is mapped to a :class:`DryPublishResult` via
:func:`dry_publish_response_to_domain`.

Args:
project_token: Base64-encoded project token identifying the target
study, study environment, and project.
dataset_name: Name of the dataset to validate.
key_columns: Column names that form the unique key for the dataset.
source_datasets: UUIDs of the source datasets the published dataset
is derived from.
data: The dataset to validate as a ``pd.DataFrame``.
datetime_formats: Optional mapping of column name → datetime format
string (e.g. ``{"visit_date": "yyyy-MM-dd"}``). Defaults to
an empty dict when omitted.

Returns:
A :class:`DryPublishResult` containing the server's validation
outcome, including per-field validity flags, error messages, and
an optional ``invalid_records`` DataFrame.

Raises:
DataConnectError: Any :class:`TransportError` from the transport
layer is translated by :func:`translate_error` into the public
API's :class:`DataConnectError` hierarchy before propagating,
for example :class:`ValidationError`,
:class:`AuthorizationError`, or :class:`ServerError`."""

request = PublishRequest(
input_config=json.dumps(
{
"is_dry_publish": True,
"project_token": project_token,
"dataset_name": dataset_name,
"dataset_description": dataset_name,
"key_columns": key_columns,
"source_datasets": [str(uuid) for uuid in source_datasets],
"datetime_formats": datetime_formats or {},
}
),
data=data,
)

try:
publish_result = self._transport.dry_publish_dataset(request)

return dry_publish_response_to_domain(publish_result)

except TransportError as ex:
raise translate_error(ex) from ex

def close(self) -> None:
"""Close the underlying transport connection."""

Expand Down
43 changes: 41 additions & 2 deletions dataconnect/service/mappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
import pyarrow as pa

from dataconnect.exceptions import NotFoundError
from dataconnect.models import Dataset, DatasetVersion, Study, StudyEnvironment
from dataconnect.transport.models import DataTable, ResourceInfo
from dataconnect.models import Dataset, DatasetVersion, DryPublishResult, Study, StudyEnvironment
from dataconnect.transport.models import DataTable, DryPublishResponse, ResourceInfo


def resource_to_study(resource: ResourceInfo) -> Study:
Expand Down Expand Up @@ -85,3 +85,42 @@ def resource_to_dataset(resource: ResourceInfo) -> Dataset:
study_env_uuid=data.get("study_env_uuid", ""),
dataset_name=data.get("dataset_name", ""),
)


def dry_publish_response_to_domain(result: DryPublishResponse | None) -> DryPublishResult:
"""Map a transport-layer ``DryPublishResponse`` to a ``DryPublishResult`` domain object.

``DryPublishResponse`` carries flat, typed fields returned by the server after a
dry-publish call. The mapping is direct for all shared fields with one
exception:

* ``DryPublishResponse.dataset_valid`` → ``DryPublishResult.is_dataset_valid``
(renamed for naming consistency with the other ``is_*_valid`` fields).

Args:
result: The transport-layer result returned by
:meth:`Transport.dry_publish_dataset`. Pass ``None`` to obtain a
default :class:`DryPublishResult` with ``status=False`` and all
other fields at their zero values.

Returns:
A :class:`DryPublishResult` suitable for returning to the caller.
"""
if result is None:
return DryPublishResult(status=False)

return DryPublishResult(
status=result.status,
is_schema_valid=result.is_schema_valid,
is_config_valid=result.is_config_valid,
is_dataset_valid=result.dataset_valid,
errors=result.errors,
invalid_datetime_formats=result.invalid_datetime_formats,
dataset_name=result.dataset_name,
dataset_version=result.dataset_version,
no_of_columns=result.no_of_columns,
valid_record_count=result.valid_record_count,
duplicate_record_count=result.duplicate_record_count,
invalid_record_count=result.invalid_record_count,
invalid_records=result.invalid_records,
)
14 changes: 13 additions & 1 deletion dataconnect/transport/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,23 @@
"""

from dataconnect.transport.base import Transport
from dataconnect.transport.models import DataRef, ResourceInfo, ResourceQuery
from dataconnect.transport.models import (
DataRef,
DatasetTicket,
DataTable,
DryPublishResponse,
PublishRequest,
ResourceInfo,
ResourceQuery,
)

__all__ = [
"Transport",
"ResourceQuery",
"ResourceInfo",
"DataRef",
"DatasetTicket",
"DataTable",
"PublishRequest",
"DryPublishResponse",
]
Loading
Loading