[MCC-1489710] Python Lib - Boilerplate Code [Dev Only]

.dockerignore

@@ -2,7 +2,7 @@
 *
 
 # Add back just needed directories
-!python_template
+!dataconnect
 !scripts
 !tests
 !pyproject.toml
@@ -14,4 +14,4 @@
 **/.git*
 **/.*project
 **/.DS_Store
-**/._*
+**/._*

.github/pull_request_template.md

@@ -9,7 +9,7 @@
 - [ ] Make sure the pull request does not have excessive number of unnecessary commits. Utilize the `git commit --amend --no-edit` command to reduce commit messages when making small file changes (*like changing linespacing*).
 - [ ] Make sure you have added unit tests for the code changes. Tests should be added in the `tests/` folder.
 - [ ] Modify docs, if required. Add any new documentation in the `doc/` folder.
-- [ ] Rebase on latest active development branch (develop/main). 
+- [ ] Rebase on latest active development branch (develop/main).
 
 ### Changes Summary
 

.github/workflows/ci.yml

@@ -29,7 +29,9 @@ jobs:
         key: ${{ runner.os }}-precommit-${{ hashFiles('.pre-commit-config.yaml') }}
 
     - name: lint
-      run: scripts/lint.sh
+      run: |
+        chmod +x scripts/lint.sh
+        scripts/lint.sh
 
     - name: build docker image
       working-directory: ${{ github.workspace }}
@@ -50,4 +52,3 @@ jobs:
 
     - name: typecheck
       run: docker run dataconnect-library-python "scripts/typecheck.sh"
-

.pre-commit-config.yaml

@@ -1,29 +1,44 @@
+# Pre-commit hook configuration
+# Docs: https://pre-commit.com/
+#
+# Install:
+#   pip install pre-commit        # or: poetry install --with lint
+#   pre-commit install            # register the git hook
+#
+# Run manually against all files:
+#   pre-commit run --all-files
+#
+# Update hook revisions to latest:
+#   pre-commit autoupdate
+
+default_language_version:
+  python: python3.11
+
 repos:
-  - repo: https://github.com/ambv/black
-    # rev must match pyproject.toml
-    rev: 23.3.0
+  # ---------------------------------------------------------------------------
+  # General hygiene
+  # ---------------------------------------------------------------------------
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
     hooks:
-      - id: black
-        args: ["-l", "100"]
-  - repo: https://github.com/myint/autoflake
-    # rev must match pyproject.toml
-    rev: v1.4 # rev depends on tag name in git not pip version number (thus the v prefix is required here)
+      - id: trailing-whitespace       # strip trailing whitespace
+      - id: end-of-file-fixer         # ensure files end with a newline
+      - id: check-yaml                # validate YAML syntax
+      - id: check-toml                # validate TOML syntax
+      - id: check-json                # validate JSON syntax
+      - id: check-merge-conflict      # forbid merge-conflict markers
+      - id: check-added-large-files   # warn on large committed files
+        args: ["--maxkb=1024"]
+      - id: debug-statements          # catch leftover debugger imports
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+
+  # ---------------------------------------------------------------------------
+  # Ruff – fast Python linter (replaces flake8, isort, pyupgrade, …)
+  # ---------------------------------------------------------------------------
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.11
     hooks:
-      - id: autoflake
-        args:
-          [
-            "--in-place",
-            "--remove-all-unused-imports",
-            "--ignore-init-module-imports",
-          ]
-  - repo: https://github.com/PyCQA/isort
-    # rev must match pyproject.toml
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        args: ["--profile", "black", "-l", "100"]
-#  - repo: git@github.com:mdsol/git-config-checker.git
-#    rev: 0.1.1
-#    hooks:
-#      - id: git-config-checker
-#        args: ["--checks", "emailDomain"]
+      - id: ruff                      # lint + auto-fix
+        args: ["--fix", "--exit-non-zero-on-fix"]
+      - id: ruff-format               # format (Black-compatible)

README.md

@@ -1,30 +1,49 @@
-# Data Connect Python Library
-Python library built by Medidata for connecting to Data Connect and programmatically querying and retrieving data.
+from dataconnect import DataConnectClientfrom dataconnect import DataConnectClient
 
+# dataconnect-library-python
+
+Python SDK for the [Medidata DataConnect](https://github.com/mdsol/dataconnect-library-r) service.
+
+---
+
+## Transport note
+
+The DataConnect service uses **Apache Arrow Flight** (gRPC binary protocol),
+**not** a plain REST/HTTP API.  `pyarrow.flight` is the primary transport
+dependency.
+
+---
 
 ## Installation
 
-See [CONTRIBUTING.md](./CONTRIBUTING.md) for installation instructions.
+```bash
+pip install dataconnect          # core (pyarrow + pydantic + httpx)
+pip install dataconnect[pandas]  # + pandas for .to_pandas() on results
+```
 
-## To Dos
+Requires **Python ≥ 3.10**.
 
-The following actions are needed when using this as the template for a new python project:
+---
 
-- [ ] Update `readme.md` for new project
-- [ ] Update `factbook.yaml` for new project
-- [ ] Update `pyproject.toml` with relevant `name`, `description`, and `authors` and remove unneeded dep groups
+## Quick start
 
-- [ ] Remove all example code files (leave `__init__.py` in place)
-- [X] Rename all references to `python-template` and `python_template` to new module/folder names
-- [ ] Remove all example test files and all code from `conftest.py` (leave the docstring in place)
-- [ ] Add placeholder tests so PR passes
-- [ ] Get GH Admins to add mdsol robot key so PR passes
-- [ ] Add Artifactory robot account info so PR passes
+```python
+import pyarrow as pa
+from dataconnect import DataConnectClient
 
-## Contributing
 
-See [CONTRIBUTING](CONTRIBUTING.md).
+with DataConnectClient.connect(
+    host="dataconnect.example.com",
+    port=443,
+    token="your-bearer-token",
+) as client:
 
-## Contact
+    studies = client.studies(search_study_name="ACME", page=1, page_size=10)
+    study = studies[0]
+```
+## Development
 
-See the [factbook](factbook.yaml).
+```bash
+pip install -e ".[dev]"
+pytest
+```

README_orig.md

@@ -0,0 +1,30 @@
+# Python Template
+
+This repository is meant to establish useful practices for python projects.
+
+## Installation
+
+See [CONTRIBUTING.md](./CONTRIBUTING.md) for installation instructions.
+
+## To Dos
+
+The following actions are needed when using this as the template for a new python project:
+
+- [ ] Update `readme.md` for new project
+- [ ] Update `factbook.yaml` for new project
+- [X] Update `pyproject.toml` with relevant `name`, `description`, and `authors` and remove unneeded dep groups
+- [ ] Update `python_template` folder to match new project name
+- [ ] Remove all example code files (leave `__init__.py` in place)
+- [X] Rename all references to `python-template` and `python_template` to new module/folder names
+- [X] Remove all example test files and all code from `conftest.py` (leave the docstring in place)
+- [ ] Add placeholder tests so PR passes
+- [ ] Get GH Admins to add mdsol robot key so PR passes
+- [ ] Add Artifactory robot account info so PR passes
+
+## Contributing
+
+See [CONTRIBUTING](CONTRIBUTING.md).
+
+## Contact
+
+See the [factbook](factbook.yaml).

dataconnect/__init__.py

@@ -0,0 +1,23 @@
+"""dataconnect — Python SDK for the Medidata DataConnect service."""
+
+from __future__ import annotations
+
+from dataconnect.client import DataConnectClient
+from dataconnect.exceptions import AuthenticationError, ConnectionError, DataConnectError
+from dataconnect.models import (
+    Dataset,
+    DatasetVersion,
+    Study,
+    StudyEnvironment,
+)
+
+__all__ = [
+    "DataConnectClient",
+    "Study",
+    "StudyEnvironment",
+    "Dataset",
+    "DatasetVersion",
+    "DataConnectError",
+    "ConnectionError",
+    "AuthenticationError",
+]

dataconnect/_encoding.py

@@ -0,0 +1,16 @@
+"""Encoding utilities for DataConnect."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def dumps(obj: Any) -> bytes:
+    """Serialize *obj* to JSON (bytes)."""
+    return json.dumps(obj, separators=(",", ":")).encode("utf-8")
+
+
+def loads(data: bytes) -> Any:
+    """Deserialize JSON from *data* (bytes)."""
+    return json.loads(data.decode("utf-8"))

dataconnect/auth.py

@@ -0,0 +1,16 @@
+"""Authentication and authorization utilities for DataConnect."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+class Credentials:
+    """Marker base class for all credentials types."""
+
+
+@dataclass(frozen=True)
+class BearerTokenAuth(Credentials):
+    """Bearer token authentication credentials (OAuth access token)."""
+
+    token: str

dataconnect/client.py

@@ -0,0 +1,93 @@
+"""Public API for the DataConnect client library."""
+
+from __future__ import annotations
+
+import json
+from types import TracebackType
+from typing import Any
+
+import pyarrow as pa
+
+from dataconnect import _encoding
+from dataconnect.auth import BearerTokenAuth
+from dataconnect.framework.pyarrow_transport import PyArrowFlightTransport
+from dataconnect.framework.transport import FlightTransport
+from dataconnect.models import Dataset, Study
+
+# Flight actions / commands
+_ACTION_LIST_STUDIES = "studies.list"
+_ACTION_LIST_DATASETS = "datasets.list"
+_ACTION_LIST_DATASET_VERSIONS = "dataset_versions.list"
+_ACTION_FETCH_TICKET = "data.fetch_ticket"
+_CMD_PUBLISH = "publish"
+_CMD_DRY_PUBLISH = "dry_publish"
+
+_DEFAULT_HOST = "enodia-gateway.platform.imedidata.com"
+_DEFAULT_PORT = 443
+
+
+class DataConnectClient:
+    """Client for interacting with DataConnect services."""
+
+    def __init__(self, transport: FlightTransport) -> None:
+        """Initialize the DataConnect client with a specified transport."""
+        self._transport = transport
+
+    @classmethod
+    def connect(
+        cls,
+        host: str = _DEFAULT_HOST,
+        port: int = _DEFAULT_PORT,
+        use_tls: bool = True,
+        token: str = "",
+    ) -> DataConnectClient:
+        """Open connection to a Flight server."""
+        location = f"grpc+tls://{host}:{port}"
+        transport = PyArrowFlightTransport(
+            location=location,
+            credentials=BearerTokenAuth(token),
+        )
+        return cls(transport)
+
+    def studies(self) -> list[Study]:
+        """List the studies the client is authorized to access."""
+        rows = self._action_json(_ACTION_LIST_STUDIES, None)
+        return [Study(**r) for r in rows]
+
+    def datasets(self, study_uuid: str) -> list[Dataset]:
+        """List the datasets available for a given study."""
+        body = {"study_uuid": study_uuid}
+        rows = self._action_json(_ACTION_LIST_DATASETS, {"study_uuid": body})
+        return [Dataset(**r) for r in rows]
+
+    def fetch_data(self, dataset_uuid: str) -> pa.Table:
+        """Fetch the data for a given dataset as a PyArrow Table."""
+        body = {"dataset_uuid": dataset_uuid}
+        results = self._transport.do_action(_ACTION_FETCH_TICKET, _encoding.dumps(body))
+        if not results:
+            raise RuntimeError("Server returned no data for the fetch_data action.")
+        return self._transport.do_get(results).read_all()
+
+    # Lifecycle
+    def close(self) -> None:
+        """Close the underlying transport connection."""
+        self._transport.close()
+
+    def __enter__(self) -> DataConnectClient:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        self._transport.close()
+
+    # Helpers
+    def _action_json(self, action: str, body: dict[str, Any] | None) -> Any:
+        """Execute a Flight action and return the result as JSON."""
+        results = self._transport.do_action(action, _encoding.dumps(body or {}))
+        if not results:
+            return []
+        return json.loads(results.decode("utf-8"))

dataconnect/exceptions.py

@@ -0,0 +1,19 @@
+"""Public exceptions for DataConnect."""
+
+from __future__ import annotations
+
+
+class DataConnectError(Exception):
+    """Base exception for all DataConnect client errors."""
+
+
+class ConnectionError(DataConnectError):
+    """Error connecting to the DataConnect server."""
+
+
+class AuthenticationError(DataConnectError):
+    """Error authenticating with the DataConnect server."""
+
+
+class QueryError(DataConnectError):
+    """Error executing a query."""